@catladder/pipeline 1.55.0 → 1.55.1

package/src/context/getEnvironmentContext.ts

@@ -0,0 +1,57 @@
+import type { Config, EnvConfigWithComponent } from "../types/config";
+
+import type { CommitInfo } from "../types/context";
+import type { EnvironmentContext } from "../types/environmentContext";
+import { getEnvConfig } from "./getEnvConfig";
+import { getEnvType } from "./getEnvType";
+
+const getEnvironmentSlug = (
+  envConfig: EnvConfigWithComponent,
+  env: string,
+  componentName: string,
+  commitInfo?: CommitInfo
+) => {
+  const envType = getEnvType(env, envConfig);
+
+  return envType === "review" && commitInfo
+    ? `${env}-${commitInfo.reviewSlug}-${componentName}`
+    : `${env}-${componentName}`;
+};
+
+export const getEnvironmentContext = (
+  config: Config,
+  env: string,
+  componentName: string,
+  commitInfo?: CommitInfo
+): EnvironmentContext<any, any> => {
+  const envConfigRaw = getEnvConfig(config, componentName, env);
+  const envType = getEnvType(env, envConfigRaw);
+
+  const environmentSlug = getEnvironmentSlug(
+    envConfigRaw,
+    env,
+    componentName,
+    commitInfo
+  );
+
+  const gitlabEnvironmentName =
+    envType === "review" && commitInfo
+      ? `${env}/${commitInfo.refName}/${componentName}`
+      : `${env}/${componentName}`;
+
+  const fullName = `${config.customerName}-${config.appName}-${environmentSlug}`;
+
+  return {
+    envConfigRaw,
+    deployConfigRaw: envConfigRaw.deploy,
+    buildConfigRaw: envConfigRaw.build,
+    environmentSlug,
+    gitlabEnvironmentName,
+    fullName,
+    envType,
+    commitInfo,
+    componentName,
+    env,
+    fullConfig: config,
+  };
+};

package/src/context/getEnvironmentVariables.ts

@@ -0,0 +1,152 @@
+import { isObject, merge } from "lodash";
+import { DEPLOY_TYPES } from "../deploy";
+import type { CommitInfo, Context } from "../types";
+import type { Config, DevLocalEnvConfig } from "../types/config";
+
+import { getEnvironmentContext } from "./getEnvironmentContext";
+import {
+  resolveReferences,
+  translateLegacyFromComponents,
+} from "./resolveReferences";
+
+export const getEnvironmentVariables = async (
+  config: Config,
+  componentName: string,
+  env: string,
+  commitInfo?: CommitInfo,
+  alreadyVisited: Record<string, Record<string, boolean>> = {} // to prevent endless loop
+): Promise<{
+  envVars: Record<string, string>;
+  secretEnvVarKeys: string[];
+  host: string;
+  url: string;
+}> => {
+  const environmentContext = getEnvironmentContext(
+    config,
+    env,
+    componentName,
+    commitInfo
+  );
+
+  const { envConfigRaw, deployConfigRaw, envType } = environmentContext;
+
+  const basePredefinedVariables = {
+    ENV_SHORT: env,
+    APP_DIR: envConfigRaw.dir,
+    ENV_TYPE: envType,
+  };
+
+  let predefinedVariables: Record<string, string>;
+  let host: string;
+  let url: string;
+
+  if (envType === "local") {
+    const devLocalConfig: DevLocalEnvConfig = envConfigRaw;
+    const port = devLocalConfig.port ?? 3000;
+    host = "localhost:" + port;
+    url = "http://" + host;
+    predefinedVariables = {
+      ENV_SHORT: "local",
+      ROOT_URL: url,
+      PORT: port.toString(),
+    };
+  } else {
+    const additionalEnvVars = deployConfigRaw
+      ? DEPLOY_TYPES[deployConfigRaw.type].getAdditionalEnvVars(
+          environmentContext
+        )
+      : {};
+
+    host =
+      envConfigRaw?.host ??
+      additionalEnvVars.HOST_CANONICAL ??
+      "unknown-host.example.com";
+    url = `https://${host}`;
+
+    predefinedVariables = {
+      ...basePredefinedVariables,
+      HOST: host,
+      ROOT_URL: url,
+      ...additionalEnvVars,
+    };
+  }
+  const publicEnvVarsRaw = envConfigRaw.vars?.public ?? {};
+
+  const additionalSecretKeys = deployConfigRaw
+    ? DEPLOY_TYPES[deployConfigRaw.type].additionalSecretKeys(
+        environmentContext
+      )
+    : [];
+
+  const secretEnvVarKeys = [
+    ...(envConfigRaw.vars?.secret ?? []),
+    ...additionalSecretKeys,
+  ];
+  const secretEnvVars = Object.fromEntries(
+    secretEnvVarKeys.map((key) => [
+      key,
+      `$${getSecretVarName(env, componentName, key)}`,
+    ])
+  );
+
+  // this is deprecated, we now support: $componentname:FOO
+  const legacyFromComponents = envConfigRaw.vars?.fromComponents ?? {};
+  const publicEnvVarsRawWithLegasyFromComponents = merge(
+    {},
+    translateLegacyFromComponents(legacyFromComponents),
+    publicEnvVarsRaw
+  );
+
+  const publicEnvVarsRawSanitized = Object.fromEntries(
+    Object.entries(publicEnvVarsRawWithLegasyFromComponents).map(
+      ([key, value]) => [
+        key,
+        isObject(value) ? JSON.stringify(value) : `${value}`,
+      ]
+    )
+  );
+
+  const envVarsRaw = addIndexVar({
+    ...predefinedVariables,
+    ...secretEnvVars,
+    ...publicEnvVarsRawSanitized,
+  });
+
+  const envVars = await resolveReferences(
+    envVarsRaw,
+    async (otherComponentName, variableName, alreadyVisited) => {
+      const { envVars: otherEnvVars } = await getEnvironmentVariables(
+        config,
+        otherComponentName,
+        env,
+        commitInfo,
+        alreadyVisited
+      );
+      return otherEnvVars[variableName];
+    },
+    alreadyVisited
+  );
+
+  return {
+    envVars,
+    secretEnvVarKeys,
+    host,
+    url,
+  };
+};
+
+const sanitizeForEnVar = (s: string) => s.replace(/-/g, "_");
+
+export const getSecretVarName = (
+  env: string,
+  componentName: string,
+  key: string
+) => `CL_${sanitizeForEnVar(env)}_${sanitizeForEnVar(componentName)}_${key}`; // remove dash from component name
+
+const addIndexVar = (vars: Record<string, unknown>) => ({
+  ...vars,
+  _ALL_ENV_VAR_KEYS: JSON.stringify(Object.keys(vars)),
+});
+
+export const getSecretVarNameForContext = (context: Context, key: string) =>
+  getSecretVarName(context.environment.shortName, context.componentName, key);

package/src/context/index.ts

@@ -1,13 +1,15 @@
 import { BUILD_TYPES } from "../build";
-import type { BuildConfig } from "../build/types";
+import type { BuildConfig, BuildConfigType } from "../build/types";
 import { DEPLOY_TYPES } from "../deploy";
-import type { DeployConfig } from "../deploy/types";
+import type { DeployConfig, DeployConfigType } from "../deploy/types";
 import type { Config } from "../types/config";
 import type { CommitInfo, Context, PackageManagerInfo } from "../types/context";
 import { mergeWithMergingArrays } from "../utils";
 import { getEnvironment } from "./getEnvironment";
+import { getEnvironmentContext } from "./getEnvironmentContext";
 
 export * from "./getEnvironment";
+export * from "./getEnvironmentVariables";
 
 export const createContext = async (
   config: Config,
@@ -21,27 +23,28 @@ export const createContext = async (
       "componentName may only contain lower case letters, numbers and -"
     );
   }
-  const rawConfig = config.components[componentName];
-  if (!rawConfig) {
-    throw new Error("unknown component " + componentName);
-  }
-  // envs can override the config
-  const envConfig = rawConfig.env?.[env] ?? {};
-  const componentConfigWithoutDefaults = mergeWithMergingArrays(
-    rawConfig,
-    envConfig
+
+  const envContext = getEnvironmentContext(
+    config,
+    env,
+    componentName,
+    commitInfo
   );
 
-  // fill in defaults of build and deploy
+  const componentConfigWithoutDefaults = envContext.envConfigRaw;
   const defaults: {
     build: Partial<BuildConfig>;
     deploy: Partial<DeployConfig>;
   } = componentConfigWithoutDefaults.deploy
     ? {
         build:
-          BUILD_TYPES[componentConfigWithoutDefaults.build.type].defaults(),
+          BUILD_TYPES[
+            componentConfigWithoutDefaults.build.type as BuildConfigType
+          ].defaults(envContext),
         deploy:
-          DEPLOY_TYPES[componentConfigWithoutDefaults.deploy.type].defaults(),
+          DEPLOY_TYPES[
+            componentConfigWithoutDefaults.deploy.type as DeployConfigType
+          ].defaults(envContext),
       }
     : {
         build: {},

package/src/deploy/cloudRun/index.ts

@@ -11,7 +11,7 @@ export const GCLOUD_RUN_DEPLOY_TYPE: DeployTypeDefinition<"google-cloudrun"> = {
   jobs: createGoogleCloudRunDeployJobs,
   defaults: () => ({}),
   additionalSecretKeys: () => [GCLOUD_DEPLOY_CREDENTIALS_KEY],
-  getAdditionalEnvVars: ({ fullName, env, componentName, deployConfig }) => {
+  getAdditionalEnvVars: ({ fullName, env, componentName, deployConfigRaw }) => {
     const HOST_CANONICAL =
       fullName.toLowerCase() +
       "-" +
@@ -20,11 +20,11 @@ export const GCLOUD_RUN_DEPLOY_TYPE: DeployTypeDefinition<"google-cloudrun"> = {
       ];
 
     const jobTriggers =
-      deployConfig && deployConfig.jobs
+      deployConfigRaw && deployConfigRaw.jobs
         ? Object.fromEntries(
-            Object.entries(deployConfig.jobs).map(([name, job]) => [
+            Object.entries(deployConfigRaw.jobs).map(([name, job]) => [
               "CLOUD_RUN_JOB_TRIGGER_URL_" + name,
-              `https://${deployConfig.region}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${deployConfig.projectId}/jobs/${name}:run`,
+              `https://${deployConfigRaw.region}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${deployConfigRaw.projectId}/jobs/${name}:run`,
             ])
           )
         : {};

package/src/deploy/index.ts

@@ -1,39 +1,27 @@
-import type { Config, EnvType } from "../types";
-import type { CommitInfo, Context } from "../types/context";
+import type { Context } from "../types/context";
+import type { EnvironmentContext } from "../types/environmentContext";
 import type { CatladderJob } from "../types/jobs";
 import { GCLOUD_RUN_DEPLOY_TYPE } from "./cloudRun";
 import { CUSTOM_DEPLOY_TYPE } from "./custom";
 import { DOCKER_TAG_DEPLOY_TYPE } from "./dockerTag";
 import { KUBERNETES_DEPLOY_TYPE } from "./kubernetes";
 import type { DeployConfigGeneric, DeployConfigType } from "./types";
+export * from "./cloudRun";
+export * from "./cloudSql";
 export * from "./kubernetes";
 export * from "./types";
 export * from "./utils";
 
-export type EnvVarContext<D extends DeployConfigType> = {
-  deployConfig: false | DeployConfigGeneric<D>;
-  commitInfo?: CommitInfo;
-  env: string;
-  envType: EnvType;
-  componentName: string;
-  fullName: string;
-
-  /**
-   * the full catladder config
-   */
-  fullConfig: Config;
-};
-
 export type DeployTypeDefinition<T extends DeployConfigType> = {
   jobs: (context: Context) => CatladderJob[];
-  defaults: () => Partial<DeployConfigGeneric<T>>;
-  additionalSecretKeys: (envVarContext: EnvVarContext<T>) => string[];
+  defaults: (
+    envContext: EnvironmentContext<any, T>
+  ) => Partial<DeployConfigGeneric<T>>;
+  additionalSecretKeys: (envContext: EnvironmentContext<any, T>) => string[];
   getAdditionalEnvVars: (
-    envVarContext: EnvVarContext<T>
+    envContext: EnvironmentContext<any, T>
   ) => Record<string, string | undefined | null>;
 };
-export * from "./cloudSql";
-export * from "./cloudRun";
 export type DeployTypes = {
   [T in DeployConfigType]: DeployTypeDefinition<T>;
 };

package/src/deploy/kubernetes/additionalSecretKeys.ts

@@ -1,20 +1,20 @@
-import type { EnvVarContext } from "..";
+import type { EnvironmentContext } from "../../types/environmentContext";
 
 export const additionalKubernetesSecretKeys = ({
-  deployConfig,
-}: EnvVarContext<"kubernetes">) => {
-  if (!deployConfig) {
+  deployConfigRaw,
+}: EnvironmentContext<any, "kubernetes">) => {
+  if (!deployConfigRaw) {
     return [];
   }
   const keys = [];
-  if (deployConfig.values?.mongodb?.enabled) {
+  if (deployConfigRaw.values?.mongodb?.enabled) {
     keys.push("MONGODB_ROOT_PASSWORD");
-    if (deployConfig.values.mongodb.architecture === "replicaset") {
+    if (deployConfigRaw.values.mongodb.architecture === "replicaset") {
       keys.push("MONGODB_REPLICASET_KEY");
     }
   }
 
-  if (deployConfig.values?.cloudsql?.enabled) {
+  if (deployConfigRaw.values?.cloudsql?.enabled) {
     keys.push("POSTGRESQL_PASSWORD");
     keys.push("cloudsqlProxyCredentials");
   }

package/src/deploy/kubernetes/index.ts

@@ -11,7 +11,7 @@ export const KUBERNETES_DEPLOY_TYPE: DeployTypeDefinition<"kubernetes"> = {
   getAdditionalEnvVars: ({
     componentName,
     fullConfig,
-    deployConfig,
+    deployConfigRaw,
     env,
     envType,
     commitInfo,
@@ -27,7 +27,7 @@ export const KUBERNETES_DEPLOY_TYPE: DeployTypeDefinition<"kubernetes"> = {
         : env;
 
     const domainCanonical =
-      (deployConfig && deployConfig.cluster?.domainCanonical) || // for convenience, we allow clusters to define a canonical domain, because a cluster has a fixed ip and you will usually have a domain pointing to that cluster
+      (deployConfigRaw && deployConfigRaw.cluster?.domainCanonical) || // for convenience, we allow clusters to define a canonical domain, because a cluster has a fixed ip and you will usually have a domain pointing to that cluster
       fullConfig.domainCanonical ||
       "panter.cloud";
     const HOST_CANONICAL = `${componentSlug}.${envInUrl}.${fullConfig.appName}.${fullConfig.customerName}.${domainCanonical}`; // default for kubernetes and rest

package/src/types/config.ts

@@ -104,6 +104,8 @@ export type EnvConfig<E extends EnvType = EnvType> = {
   host?: string;
 } & PartialDeep<DefaultEnvConfig>;
 
+export type EnvConfigWithComponent = EnvConfig<EnvType> & ComponentConfig;
+
 export type Env = {
   /**
    * local is a special env that is only used in local development

package/src/types/environmentContext.ts

@@ -0,0 +1,26 @@
+import type { BuildConfigGeneric, BuildConfigType } from "../build";
+import type { DeployConfigGeneric } from "../deploy";
+import type { DeployConfigType } from "../deploy";
+import type { Config, EnvConfigWithComponent, EnvType } from "./config";
+import type { CommitInfo } from "./context";
+
+export type EnvironmentContext<
+  B extends BuildConfigType,
+  D extends DeployConfigType
+> = {
+  envConfigRaw: EnvConfigWithComponent;
+  buildConfigRaw: false | BuildConfigGeneric<B>;
+  deployConfigRaw: false | DeployConfigGeneric<D>;
+  commitInfo?: CommitInfo;
+  env: string;
+  envType: EnvType;
+  componentName: string;
+  fullName: string;
+  environmentSlug: string;
+  gitlabEnvironmentName: string;
+
+  /**
+   * the full catladder config
+   */
+  fullConfig: Config;
+};