@catladder/pipeline 0.0.0-node16-ddfae617 → 0.0.0-perf-cli-reduce-bundle-size-with-tsconfig-optimization-3556aaa4

package/src/context/getEnvironmentVariables.ts

@@ -0,0 +1,210 @@
+import { merge } from "lodash";
+import { DEPLOY_TYPES } from "../deploy";
+import type {
+  ComponentContext,
+  EnvironmentEnvVarPart as EnvironmentVariables,
+} from "../types";
+import type { DevLocalEnvConfig } from "../types/config";
+
+import type { CreateComponentContextContext, UnspecifiedEnvVars } from "..";
+import type { StringOrBashExpression } from "../bash/BashExpression";
+import { joinBashExpressions } from "../bash/BashExpression";
+import { isStandaloneBuildConfig } from "../build/types";
+import type { EnvironmentContext } from "../types/environmentContext";
+import { getBuildInfoVariables } from "./getBuildInfoVariables";
+import { getEnvironmentContext } from "./getEnvironmentContext";
+import {
+  resolveReferences,
+  translateLegacyFromComponents,
+} from "./resolveReferences";
+import { transformJobOnlyVars } from "./transformJobOnlyVars";
+import {
+  makeSecretEnvVarMapping,
+  stringListToSecreteEnvVarList,
+  stringifyValues,
+} from "./utils/envVars";
+
+export type SecretEnvVar = {
+  key: string;
+  // hidden env vars are not shown in config-secrets
+  hidden?: boolean;
+};
+
+const getBasePredefinedVariables = (ctx: EnvironmentContext) => {
+  return {
+    ENV_SHORT: ctx.env,
+    APP_DIR: ctx.envConfigRaw.dir,
+    ENV_TYPE: ctx.envType,
+    ...(ctx.envType !== "local" ? getBuildInfoVariables(ctx) : {}),
+  };
+};
+
+type BasePredefinedVariables = ReturnType<typeof getBasePredefinedVariables>;
+
+// we export so that we have later nice autocomplete
+export type PredefinedVariables = BasePredefinedVariables & {
+  /**
+   * undefined in rails, Rails before 6.1 (mis)uses the `HOST` environment variable to specify the IP to bind to
+   */
+  HOST?: StringOrBashExpression;
+  ROOT_URL: StringOrBashExpression;
+  HOST_INTERNAL: StringOrBashExpression;
+  ROOT_URL_INTERNAL: StringOrBashExpression;
+};
+
+export const getEnvironmentVariables = async (
+  ctx: CreateComponentContextContext,
+  alreadyVisited: Record<string, Record<string, boolean>> = {}, // to prevent endless loop
+): Promise<EnvironmentVariables> => {
+  const environmentContext = getEnvironmentContext(ctx);
+
+  const { config, env, componentName } = ctx;
+  const { envConfigRaw, deployConfigRaw, buildConfigRaw, envType } =
+    environmentContext;
+
+  const basePredefinedVariables =
+    getBasePredefinedVariables(environmentContext);
+
+  let predefinedVariables: PredefinedVariables & UnspecifiedEnvVars;
+  let host: StringOrBashExpression;
+  let url: StringOrBashExpression;
+
+  if (envType === "local") {
+    const devLocalConfig: DevLocalEnvConfig = envConfigRaw;
+    const port = devLocalConfig.port ?? 3000;
+    host = "localhost:" + port.toString();
+    url = "http://" + host;
+    predefinedVariables = {
+      ...basePredefinedVariables,
+      ENV_SHORT: "local",
+      ROOT_URL: url,
+      // Rails before 6.1 (mis)uses the `HOST` environment variable to specify the IP to bind to
+      ...(isStandaloneBuildConfig(buildConfigRaw) &&
+      buildConfigRaw.type === "rails"
+        ? {}
+        : { HOST: host }),
+      HOST_INTERNAL: host,
+      ROOT_URL_INTERNAL: "http://" + host,
+      PORT: port.toString(),
+    };
+  } else {
+    const additionalEnvVars = deployConfigRaw
+      ? DEPLOY_TYPES[deployConfigRaw.type].getAdditionalEnvVars(
+          environmentContext as never,
+        )
+      : {};
+
+    const HOST_INTERNAL =
+      additionalEnvVars.HOST_INTERNAL ?? "unknown-host.example.com";
+
+    host = envConfigRaw?.host ?? HOST_INTERNAL;
+    url = joinBashExpressions(["https://", host]);
+
+    predefinedVariables = {
+      ...basePredefinedVariables,
+      // Rails before 6.1 (mis)uses the `HOST` environment variable to specify the IP to bind to
+      ...(isStandaloneBuildConfig(buildConfigRaw) &&
+      buildConfigRaw.type === "rails"
+        ? {}
+        : { HOST: host }),
+      ROOT_URL: url,
+      HOST_INTERNAL,
+      /**@deprecated */
+      HOST_CANONICAL: HOST_INTERNAL, // legacy alias for HOST_INTERNAL
+      ROOT_URL_INTERNAL: joinBashExpressions(["https://", HOST_INTERNAL]),
+      ...additionalEnvVars,
+    };
+  }
+  const publicEnvVarsRaw = envConfigRaw.vars?.public ?? {};
+
+  const additionalSecretKeys = deployConfigRaw
+    ? DEPLOY_TYPES[deployConfigRaw.type].additionalSecretKeys(
+        environmentContext as never,
+      )
+    : [];
+
+  const secretEnvVarKeys: SecretEnvVar[] = [
+    ...stringListToSecreteEnvVarList(envConfigRaw.vars?.secret ?? []),
+    ...additionalSecretKeys,
+  ];
+  const secretEnvVars = makeSecretEnvVarMapping(
+    env,
+    componentName,
+    secretEnvVarKeys,
+  );
+  // this is deprecated, we now support: $componentname:FOO
+  const legacyFromComponents = envConfigRaw.vars?.fromComponents ?? {};
+  const publicEnvVarsRawWithLegacyFromComponents = merge(
+    {},
+    translateLegacyFromComponents(legacyFromComponents),
+    publicEnvVarsRaw,
+  );
+
+  const publicEnvVarsRawSanitized = stringifyValues(
+    publicEnvVarsRawWithLegacyFromComponents,
+  );
+
+  const envVarsRaw = addIndexVar({
+    ...predefinedVariables,
+    ...secretEnvVars,
+    ...publicEnvVarsRawSanitized,
+  });
+
+  const envVars = (await resolveReferences(
+    envVarsRaw,
+    async (otherComponentName, alreadyVisited) => {
+      const { envVars: otherEnvVars } = await getEnvironmentVariables(
+        {
+          ...ctx,
+          componentName: otherComponentName,
+        },
+        alreadyVisited,
+      );
+      return otherEnvVars;
+    },
+    alreadyVisited,
+  )) as typeof envVarsRaw;
+
+  return {
+    envVars,
+    secretEnvVarKeys,
+    jobOnlyVars: {
+      build: await transformJobOnlyVars(
+        env,
+        componentName,
+        (buildConfigRaw &&
+          isStandaloneBuildConfig(buildConfigRaw) &&
+          buildConfigRaw.jobVars) ||
+          null,
+      ),
+      deploy: await transformJobOnlyVars(
+        env,
+        componentName,
+        (deployConfigRaw && deployConfigRaw.jobVars) || null,
+      ),
+    },
+
+    host,
+    url,
+  };
+};
+
+const sanitizeForEnVar = (s: string) => s.replace(/-/g, "_");
+
+export const getSecretVarName = (
+  env: string,
+  componentName: string,
+  key: string,
+) => `CL_${sanitizeForEnVar(env)}_${sanitizeForEnVar(componentName)}_${key}`; // remove dash from component name
+
+const addIndexVar = <V extends Record<string, unknown>>(
+  vars: V,
+): V & { _ALL_ENV_VAR_KEYS: string } => ({
+  ...vars,
+  _ALL_ENV_VAR_KEYS: JSON.stringify(Object.keys(vars)),
+});
+
+export const getSecretVarNameForContext = (
+  context: ComponentContext,
+  key: string,
+) => getSecretVarName(context.env, context.name, key);

package/src/context/getLabels.ts

@@ -0,0 +1,21 @@
+import slugify from "slugify";
+import type { ComponentContext } from "../types";
+
+const sanitize = (value?: string) => {
+  if (!value) return value;
+  //"The value can only contain lowercase letters, numeric characters, underscores and dashes. The value can be at most 63 characters long. International characters are allowed."
+  // slugify should do the job
+  return slugify(value).toLowerCase();
+};
+export const getLabels = (context: ComponentContext) => {
+  const labels = {
+    "customer-name": sanitize(context.fullConfig.customerName),
+    "component-name": sanitize(context.name),
+    "app-name": sanitize(context.fullConfig.appName),
+    "env-type": sanitize(context.environment.envType),
+    "env-name": sanitize(context.env),
+    "build-type": sanitize(context.build.buildType),
+    ...(context.fullConfig.meta?.labels ?? {}),
+  };
+  return labels;
+};

package/src/context/getReviewSlug.ts

@@ -0,0 +1,27 @@
+import {
+  BashExpression,
+  type StringOrBashExpression,
+} from "../bash/BashExpression";
+import type { BashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
+import { getBashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
+import type { PipelineType } from "../types";
+import type { EnvConfigWithComponent } from "../types/config";
+import { getEnvType } from "./getEnvType";
+const REVIEW_SLUG: BashExpressionPerPipelineType = {
+  default: "unknown-review-slug",
+  gitlab: new BashExpression(
+    `$([ -n "$CI_MERGE_REQUEST_IID" ] && echo "mr$CI_MERGE_REQUEST_IID" || { [ -n "$CI_COMMIT_REF_SLUG" ] && echo "$CI_COMMIT_REF_SLUG" || echo "unknown"; })`,
+  ),
+};
+
+export const getReviewSlug = (
+  envConfig: EnvConfigWithComponent,
+  env: string,
+  pipelineType?: PipelineType,
+): StringOrBashExpression | null => {
+  const envType = getEnvType(env, envConfig);
+  if (envType === "review") {
+    return getBashExpressionPerPipelineType(REVIEW_SLUG, pipelineType);
+  }
+  return null; // not a review app;
+};

package/src/context/index.ts

@@ -1,232 +1,3 @@
-import { isObject, merge } from "lodash";
-import slugify from "slugify";
-import { BUILD_TYPES } from "../build";
-import { BuildConfig } from "../build/types";
-import { DEPLOY_TYPES, getKubernetesNamespace } from "../deploy";
-import { DeployConfig } from "../deploy/types";
-import { Config, isKnowEnvType, DevLocalEnvConfig } from "../types/config";
-import {
-  CommitInfo,
-  Context,
-  Environment,
-  PackageManagerInfo,
-} from "../types/context";
-import { mergeWithMergingArrays } from "../utils";
-import {
-  resolveReferences,
-  translateLegacyFromComponents,
-} from "./resolveReferences";
-
-const sanitizeForEnVar = (s: string) => s.replace(/-/g, "_");
-export const getSecretVarName = (
-  env: string,
-  componentName: string,
-  key: string
-) => `CL_${sanitizeForEnVar(env)}_${sanitizeForEnVar(componentName)}_${key}`; // remove dash from component name
-
-export const getSecretVarNameForContext = (context: Context, key: string) =>
-  getSecretVarName(context.environment.shortName, context.componentName, key);
-export const getEnvironment = (
-  config: Config,
-  componentName: string,
-  env: string,
-  commitInfo?: CommitInfo,
-  alreadyVisited: Record<string, Record<string, boolean>> = {} // to prevent endless loop
-): Environment => {
-  const componentConfig = config.components[componentName];
-  if (!componentConfig) {
-    throw new Error("unknown component " + componentName);
-  }
-
-  const defaultConfig = componentConfig;
-
-  const envConfig = componentConfig.env?.[env] ?? {};
-  if (envConfig === false) {
-    throw new Error("env is disabled: " + env);
-  }
-
-  const mergedConfig = mergeWithMergingArrays(defaultConfig, envConfig);
-
-  // env type: if its set manually, use that, otherwise use the known env types
-
-  const envType = envConfig?.type ?? (isKnowEnvType(env) ? env : null);
-
-  if (!envType) {
-    throw new Error(
-      "Missing type in environment " + env + " in component " + componentName
-    );
-  }
-
-  const basePredefinedVariables = {
-    ENV_SHORT: env,
-    APP_DIR: componentConfig.dir,
-    ENV_TYPE: envType,
-  };
-
-  const environmentName =
-    envType === "review" && commitInfo
-      ? `${env}/${commitInfo.refName}/${componentName}`
-      : `${env}/${componentName}`;
-
-  const environmentSlug =
-    envType === "review" && commitInfo
-      ? `${env}-${commitInfo.reviewSlug}-${componentName}`
-      : `${env}-${componentName}`;
-
-  let predefinedVariables: Record<string, string>;
-  let host: string;
-  let url: string;
-  if (envType === "local") {
-    const devLocalConfig: DevLocalEnvConfig = mergedConfig;
-    const port = devLocalConfig.port ?? 3000;
-    host = "localhost:" + port;
-    url = "http://" + host;
-    predefinedVariables = {
-      ENV_SHORT: "local",
-      ROOT_URL: url,
-      PORT: port.toString(),
-    };
-  } else {
-    const RELEASE_NAME = `${config.customerName}-${config.appName}-${environmentSlug}`;
-
-    const componentSlug = slugify(componentName);
-    const envInUrl =
-      envType === "review" && commitInfo ? `${commitInfo.reviewSlug}.${env}` : env;
-
-    const domainCanonical =
-      (mergedConfig?.deploy &&
-        mergedConfig?.deploy.type === "kubernetes" &&
-        mergedConfig.deploy.cluster?.domainCanonical) || // for convenience, we allow clusters to define a canonical domain, because a cluster has a fixed ip and you will usually have a domain pointing to that cluster
-      config.domainCanonical ||
-      "panter.cloud";
-
-    const HOST_CANONICAL = `${componentSlug}.${envInUrl}.${config.appName}.${config.customerName}.${domainCanonical}`;
-    host = mergedConfig?.host ?? HOST_CANONICAL;
-    const url = `https://${host}`;
-
-    // FIXME: move to kube specific jobs
-    const KUBE_APP_NAME =
-      envType === "review" && commitInfo
-        ? `${commitInfo.reviewSlug}-${componentName}`
-        : componentName;
-    const KUBE_NAMESPACE = getKubernetesNamespace(config, env);
-
-    predefinedVariables = {
-      ...basePredefinedVariables,
-      HOST_CANONICAL,
-      ROOT_URL: url,
-      KUBE_NAMESPACE,
-      KUBE_APP_NAME,
-      RELEASE_NAME,
-    };
-  }
-  const publicEnvVarsRaw = mergedConfig.vars?.public ?? {};
-  const secretEnvVarKeys = mergedConfig.vars?.secret ?? [];
-  const secretEnvVars = Object.fromEntries(
-    secretEnvVarKeys.map((key) => [
-      key,
-      `$${getSecretVarName(env, componentName, key)}`,
-    ])
-  );
-
-  // this is deprecated, we now support: $componentname:FOO
-  const legacyFromComponents = mergedConfig.vars?.fromComponents ?? {};
-  const publicEnvVarsRawWithLegasyFromComponents = merge(
-    {},
-    translateLegacyFromComponents(legacyFromComponents),
-    publicEnvVarsRaw
-  );
-
-  const publicEnvVarsRawSanitized = Object.fromEntries(
-    Object.entries(publicEnvVarsRawWithLegasyFromComponents).map(
-      ([key, value]) => [
-        key,
-        isObject(value) ? JSON.stringify(value) : `${value}`,
-      ]
-    )
-  );
-
-  const envVarsRaw = {
-    ...predefinedVariables,
-    ...secretEnvVars,
-    ...publicEnvVarsRawSanitized,
-  };
-
-  const envVars = resolveReferences(
-    envVarsRaw,
-    (componentName, variableName, alreadyVisited) => {
-      const { envVars } = getEnvironment(
-        config,
-        componentName,
-        env,
-        commitInfo,
-        alreadyVisited
-      );
-      return envVars[variableName];
-    },
-    alreadyVisited
-  );
-  return {
-    envType,
-    host,
-    fullName: environmentName,
-    slug: environmentSlug,
-    shortName: env,
-    url: predefinedVariables.ROOT_URL,
-    envVars,
-    secretEnvVarKeys,
-  };
-};
-
-export const createContext = (
-  config: Config,
-  componentName: string,
-  env: string,
-  commitInfo?: CommitInfo,
-  packageManagerInfo?: PackageManagerInfo
-): Context => {
-  if (!/^[a-z0-9-]+$/.test(componentName)) {
-    throw new Error(
-      "componentName may only contain lower case letters, numbers and -"
-    );
-  }
-  const rawConfig = config.components[componentName];
-  if (!rawConfig) {
-    throw new Error("unknown component " + componentName);
-  }
-  // envs can override the config
-  const envConfig = rawConfig.env?.[env] ?? {};
-  const componentConfigWithoutDefaults = mergeWithMergingArrays(
-    rawConfig,
-    envConfig
-  );
-
-  // fill in defaults of build and deploy
-  const defaults: {
-    build: Partial<BuildConfig>;
-    deploy: Partial<DeployConfig>;
-  } = componentConfigWithoutDefaults.deploy
-    ? {
-        build:
-          BUILD_TYPES[componentConfigWithoutDefaults.build.type].defaults(),
-        deploy:
-          DEPLOY_TYPES[componentConfigWithoutDefaults.deploy.type].defaults(),
-      }
-    : {
-        build: {},
-        deploy: {},
-      };
-  const componentConfig = mergeWithMergingArrays(
-    defaults,
-    componentConfigWithoutDefaults
-  );
-
-  return {
-    fullConfig: config,
-    componentConfig,
-    componentName,
-    environment: getEnvironment(config, componentName, env, commitInfo),
-    commitInfo,
-    packageManagerInfo,
-  };
-};
+export * from "./createComponentContext";
+export * from "./getEnvironment";
+export * from "./getEnvironmentVariables";

package/src/context/resolveReferences.ts

@@ -1,59 +1,86 @@
 import { merge } from "lodash";
 
-const REGEX = /\$\{(([^:]+):)?([^}]+)}/gm;
+import type { BashExpression } from "../bash/BashExpression";
+import replaceAsync from "../bash/replaceAsync";
+import type { UnspecifiedEnvVars } from "..";
 
-export const resolveReferences = (
-  vars: Record<string, string>,
+// regex to resolve references in catladder variables
+// those expressions have the pattern ${componentName:variableName}
+const REGEX = /\$\{(([^:}]+):)?([^}]+)}/gm;
+
+export const resolveReferences = async (
+  vars: Record<string, string | BashExpression | undefined | null>,
   getOtherVariables?: (
     componentName: string,
-    variableName: string,
-    alreadyVisited: Record<string, Record<string, boolean>>
-  ) => string | null,
-  alreadyVisitedBase: Record<string, Record<string, boolean>> = {}
+    alreadyVisited: Record<string, Record<string, boolean>>,
+  ) => Promise<UnspecifiedEnvVars>,
+  alreadyVisitedBase: Record<string, Record<string, boolean>> = {},
 ) => {
-  const replaceSingleValue = (
-    value: string,
-    alreadyVisited: Record<string, Record<string, boolean>> = alreadyVisitedBase
-  ): string => {
-    if (REGEX.test(value)) {
-      return value.replace(REGEX, (match, _, componentName, variableName) => {
-        if (alreadyVisited[componentName]?.[variableName]) {
-          return match; // prevent endless loop
-        }
-        const newAlreadyVisited = merge({}, alreadyVisited, {
-          [componentName]: {
-            [variableName]: true,
-          },
-        });
-        const result = componentName
-          ? getOtherVariables?.(
-              componentName,
-              variableName,
-              newAlreadyVisited
-            ) ?? null
-          : vars[variableName]; // is self reference
+  /**
+   *
+   * replace referenced variables with their values in a value string
+   */
+  const replaceSingleValue = async (
+    value: string | BashExpression,
+    alreadyVisited: Record<
+      string,
+      Record<string, boolean>
+    > = alreadyVisitedBase,
+  ): Promise<string | BashExpression> => {
+    if (REGEX.test(value.toString())) {
+      // we consider variables that got references in it  BashExpressions, because the replacement may be one
+
+      return await replaceAsync(
+        value,
+        REGEX,
+        async (match, _, componentName, variableName) => {
+          if (alreadyVisited[componentName]?.[variableName]) {
+            return match; // prevent endless loop
+          }
+          const newAlreadyVisited = merge({}, alreadyVisited, {
+            [componentName]: {
+              [variableName]: true,
+            },
+          });
+          const result = componentName
+            ? (await getOtherVariables?.(componentName, newAlreadyVisited).then(
+                (r) => r?.[variableName],
+              )) ?? null
+            : vars[variableName]; // is self reference
 
-        const replaced =
-          result !== null && result !== undefined
-            ? replaceSingleValue(result, newAlreadyVisited)
-            : match;
+          const replaced =
+            result !== null && result !== undefined
+              ? await replaceSingleValue(result, newAlreadyVisited)
+              : match;
 
-        return replaced;
-      });
+          return replaced;
+        },
+      );
     } else {
       return value;
     }
   };
 
   return Object.fromEntries(
-    Object.entries(vars).map(([key, value]) => {
-      return [key, replaceSingleValue(value)];
-    })
-  );
+    await Promise.all(
+      Object.entries(vars).map(async ([key, value]) => {
+        if (value === null || value === undefined) {
+          return [key, null];
+        }
+
+        return [
+          key,
+          value !== null && value !== undefined
+            ? await replaceSingleValue(value)
+            : null,
+        ];
+      }),
+    ),
+  ) as Record<string, BashExpression>;
 };
 
 export const translateLegacyFromComponents = (
-  fromComponents: Record<string, Record<string, string>>
+  fromComponents: Record<string, Record<string, string>>,
 ) => {
   return Object.fromEntries(
     Object.entries(fromComponents).flatMap(([componentName, variables]) => {
@@ -61,6 +88,6 @@ export const translateLegacyFromComponents = (
         ourName,
         "${" + componentName + ":" + otherName + "}",
       ]);
-    })
+    }),
   );
 };

package/src/context/transformJobOnlyVars.ts

@@ -0,0 +1,45 @@
+import type { SecretEnvVar, UnspecifiedEnvVars } from "..";
+import type { EnvVars } from "../types/config";
+import {
+  makeSecretEnvVarMapping,
+  stringListToSecreteEnvVarList,
+  stringifyValues,
+} from "./utils/envVars";
+
+/**
+ * transform EnvVars for environment.
+ *
+ * currently does not resolve references and is only used for additional job-only-vars
+ * @param vars
+ * @returns
+ */
+export const transformJobOnlyVars = async (
+  env: string,
+  componentName: string,
+  vars: EnvVars | null,
+): Promise<{
+  envVars: UnspecifiedEnvVars;
+  secretEnvVarKeys: SecretEnvVar[];
+}> => {
+  if (!vars) {
+    return {
+      envVars: {},
+      secretEnvVarKeys: [],
+    };
+  }
+
+  const publicVars = stringifyValues(vars?.public ?? {});
+  const secretEnvVarKeys = stringListToSecreteEnvVarList(vars?.secret ?? []);
+  const deployJobOnlySecretEnvVars = vars?.secret
+    ? makeSecretEnvVarMapping(env, componentName, secretEnvVarKeys)
+    : {};
+
+  const envVars = {
+    ...deployJobOnlySecretEnvVars,
+    ...publicVars,
+  };
+  return {
+    envVars,
+    secretEnvVarKeys,
+  };
+};