@catladder/pipeline 0.0.0-more-collapse-deeb54d7 → 0.0.0-multiple-gitlab-includes-1a40efc0

package/src/context/createAllComponentsContext.ts

@@ -0,0 +1,31 @@
+import { getAllEnvsByTrigger } from "../config/configruedEnvs";
+import type { ComponentContext, PipelineType } from "../types";
+import type { Config, PipelineTrigger } from "../types/config";
+import { createComponentContext } from "./createComponentContext";
+
+export type CreateAllComponentsContextProps = {
+  config: Config;
+  trigger: PipelineTrigger;
+  pipelineType: PipelineType;
+};
+
+export const createAllComponentsContext = async ({
+  config,
+  trigger,
+  pipelineType,
+}: CreateAllComponentsContextProps): Promise<Array<ComponentContext>> => {
+  return await Promise.all(
+    Object.keys(config.components).flatMap((componentName) => {
+      const envs = getAllEnvsByTrigger(config, componentName, trigger);
+      return envs.map(async (env) => {
+        return await createComponentContext({
+          config,
+          componentName,
+          env,
+          trigger,
+          pipelineType,
+        });
+      });
+    }),
+  );
+};

package/src/context/createComponentContext.ts

@@ -0,0 +1,134 @@
+import { isFunction } from "lodash";
+import { BUILD_TYPES, isStandaloneBuildConfig } from "../build";
+import type { BuildConfig } from "../build/types";
+import { DEPLOY_TYPES } from "../deploy";
+import type { DeployConfig, DeployConfigType } from "../deploy/types";
+import { getPackageManagerInfoForComponent } from "../pipeline/packageManager";
+import type { PipelineType } from "../types";
+import type { Config, PipelineTrigger } from "../types/config";
+import type {
+  BuildContext,
+  BuildContextComponent,
+  ComponentContext,
+} from "../types/context";
+import type { PartialDeep } from "../types/utils";
+import { mergeWithMergingArrays } from "../utils";
+import { getEnvironment } from "./getEnvironment";
+import { getEnvironmentContext } from "./getEnvironmentContext";
+
+export type CreateComponentContextContext = {
+  config: Config;
+  componentName: string;
+  env: string;
+  pipelineType?: PipelineType;
+  trigger?: PipelineTrigger;
+};
+
+export const createComponentContext = async (
+  ctx: CreateComponentContextContext,
+): Promise<ComponentContext> => {
+  if (!/^[a-z0-9-]+$/.test(ctx.componentName)) {
+    throw new Error(
+      "componentName may only contain lower case letters, numbers and -",
+    );
+  }
+
+  const packageManagerInfo = await getPackageManagerInfoForComponent(
+    ctx.config,
+    ctx.componentName,
+  );
+
+  const envContext = getEnvironmentContext(ctx);
+
+  const componentConfigWithoutDefaults = envContext.envConfigRaw;
+
+  const resolvedBuildType = isStandaloneBuildConfig(
+    componentConfigWithoutDefaults.build,
+  )
+    ? componentConfigWithoutDefaults.build.type
+    : ctx.config.builds?.[componentConfigWithoutDefaults.build.from].type;
+  if (!resolvedBuildType) {
+    throw new Error("build type not found, is the build config correct?");
+  }
+  const defaults: {
+    build: PartialDeep<BuildConfig>;
+    deploy: PartialDeep<DeployConfig>;
+  } = componentConfigWithoutDefaults.deploy
+    ? {
+        build: BUILD_TYPES[resolvedBuildType].defaults(envContext),
+        deploy: DEPLOY_TYPES[
+          componentConfigWithoutDefaults.deploy.type as DeployConfigType
+        ].defaults(envContext as any),
+      }
+    : {
+        build: BUILD_TYPES[resolvedBuildType].defaults(envContext),
+        deploy: {},
+      };
+
+  const componentConfig = mergeWithMergingArrays(
+    defaults,
+    componentConfigWithoutDefaults,
+  );
+
+  const environment = await getEnvironment(ctx);
+  const { deploy, build, customJobs, dir } = componentConfig;
+  const getComponentDirs: BuildContext["getComponentDirs"] = (mode) => [
+    dir,
+    // also copy workspace dependencies in monorepo if packages are shared and they create build artifacts
+    ...(mode === "all"
+      ? (packageManagerInfo.currentWorkspaceDependencies ?? [])
+      : []),
+  ];
+  const _getBuildContext = (): BuildContextComponent => {
+    if (isStandaloneBuildConfig(build)) {
+      return {
+        dir: dir,
+        getComponentDirs,
+        config: build,
+        buildType: build.type,
+        type: "standalone",
+      };
+    }
+    // must be shared build
+    const referencedBuild = ctx.config.builds?.[build.from];
+    if (!referencedBuild) {
+      throw new Error("build.from not found in config");
+    }
+    return {
+      dir: dir,
+      getComponentDirs,
+      config: build,
+      workspaceBuildConfig: referencedBuild,
+      workspaceName: build.from,
+      buildType: referencedBuild.type,
+      type: "fromWorkspace",
+    };
+  };
+  const buildContext: BuildContextComponent = _getBuildContext();
+  const context: Omit<ComponentContext, "customJobs"> = {
+    type: "component",
+    name: ctx.componentName,
+    env: ctx.env,
+    fullConfig: ctx.config,
+    componentConfig,
+
+    build: buildContext,
+    deploy: deploy
+      ? {
+          config: deploy,
+        }
+      : null,
+
+    environment,
+    packageManagerInfo: packageManagerInfo,
+    pipelineType: ctx.pipelineType,
+    trigger: ctx.trigger,
+  };
+  const resolvedCustomJobs = isFunction(customJobs)
+    ? customJobs(context)
+    : customJobs;
+  return {
+    ...context,
+    customJobs: resolvedCustomJobs,
+  };
+};

package/src/context/createWorkspaceContext.ts

@@ -0,0 +1,56 @@
+import {
+  WORKSPACE_BUILD_TYPES,
+  type Config,
+  type PipelineTrigger,
+  type PipelineType,
+  type WorkspaceContext,
+} from "..";
+import { getPackageManagerInfoBase } from "../pipeline/packageManager";
+import { mergeWithMergingArrays } from "../utils";
+import { uniq } from "lodash";
+
+export async function createWorkspaceContext({
+  env,
+  components,
+  workspaceName,
+  config,
+  pipelineType,
+  trigger,
+}: {
+  env: string;
+  components: WorkspaceContext["components"];
+  workspaceName: string;
+  config: Config;
+  pipelineType: PipelineType;
+  trigger: PipelineTrigger;
+}): Promise<WorkspaceContext> {
+  const workspaceConfigRaw = config.builds?.[workspaceName];
+  if (!workspaceConfigRaw) {
+    throw new Error(`Workspace ${workspaceName} not found in config`);
+  }
+
+  const defaults = WORKSPACE_BUILD_TYPES[workspaceConfigRaw.type].defaults();
+
+  const workspaceConfig = mergeWithMergingArrays(defaults, workspaceConfigRaw);
+  return {
+    name: workspaceName,
+    pipelineType,
+    trigger,
+    type: "workspace",
+
+    workspaceConfig,
+
+    env,
+    components,
+    fullConfig: config,
+    packageManagerInfo: await getPackageManagerInfoBase(),
+    build: {
+      type: "workspace",
+      dir: workspaceConfig.dir ?? ".",
+      getComponentDirs: (mode) =>
+        uniq(components.flatMap((c) => c.build.getComponentDirs(mode))),
+      buildType: workspaceConfig.type,
+      config: workspaceConfig,
+    },
+  };
+}

package/src/context/getBuildInfoVariables.ts

@@ -0,0 +1,36 @@
+import { BashExpression } from "../bash/BashExpression";
+import type { BashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
+import { getBashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
+import type { EnvironmentContext } from "../types/environmentContext";
+
+const BUILD_VARIABLES = {
+  BUILD_INFO_BUILD_ID: {
+    default: new BashExpression(
+      `$(git describe --tags 2>/dev/null || git rev-parse HEAD)`,
+    ),
+  },
+  BUILD_INFO_BUILD_TIME: {
+    default: "unknown-build-time",
+    gitlab: new BashExpression(`$CI_JOB_STARTED_AT`),
+  },
+  BUILD_INFO_CURRENT_VERSION: {
+    default: new BashExpression(
+      // because we do shallow fetch, we need to ask the origin
+      `$(tag=$(git ls-remote origin "refs/tags/v*[0-9]" 2>/dev/null | cut -f 2- | sort -V | tail -1 | sed 's/refs\\/tags\\/v//'); [ -z "$tag" ] && echo "0.0.0" || echo "$tag")`,
+    ),
+  },
+} satisfies {
+  [key: string]: BashExpressionPerPipelineType;
+};
+
+export const ALL_BUILD_VARIABLES = Object.keys(BUILD_VARIABLES);
+export const getBuildInfoVariables = (ctx: EnvironmentContext) => {
+  const { pipelineType } = ctx;
+
+  return Object.fromEntries(
+    Object.entries(BUILD_VARIABLES).map(([key, value]) => [
+      key,
+      getBashExpressionPerPipelineType(value, pipelineType),
+    ]),
+  );
+};

package/src/context/getEnvConfig.ts

@@ -5,7 +5,7 @@ import { mergeWithMergingArrays } from "../utils";
 export const getEnvConfig = (
   config: Config,
   componentName: string,
-  env: string
+  env: string,
 ): EnvConfigWithComponent => {
   const defaultConfig = config.components[componentName];
   if (!defaultConfig) {
@@ -14,8 +14,8 @@ export const getEnvConfig = (
 
   const envCustomizations = defaultConfig.env?.[env] ?? {};
   if (envCustomizations === false) {
-    throw new Error("env is disabled: " + env + ":" + componentName);
+    // env is disabled, still return the default config
+    return defaultConfig;
   }
-
   return mergeWithMergingArrays(defaultConfig, envCustomizations);
 };

package/src/context/getEnvType.ts

@@ -5,7 +5,7 @@ export const getEnvType = (
   env: string,
   envConfig: {
     type?: EnvType;
-  }
+  },
 ): EnvType => {
   if (envConfig.type) return envConfig.type;
 

package/src/context/getEnvironment.ts

@@ -1,42 +1,26 @@
-import type { Config } from "../types/config";
+import type { CreateComponentContextContext } from "..";
 
-import type { CommitInfo, Environment } from "../types/context";
+import type { Environment } from "../types/context";
 import { getEnvironmentContext } from "./getEnvironmentContext";
 import { getEnvironmentVariables } from "./getEnvironmentVariables";
 
 export const getEnvironment = async (
-  config: Config,
-  componentName: string,
-  env: string,
-  commitInfo?: CommitInfo
+  ctx: CreateComponentContextContext,
 ): Promise<Environment> => {
-  const variables = await getEnvironmentVariables(
-    config,
-    componentName,
-    env,
-    commitInfo
-  );
+  const { env } = ctx;
+  const variables = await getEnvironmentVariables(ctx);
 
-  const envContext = getEnvironmentContext(
-    config,
-    env,
-    componentName,
-    commitInfo
-  );
+  const envContext = getEnvironmentContext(ctx);
 
   const envType = envContext.envType;
 
   return {
     envType,
-
-    gitlabEnvironment: {
-      name: envContext.gitlabEnvironmentName,
-      url: variables.url,
-    },
     fullName: envContext.fullName,
     slugPrefix: envContext.environmentSlugPrefix,
+    reviewSlug: envContext.reviewSlug,
     slug: envContext.environmentSlug,
-    shortName: env,
+
     ...variables,
   };
 };

package/src/context/getEnvironmentContext.ts

@@ -1,44 +1,40 @@
-import type { Config, EnvConfigWithComponent } from "../types/config";
+import type { CreateComponentContextContext } from "..";
+import type { StringOrBashExpression } from "../bash/BashExpression";
+import { joinBashExpressions } from "../bash/BashExpression";
 
-import type { CommitInfo } from "../types/context";
 import type { EnvironmentContext } from "../types/environmentContext";
 import { getEnvConfig } from "./getEnvConfig";
 import { getEnvType } from "./getEnvType";
+import { getReviewSlug } from "./getReviewSlug";
 
 const getEnvironmentSlugPrefix = (
-  envConfig: EnvConfigWithComponent,
   env: string,
-  commitInfo?: CommitInfo
-) => {
-  const envType = getEnvType(env, envConfig);
-
-  return envType === "review" && commitInfo
-    ? `${env}-${commitInfo.reviewSlug}`
-    : `${env}`;
+  reviewSlug: StringOrBashExpression | null,
+): StringOrBashExpression => {
+  if (reviewSlug) {
+    return joinBashExpressions([env, reviewSlug], "-");
+  }
+  return env;
 };
 
-export const getEnvironmentContext = (
-  config: Config,
-  env: string,
-  componentName: string,
-  commitInfo?: CommitInfo
-): EnvironmentContext<any, any> => {
+export const getEnvironmentContext = ({
+  env,
+  componentName,
+  config,
+  pipelineType,
+}: CreateComponentContextContext): EnvironmentContext => {
   const envConfigRaw = getEnvConfig(config, componentName, env);
   const envType = getEnvType(env, envConfigRaw);
+  const reviewSlug = getReviewSlug(envConfigRaw, env, pipelineType);
 
-  const environmentSlugPrefix = getEnvironmentSlugPrefix(
-    envConfigRaw,
-    env,
-    commitInfo
-  );
+  const environmentSlugPrefix = getEnvironmentSlugPrefix(env, reviewSlug);
 
-  const environmentSlug = `${environmentSlugPrefix}-${componentName}`;
-  const gitlabEnvironmentName =
-    envType === "review" && commitInfo
-      ? `${env}/${commitInfo.refName}/${componentName}`
-      : `${env}/${componentName}`;
+  const environmentSlug = environmentSlugPrefix.concat(`-${componentName}`);
 
-  const fullName = `${config.customerName}-${config.appName}-${environmentSlug}`;
+  const fullName = joinBashExpressions(
+    [config.customerName, config.appName, environmentSlug],
+    "-",
+  );
 
   return {
     envConfigRaw,
@@ -46,10 +42,10 @@ export const getEnvironmentContext = (
     buildConfigRaw: envConfigRaw.build,
     environmentSlugPrefix,
     environmentSlug,
-    gitlabEnvironmentName,
+    reviewSlug,
+    pipelineType,
     fullName,
     envType,
-    commitInfo,
     componentName,
     env,
     fullConfig: config,

package/src/context/getEnvironmentVariables.ts

@@ -1,89 +1,106 @@
-import { merge } from "lodash";
 import { DEPLOY_TYPES } from "../deploy";
 import type {
-  CommitInfo,
-  Context,
+  ComponentContext,
   EnvironmentEnvVarPart as EnvironmentVariables,
 } from "../types";
-import type { Config, DevLocalEnvConfig } from "../types/config";
-
+import type { DevLocalEnvConfig } from "../types/config";
+
+import type { CreateComponentContextContext, UnspecifiedEnvVars } from "..";
+import type { StringOrBashExpression } from "../bash/BashExpression";
+import { joinBashExpressions } from "../bash/BashExpression";
+import { isStandaloneBuildConfig } from "../build/types";
+import type { EnvironmentContext } from "../types/environmentContext";
+import { createVariableValueContainingReferencesFromString } from "../variables/VariableValueContainingReferences";
+import { resolveAllReferences } from "../variables/resolveAllReferences";
+import { getBuildInfoVariables } from "./getBuildInfoVariables";
 import { getEnvironmentContext } from "./getEnvironmentContext";
+import { transformJobOnlyVars } from "./transformJobOnlyVars";
 import {
-  resolveReferences,
-  translateLegacyFromComponents,
-} from "./resolveReferences";
-import {
-  stringListToSecreteEnvVarList,
   makeSecretEnvVarMapping,
+  stringListToSecreteEnvVarList,
   stringifyValues,
 } from "./utils/envVars";
-import { transformJobOnlyVars } from "./transformJobOnlyVars";
 
 export type SecretEnvVar = {
   key: string;
   // hidden env vars are not shown in config-secrets
   hidden?: boolean;
 };
+
+const getBasePredefinedVariables = (ctx: EnvironmentContext) => {
+  return {
+    ENV_SHORT: ctx.env,
+    APP_DIR: ctx.envConfigRaw.dir,
+    ENV_TYPE: ctx.envType,
+    ...(ctx.envType !== "local" ? getBuildInfoVariables(ctx) : {}),
+  };
+};
+
+type BasePredefinedVariables = ReturnType<typeof getBasePredefinedVariables>;
+
+// we export so that we have later nice autocomplete
+export type PredefinedVariables = BasePredefinedVariables & {
+  HOSTNAME?: StringOrBashExpression;
+  ROOT_URL: StringOrBashExpression;
+  HOSTNAME_INTERNAL: StringOrBashExpression;
+  ROOT_URL_INTERNAL: StringOrBashExpression;
+};
+
 export const getEnvironmentVariables = async (
-  config: Config,
-  componentName: string,
-  env: string,
-  commitInfo?: CommitInfo,
-  alreadyVisited: Record<string, Record<string, boolean>> = {} // to prevent endless loop
+  ctx: CreateComponentContextContext,
+  options: { shouldResolveReferences?: boolean } = {},
 ): Promise<EnvironmentVariables> => {
-  const environmentContext = getEnvironmentContext(
-    config,
-    env,
-    componentName,
-    commitInfo
-  );
+  const environmentContext = getEnvironmentContext(ctx);
 
+  const { config, env, componentName } = ctx;
   const { envConfigRaw, deployConfigRaw, buildConfigRaw, envType } =
     environmentContext;
 
-  const basePredefinedVariables = {
-    ENV_SHORT: env,
-    APP_DIR: envConfigRaw.dir,
-    ENV_TYPE: envType,
-    BUILD_INFO_ID: commitInfo?.buildId,
-    BUILD_INFO_BUILD_TIME: commitInfo?.buildTime,
-    BUILD_INFO_CURRENT_VERSION: commitInfo?.currentVersion,
-  };
+  const basePredefinedVariables =
+    getBasePredefinedVariables(environmentContext);
 
-  let predefinedVariables: Record<string, string | undefined>;
-  let host: string;
-  let url: string;
+  let predefinedVariables: PredefinedVariables & UnspecifiedEnvVars;
+  let host: StringOrBashExpression;
+  let url: StringOrBashExpression;
 
   if (envType === "local") {
     const devLocalConfig: DevLocalEnvConfig = envConfigRaw;
     const port = devLocalConfig.port ?? 3000;
-    host = "localhost:" + port;
+    host = "localhost:" + port.toString();
     url = "http://" + host;
     predefinedVariables = {
+      ...basePredefinedVariables,
       ENV_SHORT: "local",
       ROOT_URL: url,
+      HOSTNAME: host,
+      HOSTNAME_INTERNAL: host,
+      ROOT_URL_INTERNAL: "http://" + host,
       PORT: port.toString(),
     };
   } else {
     const additionalEnvVars = deployConfigRaw
       ? DEPLOY_TYPES[deployConfigRaw.type].getAdditionalEnvVars(
-          environmentContext as never
+          environmentContext as never,
         )
       : {};
 
-    host =
-      envConfigRaw?.host ??
-      additionalEnvVars.HOST_CANONICAL ??
-      "unknown-host.example.com";
-    url = `https://${host}`;
+    const HOSTNAME_INTERNAL =
+      additionalEnvVars.HOSTNAME_INTERNAL ?? "unknown-host.example.com";
+
+    host = envConfigRaw?.host ?? HOSTNAME_INTERNAL;
+    url = joinBashExpressions(["https://", host]);
 
     predefinedVariables = {
       ...basePredefinedVariables,
       // Rails before 6.1 (mis)uses the `HOST` environment variable to specify the IP to bind to
-      ...(config.components[componentName].build.type === "rails"
+      ...(isStandaloneBuildConfig(buildConfigRaw) &&
+      buildConfigRaw.type === "rails"
         ? {}
-        : { HOST: host }),
+        : { HOSTNAME: host }),
       ROOT_URL: url,
+      HOSTNAME_INTERNAL,
+
+      ROOT_URL_INTERNAL: joinBashExpressions(["https://", HOSTNAME_INTERNAL]),
       ...additionalEnvVars,
     };
   }
@@ -91,7 +108,7 @@ export const getEnvironmentVariables = async (
 
   const additionalSecretKeys = deployConfigRaw
     ? DEPLOY_TYPES[deployConfigRaw.type].additionalSecretKeys(
-        environmentContext as never
+        environmentContext as never,
       )
     : [];
 
@@ -102,41 +119,45 @@ export const getEnvironmentVariables = async (
   const secretEnvVars = makeSecretEnvVarMapping(
     env,
     componentName,
-    secretEnvVarKeys
-  );
-  // this is deprecated, we now support: $componentname:FOO
-  const legacyFromComponents = envConfigRaw.vars?.fromComponents ?? {};
-  const publicEnvVarsRawWithLegacyFromComponents = merge(
-    {},
-    translateLegacyFromComponents(legacyFromComponents),
-    publicEnvVarsRaw
+    secretEnvVarKeys,
   );
 
-  const publicEnvVarsRawSanitized = stringifyValues(
-    publicEnvVarsRawWithLegacyFromComponents
+  const publicEnvVarsRawSanitized = stringifyValues(publicEnvVarsRaw);
+
+  const publicEnvVarsUnresolved = Object.fromEntries(
+    Object.entries(publicEnvVarsRawSanitized).map(([key, value]) => [
+      key,
+      createVariableValueContainingReferencesFromString(value, {
+        componentName: ctx.componentName,
+      }),
+    ]),
   );
 
-  const envVarsRaw = addIndexVar({
+  const publicEnvVars =
+    (options.shouldResolveReferences ?? true)
+      ? await resolveAllReferences(
+          publicEnvVarsUnresolved,
+          async (otherComponentName) => {
+            const { envVars: otherEnvVars } = await getEnvironmentVariables(
+              {
+                ...ctx,
+                componentName: otherComponentName,
+              },
+              {
+                shouldResolveReferences: false, // we already do this here with replaceAllReferences recursivly until re replaced all
+              },
+            );
+            return otherEnvVars;
+          },
+        )
+      : publicEnvVarsUnresolved;
+
+  const envVars = addIndexVar({
     ...predefinedVariables,
     ...secretEnvVars,
-    ...publicEnvVarsRawSanitized,
+    ...publicEnvVars,
   });
 
-  const envVars = await resolveReferences(
-    envVarsRaw,
-    async (otherComponentName, alreadyVisited) => {
-      const { envVars: otherEnvVars } = await getEnvironmentVariables(
-        config,
-        otherComponentName,
-        env,
-        commitInfo,
-        alreadyVisited
-      );
-      return otherEnvVars;
-    },
-    alreadyVisited
-  );
-
   return {
     envVars,
     secretEnvVarKeys,
@@ -144,12 +165,15 @@ export const getEnvironmentVariables = async (
       build: await transformJobOnlyVars(
         env,
         componentName,
-        (buildConfigRaw && buildConfigRaw.jobVars) || null
+        (buildConfigRaw &&
+          isStandaloneBuildConfig(buildConfigRaw) &&
+          buildConfigRaw.jobVars) ||
+          null,
       ),
       deploy: await transformJobOnlyVars(
         env,
         componentName,
-        (deployConfigRaw && deployConfigRaw.jobVars) || null
+        (deployConfigRaw && deployConfigRaw.jobVars) || null,
       ),
     },
 
@@ -163,13 +187,17 @@ const sanitizeForEnVar = (s: string) => s.replace(/-/g, "_");
 export const getSecretVarName = (
   env: string,
   componentName: string,
-  key: string
+  key: string,
 ) => `CL_${sanitizeForEnVar(env)}_${sanitizeForEnVar(componentName)}_${key}`; // remove dash from component name
 
-const addIndexVar = (vars: Record<string, unknown>) => ({
+const addIndexVar = <V extends Record<string, unknown>>(
+  vars: V,
+): V & { _ALL_ENV_VAR_KEYS: string } => ({
   ...vars,
   _ALL_ENV_VAR_KEYS: JSON.stringify(Object.keys(vars)),
 });
 
-export const getSecretVarNameForContext = (context: Context, key: string) =>
-  getSecretVarName(context.environment.shortName, context.componentName, key);
+export const getSecretVarNameForContext = (
+  context: ComponentContext,
+  key: string,
+) => getSecretVarName(context.env, context.name, key);