@catladder/pipeline 0.0.0-refactore-catenv-command-validates-argument-78d05852 → 0.0.0-reference-database-url-1befbbee

package/src/context/getEnvironmentVariables.ts

@@ -1,91 +1,101 @@
-import { merge } from "lodash";
 import { DEPLOY_TYPES } from "../deploy";
 import type {
-  CommitInfo,
-  Context,
+  ComponentContext,
   EnvironmentEnvVarPart as EnvironmentVariables,
 } from "../types";
-import type { Config, DevLocalEnvConfig } from "../types/config";
-
+import type { DevLocalEnvConfig } from "../types/config";
+
+import type { CreateComponentContextContext, UnspecifiedEnvVars } from "..";
+import type { StringOrBashExpression } from "../bash/BashExpression";
+import { joinBashExpressions } from "../bash/BashExpression";
+import { isStandaloneBuildConfig } from "../build/types";
+import type { EnvironmentContext } from "../types/environmentContext";
+import { createVariableValueContainingReferencesFromString } from "../variables/VariableValueContainingReferences";
+import { resolveAllReferences } from "../variables/resolveAllReferences";
+import { getBuildInfoVariables } from "./getBuildInfoVariables";
 import { getEnvironmentContext } from "./getEnvironmentContext";
+import { transformJobOnlyVars } from "./transformJobOnlyVars";
 import {
-  resolveReferences,
-  translateLegacyFromComponents,
-} from "./resolveReferences";
-import {
-  stringListToSecreteEnvVarList,
   makeSecretEnvVarMapping,
+  stringListToSecreteEnvVarList,
   stringifyValues,
 } from "./utils/envVars";
-import { transformJobOnlyVars } from "./transformJobOnlyVars";
 
 export type SecretEnvVar = {
   key: string;
   // hidden env vars are not shown in config-secrets
   hidden?: boolean;
 };
+
+const getBasePredefinedVariables = (ctx: EnvironmentContext) => {
+  return {
+    ENV_SHORT: ctx.env,
+    APP_DIR: ctx.envConfigRaw.dir,
+    ENV_TYPE: ctx.envType,
+    ...(ctx.envType !== "local" ? getBuildInfoVariables(ctx) : {}),
+  };
+};
+
+type BasePredefinedVariables = ReturnType<typeof getBasePredefinedVariables>;
+
+// we export so that we have later nice autocomplete
+export type PredefinedVariables = BasePredefinedVariables & {
+  HOSTNAME: StringOrBashExpression;
+  ROOT_URL: StringOrBashExpression;
+  HOSTNAME_INTERNAL: StringOrBashExpression;
+  ROOT_URL_INTERNAL: StringOrBashExpression;
+};
+
 export const getEnvironmentVariables = async (
-  config: Config,
-  componentName: string,
-  env: string,
-  commitInfo?: CommitInfo,
-  alreadyVisited: Record<string, Record<string, boolean>> = {} // to prevent endless loop
+  ctx: CreateComponentContextContext,
+  options: { shouldResolveReferences?: boolean } = {},
 ): Promise<EnvironmentVariables> => {
-  const environmentContext = getEnvironmentContext(
-    config,
-    env,
-    componentName,
-    commitInfo
-  );
+  const environmentContext = getEnvironmentContext(ctx);
 
+  const { config, env, componentName } = ctx;
   const { envConfigRaw, deployConfigRaw, buildConfigRaw, envType } =
     environmentContext;
 
-  const basePredefinedVariables = {
-    ENV_SHORT: env,
-    APP_DIR: envConfigRaw.dir,
-    ENV_TYPE: envType,
-    BUILD_INFO_ID: commitInfo?.buildId,
-    BUILD_INFO_BUILD_TIME: commitInfo?.buildTime,
-    BUILD_INFO_CURRENT_VERSION: commitInfo?.currentVersion,
-  };
+  const basePredefinedVariables =
+    getBasePredefinedVariables(environmentContext);
 
-  let predefinedVariables: Record<string, string | undefined>;
-  let host: string;
-  let url: string;
+  let predefinedVariables: PredefinedVariables & UnspecifiedEnvVars;
+  let host: StringOrBashExpression;
+  let url: StringOrBashExpression;
 
   if (envType === "local") {
     const devLocalConfig: DevLocalEnvConfig = envConfigRaw;
     const port = devLocalConfig.port ?? 3000;
-    host = "localhost:" + port;
+    host = "localhost:" + port.toString();
     url = "http://" + host;
     predefinedVariables = {
+      ...basePredefinedVariables,
       ENV_SHORT: "local",
       ROOT_URL: url,
-      HOST_INTERNAL: host,
+      HOSTNAME: host,
+      HOSTNAME_INTERNAL: host,
       ROOT_URL_INTERNAL: "http://" + host,
       PORT: port.toString(),
     };
   } else {
     const additionalEnvVars = deployConfigRaw
       ? DEPLOY_TYPES[deployConfigRaw.type].getAdditionalEnvVars(
-          environmentContext as never
+          environmentContext as never,
         )
       : {};
-    const HOST_INTERNAL =
-      additionalEnvVars.HOST_INTERNAL ?? "unknown-host.example.com";
-    host = envConfigRaw?.host ?? HOST_INTERNAL;
-    url = `https://${host}`;
+
+    const HOSTNAME_INTERNAL =
+      additionalEnvVars.HOSTNAME_INTERNAL ?? "unknown-host.example.com";
+
+    host = envConfigRaw?.host ?? HOSTNAME_INTERNAL;
+    url = joinBashExpressions(["https://", host]);
 
     predefinedVariables = {
       ...basePredefinedVariables,
-      // Rails before 6.1 (mis)uses the `HOST` environment variable to specify the IP to bind to
-      ...(config.components[componentName].build.type === "rails"
-        ? {}
-        : { HOST: host }),
+      HOSTNAME: host,
       ROOT_URL: url,
-      HOST_CANONICAL: HOST_INTERNAL, // legacy
-      ROOT_URL_INTERNAL: "https://" + HOST_INTERNAL,
+      HOSTNAME_INTERNAL,
+      ROOT_URL_INTERNAL: joinBashExpressions(["https://", HOSTNAME_INTERNAL]),
       ...additionalEnvVars,
     };
   }
@@ -93,7 +103,7 @@ export const getEnvironmentVariables = async (
 
   const additionalSecretKeys = deployConfigRaw
     ? DEPLOY_TYPES[deployConfigRaw.type].additionalSecretKeys(
-        environmentContext as never
+        environmentContext as never,
       )
     : [];
 
@@ -104,41 +114,45 @@ export const getEnvironmentVariables = async (
   const secretEnvVars = makeSecretEnvVarMapping(
     env,
     componentName,
-    secretEnvVarKeys
-  );
-  // this is deprecated, we now support: $componentname:FOO
-  const legacyFromComponents = envConfigRaw.vars?.fromComponents ?? {};
-  const publicEnvVarsRawWithLegacyFromComponents = merge(
-    {},
-    translateLegacyFromComponents(legacyFromComponents),
-    publicEnvVarsRaw
+    secretEnvVarKeys,
   );
 
-  const publicEnvVarsRawSanitized = stringifyValues(
-    publicEnvVarsRawWithLegacyFromComponents
+  const publicEnvVarsRawSanitized = stringifyValues(publicEnvVarsRaw);
+
+  const publicEnvVarsUnresolved = Object.fromEntries(
+    Object.entries(publicEnvVarsRawSanitized).map(([key, value]) => [
+      key,
+      createVariableValueContainingReferencesFromString(value, {
+        componentName: ctx.componentName,
+      }),
+    ]),
   );
 
-  const envVarsRaw = addIndexVar({
+  const publicEnvVars =
+    (options.shouldResolveReferences ?? true)
+      ? await resolveAllReferences(
+          publicEnvVarsUnresolved,
+          async (otherComponentName) => {
+            const { envVars: otherEnvVars } = await getEnvironmentVariables(
+              {
+                ...ctx,
+                componentName: otherComponentName,
+              },
+              {
+                shouldResolveReferences: false, // we already do this here with replaceAllReferences recursivly until re replaced all
+              },
+            );
+            return otherEnvVars;
+          },
+        )
+      : publicEnvVarsUnresolved;
+
+  const envVars = addIndexVar({
     ...predefinedVariables,
     ...secretEnvVars,
-    ...publicEnvVarsRawSanitized,
+    ...publicEnvVars,
   });
 
-  const envVars = await resolveReferences(
-    envVarsRaw,
-    async (otherComponentName, alreadyVisited) => {
-      const { envVars: otherEnvVars } = await getEnvironmentVariables(
-        config,
-        otherComponentName,
-        env,
-        commitInfo,
-        alreadyVisited
-      );
-      return otherEnvVars;
-    },
-    alreadyVisited
-  );
-
   return {
     envVars,
     secretEnvVarKeys,
@@ -146,17 +160,17 @@ export const getEnvironmentVariables = async (
       build: await transformJobOnlyVars(
         env,
         componentName,
-        (buildConfigRaw && buildConfigRaw.jobVars) || null
+        (buildConfigRaw &&
+          isStandaloneBuildConfig(buildConfigRaw) &&
+          buildConfigRaw.jobVars) ||
+          null,
       ),
       deploy: await transformJobOnlyVars(
         env,
         componentName,
-        (deployConfigRaw && deployConfigRaw.jobVars) || null
+        (deployConfigRaw && deployConfigRaw.jobVars) || null,
       ),
     },
-
-    host,
-    url,
   };
 };
 
@@ -165,13 +179,17 @@ const sanitizeForEnVar = (s: string) => s.replace(/-/g, "_");
 export const getSecretVarName = (
   env: string,
   componentName: string,
-  key: string
+  key: string,
 ) => `CL_${sanitizeForEnVar(env)}_${sanitizeForEnVar(componentName)}_${key}`; // remove dash from component name
 
-const addIndexVar = (vars: Record<string, unknown>) => ({
+const addIndexVar = <V extends Record<string, unknown>>(
+  vars: V,
+): V & { _ALL_ENV_VAR_KEYS: string } => ({
   ...vars,
   _ALL_ENV_VAR_KEYS: JSON.stringify(Object.keys(vars)),
 });
 
-export const getSecretVarNameForContext = (context: Context, key: string) =>
-  getSecretVarName(context.environment.shortName, context.componentName, key);
+export const getSecretVarNameForContext = (
+  context: ComponentContext,
+  key: string,
+) => getSecretVarName(context.env, context.name, key);

package/src/context/getLabels.ts

@@ -1,5 +1,5 @@
 import slugify from "slugify";
-import type { Context } from "../types";
+import type { ComponentContext } from "../types";
 
 const sanitize = (value?: string) => {
   if (!value) return value;
@@ -7,14 +7,17 @@ const sanitize = (value?: string) => {
   // slugify should do the job
   return slugify(value).toLowerCase();
 };
-export const getLabels = (context: Context) => {
+export const getLabels = (context: ComponentContext) => {
   const labels = {
     "customer-name": sanitize(context.fullConfig.customerName),
-    "component-name": sanitize(context.componentName),
+    "component-name": sanitize(context.name),
     "app-name": sanitize(context.fullConfig.appName),
     "env-type": sanitize(context.environment.envType),
-    "env-name": sanitize(context.environment.shortName),
-    "build-type": sanitize(context.componentConfig.build?.type),
+    "env-name": sanitize(context.env),
+    "build-type":
+      context.build.type === "disabled"
+        ? "disabled"
+        : sanitize(context.build.buildType),
     ...(context.fullConfig.meta?.labels ?? {}),
   };
   return labels;

package/src/context/getReviewSlug.ts

@@ -0,0 +1,27 @@
+import {
+  BashExpression,
+  type StringOrBashExpression,
+} from "../bash/BashExpression";
+import type { BashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
+import { getBashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
+import type { PipelineType } from "../types";
+import type { EnvConfigWithComponent } from "../types/config";
+import { getEnvType } from "./getEnvType";
+const REVIEW_SLUG: BashExpressionPerPipelineType = {
+  default: "unknown-review-slug",
+  gitlab: new BashExpression(
+    `$([ -n "$CI_MERGE_REQUEST_IID" ] && echo "mr$CI_MERGE_REQUEST_IID" || { [ -n "$CI_COMMIT_REF_SLUG" ] && echo "$CI_COMMIT_REF_SLUG" || echo "unknown"; })`,
+  ),
+};
+
+export const getReviewSlug = (
+  envConfig: EnvConfigWithComponent,
+  env: string,
+  pipelineType?: PipelineType,
+): StringOrBashExpression | null => {
+  const envType = getEnvType(env, envConfig);
+  if (envType === "review") {
+    return getBashExpressionPerPipelineType(REVIEW_SLUG, pipelineType);
+  }
+  return null; // not a review app;
+};

package/src/context/index.ts

@@ -1,67 +1,3 @@
-import { BUILD_TYPES } from "../build";
-import type { BuildConfig, BuildConfigType } from "../build/types";
-import { DEPLOY_TYPES } from "../deploy";
-import type { DeployConfig, DeployConfigType } from "../deploy/types";
-import type { Config } from "../types/config";
-import type { CommitInfo, Context, PackageManagerInfo } from "../types/context";
-import type { PartialDeep } from "../types/utils";
-import { mergeWithMergingArrays } from "../utils";
-import { getEnvironment } from "./getEnvironment";
-import { getEnvironmentContext } from "./getEnvironmentContext";
-
+export * from "./createComponentContext";
 export * from "./getEnvironment";
 export * from "./getEnvironmentVariables";
-
-export const createContext = async (
-  config: Config,
-  componentName: string,
-  env: string,
-  commitInfo?: CommitInfo,
-  packageManagerInfo?: PackageManagerInfo
-): Promise<Context> => {
-  if (!/^[a-z0-9-]+$/.test(componentName)) {
-    throw new Error(
-      "componentName may only contain lower case letters, numbers and -"
-    );
-  }
-
-  const envContext = getEnvironmentContext(
-    config,
-    env,
-    componentName,
-    commitInfo
-  );
-
-  const componentConfigWithoutDefaults = envContext.envConfigRaw;
-  const defaults: {
-    build: PartialDeep<BuildConfig>;
-    deploy: PartialDeep<DeployConfig>;
-  } = componentConfigWithoutDefaults.deploy
-    ? {
-        build:
-          BUILD_TYPES[
-            componentConfigWithoutDefaults.build.type as BuildConfigType
-          ].defaults(envContext),
-        deploy:
-          DEPLOY_TYPES[
-            componentConfigWithoutDefaults.deploy.type as DeployConfigType
-          ].defaults(envContext),
-      }
-    : {
-        build: {},
-        deploy: {},
-      };
-  const componentConfig = mergeWithMergingArrays(
-    defaults,
-    componentConfigWithoutDefaults
-  );
-
-  return {
-    fullConfig: config,
-    componentConfig,
-    componentName,
-    environment: await getEnvironment(config, componentName, env, commitInfo),
-    commitInfo,
-    packageManagerInfo,
-  };
-};

package/src/context/transformJobOnlyVars.ts

@@ -1,4 +1,4 @@
-import type { EnvironmentEnvVars } from "..";
+import type { SecretEnvVar, UnspecifiedEnvVars } from "..";
 import type { EnvVars } from "../types/config";
 import {
   makeSecretEnvVarMapping,
@@ -16,8 +16,11 @@ import {
 export const transformJobOnlyVars = async (
   env: string,
   componentName: string,
-  vars: EnvVars | null
-): Promise<EnvironmentEnvVars> => {
+  vars: EnvVars | null,
+): Promise<{
+  envVars: UnspecifiedEnvVars;
+  secretEnvVarKeys: SecretEnvVar[];
+}> => {
   if (!vars) {
     return {
       envVars: {},

package/src/context/utils/envVars.ts

@@ -1,13 +1,14 @@
 import { isObject } from "lodash";
 import type { SecretEnvVar } from "../getEnvironmentVariables";
 import { getSecretVarName } from "../getEnvironmentVariables";
+import { getBashVariable } from "../../bash/BashExpression";
 
 export const stringifyValues = (obj: Record<string, unknown>) =>
   Object.fromEntries(
     Object.entries(obj).map(([key, value]) => [
       key,
       isObject(value) ? JSON.stringify(value) : `${value}`,
-    ])
+    ]),
   );
 
 export const stringListToSecreteEnvVarList = (keys: string[]): SecretEnvVar[] =>
@@ -15,12 +16,12 @@ export const stringListToSecreteEnvVarList = (keys: string[]): SecretEnvVar[] =>
 export const makeSecretEnvVarMapping = (
   env: string,
   componentName: string,
-  secretEnvVars: SecretEnvVar[]
+  secretEnvVars: SecretEnvVar[],
 ) => {
   return Object.fromEntries(
     secretEnvVars.map(({ key }) => [
       key,
-      `$${getSecretVarName(env, componentName, key)}`,
-    ])
+      getBashVariable(getSecretVarName(env, componentName, key)),
+    ]),
   );
 };

package/src/deploy/base/deploy.ts

@@ -1,10 +1,15 @@
+import { createJobCacheFromCacheConfigs } from "../../build/cache/createJobCache";
 import {
   getDockerImageVariables,
   requiresDockerBuild,
 } from "../../build/docker";
 import { SBOM_BUILD_JOB_NAME } from "../../build/sbom";
-import type { Context } from "../../types/context";
-import type { CatladderJob } from "../../types/jobs";
+import type { DeployJobDefinition } from "../../types";
+import {
+  componentContextHasWorkspaceBuild,
+  type ComponentContext,
+} from "../../types/context";
+import type { BaseStage, CatladderJob } from "../../types/jobs";
 import { sbomDeactivated } from "../sbom";
 import { contextIsStoppable } from "../utils";
 import { STOP_JOB_NAME } from "./stop";
@@ -13,13 +18,9 @@ import { DEPLOY_RUNNER_VARIABLES } from "./variables";
 
 export const DEPLOY_JOB_NAME = "🚀 Deploy";
 
-export type DeployJobDefinition = Pick<
-  CatladderJob,
-  "script" | "variables" | "image" | "cache" | "artifacts" | "services"
->;
 export const createDeployJob = (
-  context: Context,
-  jobDefinition: DeployJobDefinition
+  context: ComponentContext,
+  jobDefinition: DeployJobDefinition,
 ): CatladderJob => {
   const hasDocker = requiresDockerBuild(context);
   const isStoppable = contextIsStoppable(context);
@@ -28,28 +29,30 @@ export const createDeployJob = (
     context.environment.envType === "review"
       ? "1 week"
       : context.environment.envType === "dev"
-      ? "4 weeks"
-      : undefined;
+        ? "4 weeks"
+        : undefined;
+
+  const deployConfig = context.deploy?.config;
 
   // if auto or manual is configured explicitly, use that
   const whenDeployDefined =
-    context.componentConfig.deploy && context.componentConfig.deploy.when
-      ? context.componentConfig.deploy.when
-      : undefined;
+    deployConfig && deployConfig.when ? deployConfig.when : undefined;
   // otherwise auto deploy if env is not prod. If its prod, deploy automatically if stage is disabled
   const whenDeployDefault =
     context.environment.envType !== "prod"
       ? "auto" // is not stage, auto deploy
       : context.componentConfig.env?.stage === false
-      ? "auto" // is prod, but no staging, auto deploy
-      : "manual"; // manually deploy
+        ? "auto" // is prod, but no staging, auto deploy
+        : "manual"; // manually deploy
   const whenDeploy = whenDeployDefined ? whenDeployDefined : whenDeployDefault;
 
   return {
     name: DEPLOY_JOB_NAME,
     script: jobDefinition.script,
     image: jobDefinition.image,
-    cache: jobDefinition.cache,
+    cache: jobDefinition.cache
+      ? createJobCacheFromCacheConfigs(context, jobDefinition.cache)
+      : undefined,
     artifacts: jobDefinition.artifacts,
     services: jobDefinition.services,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
@@ -58,57 +61,70 @@ export const createDeployJob = (
       ...(sbomDeactivated(context)
         ? []
         : [{ job: SBOM_BUILD_JOB_NAME, artifacts: true }]),
-      ...(context.componentConfig.deploy
-        ? context.componentConfig.deploy.waitFor?.map((c) => ({
+      ...(deployConfig
+        ? (deployConfig.waitFor?.map((c) => ({
             componentName: c,
             job: DEPLOY_JOB_NAME,
             artifacts: false,
-          })) ?? []
+          })) ?? [])
         : []),
     ],
     // we don't want to deploy when there is a broken test
     needsStages: [
-      {
-        stage: "build",
-        artifacts: hasDocker ? false : true, // we asume that no-docker deployments need build artifacts,
-      },
+      ...(componentContextHasWorkspaceBuild(context)
+        ? hasDocker // docker build is per component,
+          ? [
+              // we don't need artifacts, but have to wait for the component build
+              {
+                stage: "build" as BaseStage,
+                artifacts: false,
+              },
+            ]
+          : [
+              {
+                // pick build artifacts from workspace build
+                stage: "build" as BaseStage,
+                artifacts: true,
+                workspaceName: context.build.workspaceName,
+              },
+            ]
+        : [
+            {
+              stage: "build" as BaseStage,
+              artifacts: hasDocker ? false : true, // we asume that no-docker deployments need build artifacts,
+            },
+          ]),
       {
         stage: "test",
         artifacts: false,
+        // use test from workspace build
+        workspaceName: componentContextHasWorkspaceBuild(context)
+          ? context.build.workspaceName
+          : undefined,
       },
-    ], // workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/220758
-    rules: [
-      whenDeploy === "auto"
-        ? {
-            when: "on_success",
-          }
-        : {
-            when: "manual",
-          },
     ],
+    when: whenDeploy === "auto" ? "on_success" : "manual",
+
     allow_failure: whenDeploy === "manual" ? true : false,
     stage: "deploy",
     variables: {
-      ...DEPLOY_RUNNER_VARIABLES,
       ...context.environment.envVars,
       ...(hasDocker ? getDockerImageVariables(context) : {}),
       ...context.environment.jobOnlyVars.deploy.envVars,
-      ...(context.componentConfig.deploy
-        ? context.componentConfig.deploy.extraVars ?? {}
-        : {}),
+
       ...jobDefinition.variables,
     },
-    environment: {
-      ...context.environment.gitlabEnvironment,
-      ...(isStoppable
-        ? {
-            on_stop: STOP_JOB_NAME,
-            auto_stop_in: autoStop,
-          }
-        : {}),
+    runnerVariables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+      ...(jobDefinition.runnerVariables ?? {}),
+      ...(deployConfig ? (deployConfig.runnerVariables ?? {}) : {}),
     },
-    jobTags: context.componentConfig.deploy
-      ? context.componentConfig.deploy.jobTags
-      : undefined,
+    environment: isStoppable
+      ? {
+          on_stop: STOP_JOB_NAME,
+          auto_stop_in: autoStop,
+        }
+      : {},
+    jobTags: deployConfig ? deployConfig.jobTags : undefined,
   };
 };

package/src/deploy/base/index.ts

@@ -1,16 +1,21 @@
-import type { Context } from "../../types/context";
+import type { ComponentContext } from "../../types/context";
+import type {
+  DeployJobDefinition,
+  RollbackJobDefinition,
+  StopJobDefinition,
+} from "../../types/jobDefinition";
 import type { CatladderJob } from "../../types/jobs";
-import { createDeployJob, type DeployJobDefinition } from "./deploy";
-import { createRollbackJob, type RollbackJobDefinition } from "./rollback";
-import { createStopJob, type StopJobDefinition } from "./stop";
+import { createDeployJob } from "./deploy";
+import { createRollbackJob } from "./rollback";
+import { createStopJob } from "./stop";
 
 export const createDeployementJobs = (
-  context: Context,
+  context: ComponentContext,
   definitions: {
     deploy: DeployJobDefinition;
     stop?: StopJobDefinition;
     rollback?: RollbackJobDefinition;
-  }
+  },
 ): CatladderJob[] => {
   return [
     createDeployJob(context, definitions.deploy),