@catladder/pipeline 1.144.0 → 1.145.0

package/src/config/readConfig.ts

@@ -1,6 +1,6 @@
 import { existsSync, readFileSync } from "fs";
 import { register } from "ts-node";
-import { load } from "js-yaml";
+import { parse } from "yaml";
 import type { Config } from "../types";
 
 // allows us to load ts files
@@ -33,14 +33,13 @@ export const readConfigSync = (
       return {
         path: filePath,
         ext: found,
-        // eslint-disable-next-line @typescript-eslint/no-var-requires
         config: requireUncached(filePath).default,
       };
     } else {
       return {
         path: filePath,
         ext: found,
-        config: load(readFileSync(filePath, { encoding: "utf-8" })) as Config,
+        config: parse(readFileSync(filePath, { encoding: "utf-8" })) as Config,
       };
     }
   }

package/src/constants.ts

@@ -1,2 +1,5 @@
-export const PIPELINE_IMAGE_TAG = process.env.INLINE_PIPELINE_IMAGE_TAG || "";
-export const DOCKER_REGISTRY = process.env.INLINE_DOCKER_REGISTRY || "";
+export const PIPELINE_IMAGE_TAG =
+  process.env.INLINE_PIPELINE_IMAGE_TAG || "latest";
+export const DOCKER_REGISTRY =
+  process.env.INLINE_DOCKER_REGISTRY ||
+  "git.panter.ch:5001/catladder/catladder";

package/src/context/__tests__/resolveReferences.test.ts

@@ -1,8 +1,13 @@
+import type { BashExpression } from "../../bash/BashExpression";
 import {
   resolveReferences,
   translateLegacyFromComponents,
 } from "../resolveReferences";
 
+const unpackBashExpressions = (obj: Record<string, string | BashExpression>) =>
+  Object.fromEntries(
+    Object.entries(obj).map(([key, value]) => [key, value.toString()])
+  );
 describe("resolveReferences", () => {
   it("replaces occurences of ${componentName:VARIABLE_NAME}", async () => {
     const variables = {
@@ -21,7 +26,7 @@ describe("resolveReferences", () => {
       return otherVariables[componentName];
     });
 
-    expect(result).toEqual({
+    expect(unpackBashExpressions(result)).toEqual({
       a: "hello world",
       b: "a replaced value looks like this: 'foo from api', nice!",
     });
@@ -34,7 +39,7 @@ describe("resolveReferences", () => {
     };
     const result = await resolveReferences(variables);
 
-    expect(result).toEqual({
+    expect(unpackBashExpressions(result)).toEqual({
       FOO: "hello world",
       BAR: "this: hello world!",
     });
@@ -54,7 +59,7 @@ describe("resolveReferences", () => {
       return otherVariables[componentName];
     });
 
-    expect(result).toEqual({
+    expect(unpackBashExpressions(result)).toEqual({
       FOO: "hello from foo from api",
       BAR: "this: hello from foo from api!",
     });
@@ -70,7 +75,7 @@ describe("resolveReferences", () => {
       return otherVariables[componentName];
     });
 
-    expect(result).toEqual({
+    expect(unpackBashExpressions(result)).toEqual({
       a: "hello world",
       b: "a not found value looks like this: '${api:FOO}'",
     });
@@ -92,7 +97,7 @@ describe("resolveReferences", () => {
       return otherVariables[componentName];
     });
 
-    expect(result).toEqual({
+    expect(unpackBashExpressions(result)).toEqual({
       a: "value is hi, foo from api!",
     });
   });
@@ -113,7 +118,7 @@ describe("resolveReferences", () => {
       return otherVariables[componentName];
     });
 
-    expect(result).toEqual({
+    expect(unpackBashExpressions(result)).toEqual({
       a: "value is frontend api ${frontend:FOO}!",
     });
   });

package/src/context/getBuildInfoVariables.ts

@@ -0,0 +1,40 @@
+import { BashExpression } from "../bash/BashExpression";
+import type { BashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
+import { getBashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
+import type { EnvironmentContext } from "../types/environmentContext";
+
+const BUILD_TIME: BashExpressionPerPipelineType = {
+  default: "unknown-build-time",
+  gitlab: new BashExpression(`$CI_JOB_STARTED_AT`),
+};
+
+const BUILD_ID: BashExpressionPerPipelineType = {
+  default: new BashExpression(
+    `$(git describe --tags 2>/dev/null || git rev-parse HEAD)`
+  ),
+};
+const CURRENT_VERSION: BashExpressionPerPipelineType = {
+  default: new BashExpression(
+    // because we do shallow fetch, we need to ask the origin
+    `$(tag=$(git ls-remote origin "refs/tags/v*[0-9]" 2>/dev/null | cut -f 2- | sort -V | tail -1 | sed 's/refs\\/tags\\///'); [ -z "$tag" ] && echo "v0.0.0" || echo "$tag")`
+  ),
+};
+
+export const getBuildInfoVariables = (ctx: EnvironmentContext<any, any>) => {
+  const { pipelineType } = ctx;
+
+  return {
+    BUILD_INFO_BUILD_ID: getBashExpressionPerPipelineType(
+      BUILD_ID,
+      pipelineType
+    ),
+    BUILD_INFO_BUILD_TIME: getBashExpressionPerPipelineType(
+      BUILD_TIME,
+      pipelineType
+    ),
+    BUILD_INFO_CURRENT_VERSION: getBashExpressionPerPipelineType(
+      CURRENT_VERSION,
+      pipelineType
+    ),
+  };
+};

package/src/context/getEnvironment.ts

@@ -1,47 +1,24 @@
-import { isOfDeployType } from "../deploy";
-import type { Config } from "../types/config";
+import type { CreateContextContext } from "..";
 
-import type { CommitInfo, Environment } from "../types/context";
+import type { Environment } from "../types/context";
 import { getEnvironmentContext } from "./getEnvironmentContext";
 import { getEnvironmentVariables } from "./getEnvironmentVariables";
 
 export const getEnvironment = async (
-  config: Config,
-  componentName: string,
-  env: string,
-  commitInfo?: CommitInfo
+  ctx: CreateContextContext
 ): Promise<Environment> => {
-  const variables = await getEnvironmentVariables(
-    config,
-    componentName,
-    env,
-    commitInfo
-  );
+  const { env } = ctx;
+  const variables = await getEnvironmentVariables(ctx);
 
-  const envContext = getEnvironmentContext(
-    config,
-    env,
-    componentName,
-    commitInfo
-  );
+  const envContext = getEnvironmentContext(ctx);
 
   const envType = envContext.envType;
-  const deployConfig = config.components[componentName].deploy;
-
-  const gitlabEnvironment = {
-    name: envContext.gitlabEnvironmentName,
-    ...(!isOfDeployType(deployConfig, "google-cloudrun") ||
-    deployConfig.service !== false
-      ? { url: variables.url }
-      : {}),
-  };
 
   return {
     envType,
-
-    gitlabEnvironment,
     fullName: envContext.fullName,
     slugPrefix: envContext.environmentSlugPrefix,
+    reviewSlug: envContext.reviewSlug,
     slug: envContext.environmentSlug,
     shortName: env,
     ...variables,

package/src/context/getEnvironmentContext.ts

@@ -1,44 +1,40 @@
-import type { Config, EnvConfigWithComponent } from "../types/config";
+import type { CreateContextContext } from "..";
+import type { StringOrBashExpression } from "../bash/BashExpression";
+import { joinBashExpressions } from "../bash/BashExpression";
 
-import type { CommitInfo } from "../types/context";
 import type { EnvironmentContext } from "../types/environmentContext";
 import { getEnvConfig } from "./getEnvConfig";
 import { getEnvType } from "./getEnvType";
+import { getReviewSlug } from "./getReviewSlug";
 
 const getEnvironmentSlugPrefix = (
-  envConfig: EnvConfigWithComponent,
   env: string,
-  commitInfo?: CommitInfo
-) => {
-  const envType = getEnvType(env, envConfig);
-
-  return envType === "review" && commitInfo
-    ? `${env}-${commitInfo.reviewSlug}`
-    : `${env}`;
+  reviewSlug: StringOrBashExpression | null
+): StringOrBashExpression => {
+  if (reviewSlug) {
+    return joinBashExpressions([env, reviewSlug], "-");
+  }
+  return env;
 };
 
-export const getEnvironmentContext = (
-  config: Config,
-  env: string,
-  componentName: string,
-  commitInfo?: CommitInfo
-): EnvironmentContext<any, any> => {
+export const getEnvironmentContext = ({
+  env,
+  componentName,
+  config,
+  pipelineType,
+}: CreateContextContext): EnvironmentContext<any, any> => {
   const envConfigRaw = getEnvConfig(config, componentName, env);
   const envType = getEnvType(env, envConfigRaw);
+  const reviewSlug = getReviewSlug(envConfigRaw, env, pipelineType);
 
-  const environmentSlugPrefix = getEnvironmentSlugPrefix(
-    envConfigRaw,
-    env,
-    commitInfo
-  );
+  const environmentSlugPrefix = getEnvironmentSlugPrefix(env, reviewSlug);
 
-  const environmentSlug = `${environmentSlugPrefix}-${componentName}`;
-  const gitlabEnvironmentName =
-    envType === "review" && commitInfo
-      ? `${env}/${commitInfo.refName}/${componentName}`
-      : `${env}/${componentName}`;
+  const environmentSlug = environmentSlugPrefix.concat(`-${componentName}`);
 
-  const fullName = `${config.customerName}-${config.appName}-${environmentSlug}`;
+  const fullName = joinBashExpressions(
+    [config.customerName, config.appName, environmentSlug],
+    "-"
+  );
 
   return {
     envConfigRaw,
@@ -46,10 +42,10 @@ export const getEnvironmentContext = (
     buildConfigRaw: envConfigRaw.build,
     environmentSlugPrefix,
     environmentSlug,
-    gitlabEnvironmentName,
+    reviewSlug,
+    pipelineType,
     fullName,
     envType,
-    commitInfo,
     componentName,
     env,
     fullConfig: config,

package/src/context/getEnvironmentVariables.ts

@@ -1,67 +1,86 @@
 import { merge } from "lodash";
 import { DEPLOY_TYPES } from "../deploy";
 import type {
-  CommitInfo,
   Context,
   EnvironmentEnvVarPart as EnvironmentVariables,
 } from "../types";
-import type { Config, DevLocalEnvConfig } from "../types/config";
+import type { DevLocalEnvConfig } from "../types/config";
 
+import type { CreateContextContext, UnspecifiedEnvVars } from "..";
+import type { StringOrBashExpression } from "../bash/BashExpression";
+import { getBashVariable, joinBashExpressions } from "../bash/BashExpression";
+import type { EnvironmentContext } from "../types/environmentContext";
+import { getBuildInfoVariables } from "./getBuildInfoVariables";
 import { getEnvironmentContext } from "./getEnvironmentContext";
 import {
   resolveReferences,
   translateLegacyFromComponents,
 } from "./resolveReferences";
+import { transformJobOnlyVars } from "./transformJobOnlyVars";
 import {
-  stringListToSecreteEnvVarList,
   makeSecretEnvVarMapping,
+  stringListToSecreteEnvVarList,
   stringifyValues,
 } from "./utils/envVars";
-import { transformJobOnlyVars } from "./transformJobOnlyVars";
 
 export type SecretEnvVar = {
   key: string;
   // hidden env vars are not shown in config-secrets
   hidden?: boolean;
 };
+
+const getBasePredefinedVariables = (ctx: EnvironmentContext<any, any>) => {
+  return {
+    ENV_SHORT: ctx.env,
+    APP_DIR: ctx.envConfigRaw.dir,
+    ENV_TYPE: ctx.envType,
+    ...(ctx.envType !== "local" ? getBuildInfoVariables(ctx) : {}),
+  };
+};
+
+type BasePredefinedVariables = ReturnType<typeof getBasePredefinedVariables>;
+
+// we export so that we have later nice autocomplete
+export type PredefinedVariables = BasePredefinedVariables & {
+  /**
+   * undefined in rails, Rails before 6.1 (mis)uses the `HOST` environment variable to specify the IP to bind to
+   */
+  HOST?: StringOrBashExpression;
+  ROOT_URL: StringOrBashExpression;
+  HOST_INTERNAL: StringOrBashExpression;
+  ROOT_URL_INTERNAL: StringOrBashExpression;
+};
+
 export const getEnvironmentVariables = async (
-  config: Config,
-  componentName: string,
-  env: string,
-  commitInfo?: CommitInfo,
+  ctx: CreateContextContext,
   alreadyVisited: Record<string, Record<string, boolean>> = {} // to prevent endless loop
 ): Promise<EnvironmentVariables> => {
-  const environmentContext = getEnvironmentContext(
-    config,
-    env,
-    componentName,
-    commitInfo
-  );
+  const environmentContext = getEnvironmentContext(ctx);
 
+  const { config, env, componentName } = ctx;
   const { envConfigRaw, deployConfigRaw, buildConfigRaw, envType } =
     environmentContext;
 
-  const basePredefinedVariables = {
-    ENV_SHORT: env,
-    APP_DIR: envConfigRaw.dir,
-    ENV_TYPE: envType,
-    BUILD_INFO_ID: commitInfo?.buildId,
-    BUILD_INFO_BUILD_TIME: commitInfo?.buildTime,
-    BUILD_INFO_CURRENT_VERSION: commitInfo?.currentVersion,
-  };
+  const basePredefinedVariables =
+    getBasePredefinedVariables(environmentContext);
 
-  let predefinedVariables: Record<string, string | undefined>;
-  let host: string;
-  let url: string;
+  let predefinedVariables: PredefinedVariables & UnspecifiedEnvVars;
+  let host: StringOrBashExpression;
+  let url: StringOrBashExpression;
 
   if (envType === "local") {
     const devLocalConfig: DevLocalEnvConfig = envConfigRaw;
     const port = devLocalConfig.port ?? 3000;
-    host = "localhost:" + port;
+    host = "localhost:" + port.toString();
     url = "http://" + host;
     predefinedVariables = {
+      ...basePredefinedVariables,
       ENV_SHORT: "local",
       ROOT_URL: url,
+      // Rails before 6.1 (mis)uses the `HOST` environment variable to specify the IP to bind to
+      ...(config.components[componentName].build.type === "rails"
+        ? {}
+        : { HOST: host }),
       HOST_INTERNAL: host,
       ROOT_URL_INTERNAL: "http://" + host,
       PORT: port.toString(),
@@ -72,10 +91,12 @@ export const getEnvironmentVariables = async (
           environmentContext as never
         )
       : {};
+
     const HOST_INTERNAL =
       additionalEnvVars.HOST_INTERNAL ?? "unknown-host.example.com";
+
     host = envConfigRaw?.host ?? HOST_INTERNAL;
-    url = `https://${host}`;
+    url = joinBashExpressions(["https://", host]);
 
     predefinedVariables = {
       ...basePredefinedVariables,
@@ -84,8 +105,10 @@ export const getEnvironmentVariables = async (
         ? {}
         : { HOST: host }),
       ROOT_URL: url,
-      HOST_CANONICAL: HOST_INTERNAL, // legacy
-      ROOT_URL_INTERNAL: "https://" + HOST_INTERNAL,
+      HOST_INTERNAL,
+      /**@deprecated */
+      HOST_CANONICAL: HOST_INTERNAL, // legacy alias for HOST_INTERNAL
+      ROOT_URL_INTERNAL: joinBashExpressions(["https://", HOST_INTERNAL]),
       ...additionalEnvVars,
     };
   }
@@ -124,20 +147,20 @@ export const getEnvironmentVariables = async (
     ...publicEnvVarsRawSanitized,
   });
 
-  const envVars = await resolveReferences(
+  const envVars = (await resolveReferences(
     envVarsRaw,
     async (otherComponentName, alreadyVisited) => {
       const { envVars: otherEnvVars } = await getEnvironmentVariables(
-        config,
-        otherComponentName,
-        env,
-        commitInfo,
+        {
+          ...ctx,
+          componentName: otherComponentName,
+        },
         alreadyVisited
       );
       return otherEnvVars;
     },
     alreadyVisited
-  );
+  )) as typeof envVarsRaw;
 
   return {
     envVars,
@@ -168,7 +191,9 @@ export const getSecretVarName = (
   key: string
 ) => `CL_${sanitizeForEnVar(env)}_${sanitizeForEnVar(componentName)}_${key}`; // remove dash from component name
 
-const addIndexVar = (vars: Record<string, unknown>) => ({
+const addIndexVar = <V extends Record<string, unknown>>(
+  vars: V
+): V & { _ALL_ENV_VAR_KEYS: string } => ({
   ...vars,
   _ALL_ENV_VAR_KEYS: JSON.stringify(Object.keys(vars)),
 });

package/src/context/getReviewSlug.ts

@@ -0,0 +1,27 @@
+import {
+  BashExpression,
+  type StringOrBashExpression,
+} from "../bash/BashExpression";
+import type { BashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
+import { getBashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
+import type { PipelineType } from "../types";
+import type { EnvConfigWithComponent } from "../types/config";
+import { getEnvType } from "./getEnvType";
+const REVIEW_SLUG: BashExpressionPerPipelineType = {
+  default: "unknown-review-slug",
+  gitlab: new BashExpression(
+    `$([ -n "$CI_MERGE_REQUEST_IID" ] && echo "mr$CI_MERGE_REQUEST_IID" || { [ -n "$CI_COMMIT_REF_SLUG" ] && echo "$CI_COMMIT_REF_SLUG" || echo "unknown"; })`
+  ),
+};
+
+export const getReviewSlug = (
+  envConfig: EnvConfigWithComponent,
+  env: string,
+  pipelineType?: PipelineType
+): StringOrBashExpression | null => {
+  const envType = getEnvType(env, envConfig);
+  if (envType === "review") {
+    return getBashExpressionPerPipelineType(REVIEW_SLUG, pipelineType);
+  }
+  return null; // not a review app;
+};

package/src/context/index.ts

@@ -2,8 +2,9 @@ import { BUILD_TYPES } from "../build";
 import type { BuildConfig, BuildConfigType } from "../build/types";
 import { DEPLOY_TYPES } from "../deploy";
 import type { DeployConfig, DeployConfigType } from "../deploy/types";
-import type { Config } from "../types/config";
-import type { CommitInfo, Context, PackageManagerInfo } from "../types/context";
+import type { PipelineType } from "../types";
+import type { Config, PipelineTrigger } from "../types/config";
+import type { Context, PackageManagerInfo } from "../types/context";
 import type { PartialDeep } from "../types/utils";
 import { mergeWithMergingArrays } from "../utils";
 import { getEnvironment } from "./getEnvironment";
@@ -12,25 +13,25 @@ import { getEnvironmentContext } from "./getEnvironmentContext";
 export * from "./getEnvironment";
 export * from "./getEnvironmentVariables";
 
+export type CreateContextContext = {
+  config: Config;
+  componentName: string;
+  env: string;
+  pipelineType?: PipelineType;
+  trigger?: PipelineTrigger;
+  packageManagerInfo?: PackageManagerInfo;
+};
+
 export const createContext = async (
-  config: Config,
-  componentName: string,
-  env: string,
-  commitInfo?: CommitInfo,
-  packageManagerInfo?: PackageManagerInfo
+  ctx: CreateContextContext
 ): Promise<Context> => {
-  if (!/^[a-z0-9-]+$/.test(componentName)) {
+  if (!/^[a-z0-9-]+$/.test(ctx.componentName)) {
     throw new Error(
       "componentName may only contain lower case letters, numbers and -"
     );
   }
 
-  const envContext = getEnvironmentContext(
-    config,
-    env,
-    componentName,
-    commitInfo
-  );
+  const envContext = getEnvironmentContext(ctx);
 
   const componentConfigWithoutDefaults = envContext.envConfigRaw;
   const defaults: {
@@ -42,10 +43,9 @@ export const createContext = async (
           BUILD_TYPES[
             componentConfigWithoutDefaults.build.type as BuildConfigType
           ].defaults(envContext),
-        deploy:
-          DEPLOY_TYPES[
-            componentConfigWithoutDefaults.deploy.type as DeployConfigType
-          ].defaults(envContext),
+        deploy: DEPLOY_TYPES[
+          componentConfigWithoutDefaults.deploy.type as DeployConfigType
+        ].defaults(envContext as any),
       }
     : {
         build: {},
@@ -57,11 +57,12 @@ export const createContext = async (
   );
 
   return {
-    fullConfig: config,
+    fullConfig: ctx.config,
     componentConfig,
-    componentName,
-    environment: await getEnvironment(config, componentName, env, commitInfo),
-    commitInfo,
-    packageManagerInfo,
+    componentName: ctx.componentName,
+    environment: await getEnvironment(ctx),
+    packageManagerInfo: ctx.packageManagerInfo,
+    pipelineType: ctx.pipelineType,
+    trigger: ctx.trigger,
   };
 };

package/src/context/resolveReferences.ts

@@ -1,21 +1,32 @@
 import { merge } from "lodash";
-import replaceAsync from "string-replace-async";
 
+import type { BashExpression } from "../bash/BashExpression";
+import replaceAsync from "../bash/replaceAsync";
+import type { UnspecifiedEnvVars } from "..";
+
+// regex to resolve references in catladder variables
+// those expressions have the pattern ${componentName:variableName}
 const REGEX = /\$\{(([^:}]+):)?([^}]+)}/gm;
 
 export const resolveReferences = async (
-  vars: Record<string, string>,
+  vars: Record<string, string | BashExpression | undefined | null>,
   getOtherVariables?: (
     componentName: string,
     alreadyVisited: Record<string, Record<string, boolean>>
-  ) => Promise<Record<string, string | null>>,
+  ) => Promise<UnspecifiedEnvVars>,
   alreadyVisitedBase: Record<string, Record<string, boolean>> = {}
 ) => {
+  /**
+   *
+   * replace referenced variables with their values in a value string
+   */
   const replaceSingleValue = async (
-    value: string,
+    value: string | BashExpression,
     alreadyVisited: Record<string, Record<string, boolean>> = alreadyVisitedBase
-  ): Promise<string> => {
-    if (REGEX.test(value)) {
+  ): Promise<string | BashExpression> => {
+    if (REGEX.test(value.toString())) {
+      // we consider variables that got references in it  BashExpressions, because the replacement may be one
+
       return await replaceAsync(
         value,
         REGEX,
@@ -50,10 +61,19 @@ export const resolveReferences = async (
   return Object.fromEntries(
     await Promise.all(
       Object.entries(vars).map(async ([key, value]) => {
-        return [key, await replaceSingleValue(value)];
+        if (value === null || value === undefined) {
+          return [key, null];
+        }
+
+        return [
+          key,
+          value !== null && value !== undefined
+            ? await replaceSingleValue(value)
+            : null,
+        ];
       })
     )
-  ) as Record<string, string>;
+  ) as Record<string, BashExpression>;
 };
 
 export const translateLegacyFromComponents = (

package/src/context/transformJobOnlyVars.ts

@@ -1,4 +1,4 @@
-import type { EnvironmentEnvVars } from "..";
+import type { SecretEnvVar, UnspecifiedEnvVars } from "..";
 import type { EnvVars } from "../types/config";
 import {
   makeSecretEnvVarMapping,
@@ -17,7 +17,10 @@ export const transformJobOnlyVars = async (
   env: string,
   componentName: string,
   vars: EnvVars | null
-): Promise<EnvironmentEnvVars> => {
+): Promise<{
+  envVars: UnspecifiedEnvVars;
+  secretEnvVarKeys: SecretEnvVar[];
+}> => {
   if (!vars) {
     return {
       envVars: {},

package/src/context/utils/envVars.ts

@@ -1,6 +1,7 @@
 import { isObject } from "lodash";
 import type { SecretEnvVar } from "../getEnvironmentVariables";
 import { getSecretVarName } from "../getEnvironmentVariables";
+import { getBashVariable } from "../../bash/BashExpression";
 
 export const stringifyValues = (obj: Record<string, unknown>) =>
   Object.fromEntries(
@@ -20,7 +21,7 @@ export const makeSecretEnvVarMapping = (
   return Object.fromEntries(
     secretEnvVars.map(({ key }) => [
       key,
-      `$${getSecretVarName(env, componentName, key)}`,
+      getBashVariable(getSecretVarName(env, componentName, key)),
     ])
   );
 };