@catladder/pipeline 0.0.0-more-collapse-deeb54d7 → 0.0.0-multiple-gitlab-includes-3df5566d

package/src/build/base/constants.ts

@@ -1,7 +1,7 @@
 export const APP_BUILD_JOB_NAME = "🔨 app";
 export const RUNNER_BUILD_RESOURCE_VARIABLES = {
-  KUBERNETES_CPU_REQUEST: "0.5",
-  KUBERNETES_CPU_LIMIT: "2",
+  KUBERNETES_CPU_REQUEST: "0.45",
+
   KUBERNETES_MEMORY_REQUEST: "1Gi",
   KUBERNETES_MEMORY_LIMIT: "4Gi",
 };

package/src/build/base/createAppBuildJob.ts

@@ -1,21 +1,29 @@
 import { merge } from "lodash";
-import { join } from "path";
-import type { Context } from "../..";
+import type {
+  BuildContextStandalone,
+  ComponentContext,
+  WorkspaceContext,
+} from "../..";
 import { getRunnerImage } from "../..";
+import type { AppBuildJobDefinition } from "../../types/jobDefinition";
 import type { CatladderJob } from "../../types/jobs";
 import { ensureArray } from "../../utils";
+import { createBuildJobArtifacts } from "../artifacts/createBuildJobArtifact";
+import { createJobCacheFromCacheConfigs } from "../cache/createJobCache";
 import { ensureNodeVersion } from "../node/yarn";
 import {
   APP_BUILD_JOB_NAME,
   RUNNER_BUILD_RESOURCE_VARIABLES,
 } from "./constants";
 import { writeBuildInfo } from "./writeBuildInfo";
-import { writeDotEnv } from "./writeDotEnv";
+import {
+  componentContextNeedsBuildTimeDotEnv,
+  writeDotEnv,
+} from "./writeDotEnv";
 
-export type AppBuildJobDefinition = Partial<CatladderJob>;
 export const createAppBuildJob = (
-  context: Context,
-  { script, variables, ...def }: AppBuildJobDefinition
+  context: ComponentContext<BuildContextStandalone> | WorkspaceContext,
+  { script, variables, runnerVariables, cache, ...def }: AppBuildJobDefinition,
 ): CatladderJob => {
   return merge(
     {
@@ -24,31 +32,39 @@ export const createAppBuildJob = (
       stage: "build",
       image: getRunnerImage("jobs-default"),
       needs: [],
-      cache: [],
+      cache: cache ? createJobCacheFromCacheConfigs(context, cache) : undefined,
       variables: {
-        ...RUNNER_BUILD_RESOURCE_VARIABLES,
         ...(variables ?? {}),
-        ...context.environment.envVars,
-        ...context.environment.jobOnlyVars.build.envVars,
-        ...(context.componentConfig.build.extraVars ?? {}),
+        ...(context.type === "component"
+          ? {
+              ...context.environment.envVars,
+              ...context.environment.jobOnlyVars.build.envVars,
+            }
+          : {}),
+      },
+      runnerVariables: {
+        ...RUNNER_BUILD_RESOURCE_VARIABLES,
+        ...(runnerVariables ?? {}),
+        ...(context.build.config.runnerVariables ?? {}),
       },
 
       script: [
-        ...(context.componentConfig.dotEnv ? writeDotEnv(context) : []),
-        ...writeBuildInfo(context),
+        ...(context.type === "component"
+          ? componentContextNeedsBuildTimeDotEnv(context)
+            ? writeDotEnv(context)
+            : []
+          : context.type === "workspace"
+            ? context.components
+                .filter((c) => componentContextNeedsBuildTimeDotEnv(c))
+                .flatMap((c) => writeDotEnv(c))
+            : []),
+        ...(context.type === "component" ? writeBuildInfo(context) : []),
         ...ensureNodeVersion(context), // in pure node repos, we might want to have the nvmrc file in top-level
-        `cd ${context.componentConfig.dir}`,
-        ...(ensureArray(script) ?? []),
+        `cd ${context.build.dir}`,
+        ...ensureArray(script),
       ],
-      artifacts: {
-        paths: [join(context.componentConfig.dir, "__build_info.json")],
-        reports: {
-          junit: context.componentConfig.build.artifactsReports?.junit?.map(
-            (p) => join(context.componentConfig.dir, p)
-          ),
-        },
-      },
+      artifacts: createBuildJobArtifacts(context),
     },
-    def
+    def,
   );
 };

package/src/build/base/createArtifactsConfig.ts

@@ -6,7 +6,7 @@ import type { OptionalArtifacts } from "../custom/testJob";
 export const createArtifactsConfig = (
   rootPath: string,
   artifactsReports?: BuildConfigArtifactsReports,
-  artifacts?: Artifacts
+  artifacts?: Artifacts,
 ): OptionalArtifacts =>
   artifactsReports || artifacts
     ? {
@@ -26,7 +26,7 @@ export const createArtifactsConfig = (
               .filter(Array.isArray)
               .reduce(
                 (acc, curr) => [...acc, ...curr.map((p) => join(rootPath, p))],
-                []
+                [],
               ),
           },
         },

package/src/build/base/createBuildJobDefinition.ts

@@ -0,0 +1,37 @@
+import type { BuildConfigStandalone, WorkspaceBuildConfig } from "..";
+import { getRunnerImage } from "../../runner";
+import type { AppBuildJobDefinition, Context } from "../../types";
+import { ensureArray } from "../../utils";
+import { getAllCacheConfigsFromConfig } from "../cache/getAllCacheConfigsFromConfig";
+import { NODE_RUNNER_BUILD_VARIABLES } from "../node/constants";
+
+export const createBuildJobDefinition = (
+  context: Context,
+  buildConfig: BuildConfigStandalone | WorkspaceBuildConfig,
+  customize: Pick<AppBuildJobDefinition, "cache"> & {
+    prescript?: string[];
+    postscript?: string[];
+  } = {},
+): AppBuildJobDefinition | undefined => {
+  if (buildConfig.buildCommand === null) return undefined;
+
+  const defaultImage = getRunnerImage("jobs-default");
+
+  return {
+    image: buildConfig.jobImage ?? defaultImage,
+    runnerVariables: NODE_RUNNER_BUILD_VARIABLES,
+    cache: [
+      ...(customize.cache ?? []),
+      ...getAllCacheConfigsFromConfig(context, buildConfig),
+    ],
+    services:
+      "jobServices" in buildConfig ? buildConfig.jobServices : undefined,
+    script: [
+      ...(customize.prescript ?? []),
+      ...ensureArray(buildConfig.buildCommand),
+      ...(customize.postscript ?? []),
+    ],
+
+    jobTags: buildConfig.jobTags,
+  };
+};

package/src/build/base/index.ts

@@ -1,22 +1,29 @@
 import { sbomDeactivated } from "../../deploy/sbom";
-import type { Context } from "../../types/context";
+import type { WorkspaceContext } from "../../types/context";
+import {
+  componentContextHasWorkspaceBuild,
+  componentContextIsStandaloneBuild,
+  type ComponentContext,
+} from "../../types/context";
+import type {
+  AppBuildJobDefinition,
+  DockerBuildJobDefinition,
+} from "../../types/jobDefinition";
 import type { CatladderJob } from "../../types/jobs";
-import type { DockerBuildJobDefinition } from "../docker";
 import { createDockerBuildJobBase, requiresDockerBuild } from "../docker";
 import { createSbomBuildJob } from "../sbom";
 import { APP_BUILD_JOB_NAME } from "./constants";
-import type { AppBuildJobDefinition } from "./createAppBuildJob";
 import { createAppBuildJob } from "./createAppBuildJob";
 
-export const createBuildJobs = (
-  context: Context,
+export const createComponentBuildJobs = (
+  context: ComponentContext,
   definitions: {
     appBuild?: AppBuildJobDefinition;
     dockerBuild: DockerBuildJobDefinition;
-  }
+  },
 ): CatladderJob[] => {
   return [
-    ...(definitions.appBuild
+    ...(definitions.appBuild && componentContextIsStandaloneBuild(context)
       ? [createAppBuildJob(context, definitions.appBuild)]
       : []),
     ...(requiresDockerBuild(context)
@@ -25,7 +32,18 @@ export const createBuildJobs = (
             ...definitions.dockerBuild,
             needs: [
               ...(definitions.dockerBuild.needs ?? []),
-              ...(definitions.appBuild ? [APP_BUILD_JOB_NAME] : []),
+              ...(definitions.appBuild &&
+              componentContextIsStandaloneBuild(context)
+                ? [APP_BUILD_JOB_NAME]
+                : componentContextHasWorkspaceBuild(context)
+                  ? [
+                      {
+                        job: APP_BUILD_JOB_NAME,
+                        artifacts: true,
+                        workspaceName: context.build.workspaceName,
+                      },
+                    ]
+                  : []),
             ],
           }),
         ]
@@ -33,3 +51,14 @@ export const createBuildJobs = (
     ...(sbomDeactivated(context) ? [] : [createSbomBuildJob(context)]),
   ];
 };
+
+export const createWorkspaceBuildJobs = (
+  context: WorkspaceContext,
+  definitions: {
+    appBuild?: AppBuildJobDefinition;
+  },
+): CatladderJob[] => {
+  return definitions.appBuild
+    ? [createAppBuildJob(context, definitions.appBuild)]
+    : [];
+};

package/src/build/base/writeBuildInfo.ts

@@ -1,7 +1,7 @@
-import type { Context } from "../../types";
+import type { ComponentContext } from "../../types";
 
-export const writeBuildInfo = (context: Context) => {
+export const writeBuildInfo = (context: ComponentContext) => {
   return [
-    `echo '{"id":"${context.commitInfo?.buildId}","time":"${context.commitInfo?.buildTime}"}' > ${context.componentConfig.dir}/__build_info.json`,
+    `echo '{"id":"${context.environment.envVars.BUILD_INFO_BUILD_ID}","time":"${context.environment.envVars.BUILD_INFO_BUILD_TIME}"}' > ${context.build.dir}/__build_info.json`,
   ];
 };

package/src/build/base/writeDotEnv.ts

@@ -1,21 +1,40 @@
-import type { Context } from "../../types";
+import { isNil } from "lodash";
+import { escapeForDotEnv } from "../../bash/bashEscape";
+import { ALL_BUILD_VARIABLES } from "../../context/getBuildInfoVariables";
+import type { ComponentContext } from "../../types";
+import { collapseableSection } from "../../utils/gitlab";
 
 /**
  * writes a .env file in the components folder
  * @param context
  * @returns
  */
-export const writeDotEnv = (context: Context) => {
+export const writeDotEnv = (context: ComponentContext) => {
   const envVars = context.environment.envVars;
 
   // make key=value and sanitize multiline
   const keyValueString = Object.entries(envVars)
-    .map(([key, value]) => `${key}=${value.replaceAll("\n", "\\n")}`)
+    // filter out null and undefined values
+    .filter(([, value]) => !isNil(value))
+    // filter out build variables, since they may interfer with caching like turbo
+    // build variables are rarely used anyway and we may treat them differently in the future
+    .filter(([key]) => !ALL_BUILD_VARIABLES.includes(key))
+
+    .map(([key, value]) => `${key}=${escapeForDotEnv(value)}`)
     .join("\n");
 
-  return [
-    `cat <<EOF > ${context.componentConfig.dir}/.env
+  return collapseableSection(
+    "write-dotenv-" + context.name,
+    "write dot env for " + context.name,
+  )([
+    `cat <<EOF > ${context.build.dir}/.env
 ${keyValueString}
 EOF`,
-  ];
+  ]);
+};
+
+export const componentContextNeedsBuildTimeDotEnv = (
+  context: ComponentContext,
+) => {
+  return (context.componentConfig.dotEnv ?? true) === true; // don't build when set to `local`
 };

package/src/build/cache/createJobCache.ts

@@ -0,0 +1,83 @@
+import { join } from "path";
+import type { CacheConfig, CacheConfigAdvanced, CacheConfigSimple } from "..";
+import type { Context } from "../../types";
+import type { CatladderJobCache } from "../../types/jobs";
+import type { WithCacheConfig } from "../types";
+import { getAllCacheConfigsFromConfig } from "./getAllCacheConfigsFromConfig";
+import slugify from "slugify";
+
+export const createJobCacheFromCacheConfigs = (
+  context: Context,
+  caches: CacheConfig[],
+): CatladderJobCache[] | undefined => {
+  if (caches.length === 0) return undefined;
+
+  const simpleCacheDefs = caches.filter(
+    (c): c is CacheConfigSimple => !("key" in c),
+  );
+  const advancedCacheDefs = caches.filter(
+    (c): c is CacheConfigAdvanced => "key" in c,
+  );
+  // simple caches are merged together into one. This is because e.g. gitlab has a limit of 4 caches, see https://gitlab.com/gitlab-org/gitlab/-/issues/421962
+  const simpleCaches =
+    simpleCacheDefs.length > 0
+      ? [
+          {
+            key: context.name + "-default",
+            policy: "pull-push",
+            paths: simpleCacheDefs.flatMap((c) =>
+              c.paths.map((p) => {
+                const baseDir =
+                  c.pathMode === "absolute"
+                    ? ""
+                    : (c.baseDir ?? context.build.dir);
+                return join(baseDir, p);
+              }),
+            ),
+          },
+        ]
+      : [];
+
+  const advancedCaches = advancedCacheDefs.map(
+    ({ key, paths, policy, scope, pathMode, buildDir, ...rest }) => {
+      const baseDir =
+        pathMode === "absolute" ? "" : (buildDir ?? context.build.dir);
+      const transformedKey =
+        scope === "global"
+          ? key
+          : typeof key === "string"
+            ? (scope === "buildDir" // really edge case...
+                ? slugify(baseDir)
+                : context.name) +
+              "-" +
+              key
+            : {
+                ...key,
+                files: key.files?.map((f) =>
+                  pathMode === "absolute" ? f : join(baseDir, f),
+                ),
+              };
+
+      return {
+        key: transformedKey,
+        policy: policy ?? "pull-push",
+        paths:
+          pathMode === "absolute" ? paths : paths?.map((p) => join(baseDir, p)),
+        ...rest,
+      };
+    },
+  );
+
+  return [...advancedCaches, ...simpleCaches];
+};
+
+/** shortcut, used in some build types */
+export const createJobCacheFromConfig = (
+  context: Context,
+  configWithCache: WithCacheConfig,
+) => {
+  return createJobCacheFromCacheConfigs(
+    context,
+    getAllCacheConfigsFromConfig(context, configWithCache),
+  );
+};

package/src/build/cache/getAllCacheConfigsFromConfig.ts

@@ -0,0 +1,28 @@
+import type { Context } from "../../types";
+import { ensureArray } from "../../utils/index";
+import type { CacheConfig, WithCacheConfig } from "../types";
+import { transformLegacyJobCache } from "./transformLegacyJobCache";
+
+export const getAllCacheConfigsFromConfig = (
+  context: Context,
+  configWithCache: WithCacheConfig,
+): CacheConfig[] => {
+  return [
+    ...("jobCache" in configWithCache
+      ? transformLegacyJobCache(configWithCache.jobCache)
+      : []),
+    ...ensureArray(configWithCache.cache),
+    ...(context.type === "workspace"
+      ? // also add cache configs of the components of that workspace
+        // FIXNME: this only works currently for the build config
+        // we would probably need to introduce a path to the right config property (build-->test or so)
+        context.components.flatMap<CacheConfig>(
+          (componentContext) =>
+            ensureArray(componentContext.build.config.cache).map((c) => ({
+              ...c,
+              baseDir: componentContext.build.dir,
+            })) ?? [],
+        )
+      : []),
+  ];
+};

package/src/build/cache/transformLegacyJobCache.ts

@@ -0,0 +1,14 @@
+import type { CacheConfigAdvanced } from "..";
+import type { CatladderJobCache } from "../../types/jobs";
+import { ensureArray } from "../../utils";
+
+export function transformLegacyJobCache(
+  jobCache: CatladderJobCache | CatladderJobCache[] | undefined,
+): CacheConfigAdvanced[] {
+  return ensureArray(jobCache).map((cache) => ({
+    pathMode: "absolute",
+    ...cache,
+    key: cache.key ?? "default",
+    paths: cache.paths ?? [],
+  }));
+}

package/src/build/custom/__tests__/testJob.test.ts

@@ -1,28 +1,25 @@
-import type { Context } from "../../..";
+import type { BuildContextStandalone, ComponentContext } from "../../..";
 import { createCustomTestJobs } from "../testJob";
 
 describe("createCustomTestJobs", () => {
-  const componentConfig: Context["componentConfig"] = {
-    dir: ".",
-    build: { type: "custom", docker: { type: "custom" }, jobImage: "" },
-    deploy: {},
-  } as Context["componentConfig"];
-  const baseContext: Context = {
-    componentName: "testComponent",
+  const baseContext: ComponentContext<BuildContextStandalone> = {
+    name: "testComponent",
     environment: {
       jobOnlyVars: {
         build: {},
       },
     },
-    componentConfig,
-  } as Context;
+    build: {
+      config: { type: "custom", docker: { type: "custom" }, jobImage: "" },
+    },
+  } as ComponentContext<BuildContextStandalone>;
 
   it("throws error when not build type custom", () => {
-    componentConfig.build.type = "node";
+    baseContext.build.config.type = "node";
     expect(() => createCustomTestJobs(baseContext)).toThrowError(
-      "deploy config is not custom"
+      "deploy config is not custom",
     );
-    componentConfig.build.type = "custom";
+    baseContext.build.config.type = "custom";
   });
 
   it("returns empty array if no audit, lint, and test definition", () => {

package/src/build/custom/buildJob.ts

@@ -1,53 +1,27 @@
-import { join } from "path";
-import type { Context } from "../../types/context";
-import { ensureArray } from "../../utils";
-import { getDockerBuildDefaultScript, requiresDockerBuild } from "../docker";
+import type { ComponentContext } from "../../types/context";
+import { getDockerBuildScriptWithBuiltInDockerFile } from "../docker";
 import { isOfBuildType } from "../types";
 
 import type { CatladderJob } from "../../types/jobs";
-import { createBuildJobs } from "../base";
+import { createComponentBuildJobs } from "../base";
+import { createBuildJobDefinition } from "../base/createBuildJobDefinition";
 
-const RUNNER_BUILD_VARIABLES = {
-  KUBERNETES_CPU_REQUEST: "0.5",
-  KUBERNETES_CPU_LIMIT: "2",
-  KUBERNETES_MEMORY_REQUEST: "2Gi",
-  KUBERNETES_MEMORY_LIMIT: "4Gi",
-};
-
-export const createCustomBuildJobs = (context: Context): CatladderJob[] => {
-  const buildConfig = context.componentConfig.build;
+export const createCustomBuildJobs = (
+  context: ComponentContext,
+): CatladderJob[] => {
+  const buildConfig = context.build.config;
 
   if (!isOfBuildType(buildConfig, "custom")) {
     throw new Error("deploy config is not custom");
   }
 
-  return createBuildJobs(context, {
+  return createComponentBuildJobs(context, {
     appBuild:
       buildConfig.buildCommand !== null
-        ? {
-            image: buildConfig.jobImage,
-            variables: {
-              ...RUNNER_BUILD_VARIABLES,
-            },
-            cache: buildConfig.jobCache,
-            services: buildConfig.jobServices,
-            script: [...(ensureArray(buildConfig.buildCommand) ?? [])],
-            artifacts: {
-              paths: [
-                join(context.componentConfig.dir, "__build_info.json"),
-                join(context.componentConfig.dir, "dist"),
-                ...(buildConfig.artifactsPaths?.map((path) =>
-                  join(context.componentConfig.dir, path)
-                ) ?? []),
-              ],
-              expire_in: "1 day",
-            },
-          }
+        ? createBuildJobDefinition(context, buildConfig)
         : undefined,
     dockerBuild: {
-      script: getDockerBuildDefaultScript(
-        buildConfig.docker?.type === "nginx" ? "ensureNginxDockerfile" : ""
-      ),
+      script: getDockerBuildScriptWithBuiltInDockerFile(context),
 
       variables: {},
     },

package/src/build/custom/index.ts

@@ -1,8 +1,8 @@
-import type { Context } from "../../types/context";
+import type { ComponentContext } from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
 import { createCustomBuildJobs } from "./buildJob";
 import { createCustomTestJobs } from "./testJob";
 
-export const createCustomJobs = (context: Context): CatladderJob[] => {
+export const createCustomJobs = (context: ComponentContext): CatladderJob[] => {
   return [...createCustomTestJobs(context), ...createCustomBuildJobs(context)];
 };

package/src/build/custom/testJob.ts

@@ -1,24 +1,27 @@
-import type { Context } from "../../types/context";
+import type { Artifacts } from "../../types";
+import type { ComponentContext } from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
 import { ensureArray, notNil } from "../../utils";
-import { isOfBuildType } from "../types";
-import type { Artifacts } from "../../types";
 import { createArtifactsConfig } from "../base/createArtifactsConfig";
+import { createJobCacheFromConfig } from "../cache/createJobCache";
+import { isOfBuildType } from "../types";
 
 const RUNNER_CUSTOM_TEST_VARIABLES = {
-  KUBERNETES_CPU_REQUEST: "0.5",
-  KUBERNETES_CPU_LIMIT: "2",
-  KUBERNETES_MEMORY_REQUEST: "2Gi",
+  KUBERNETES_CPU_REQUEST: "0.45",
+  KUBERNETES_MEMORY_REQUEST: "1Gi",
   KUBERNETES_MEMORY_LIMIT: "4Gi",
 };
 
-export const createCustomTestJobs = (context: Context): CatladderJob[] => {
+export const createCustomTestJobs = (
+  context: ComponentContext,
+): CatladderJob[] => {
   // don't run tests after release
-  if (context.commitInfo?.trigger === "taggedRelease") {
+  // TODO: this will be replaced by using rules
+  if (context.trigger === "taggedRelease") {
     return [];
   }
 
-  const buildConfig = context.componentConfig.build;
+  const buildConfig = context.build.config;
 
   if (!isOfBuildType(buildConfig, "custom")) {
     throw new Error("deploy config is not custom");
@@ -26,13 +29,12 @@ export const createCustomTestJobs = (context: Context): CatladderJob[] => {
 
   const base: Omit<CatladderJob, "script" | "name"> = {
     variables: {
-      APP_PATH: context.componentConfig.dir,
-      ...RUNNER_CUSTOM_TEST_VARIABLES,
+      APP_PATH: context.build.dir,
       ...context.environment.jobOnlyVars.build.envVars,
-      ...(buildConfig.extraVars ?? {}),
     },
+    runnerVariables: RUNNER_CUSTOM_TEST_VARIABLES,
     services: buildConfig.jobServices,
-    cache: buildConfig.jobCache,
+    cache: createJobCacheFromConfig(context, buildConfig),
     stage: "test",
     needs: [],
     envMode: "none",
@@ -43,12 +45,12 @@ export const createCustomTestJobs = (context: Context): CatladderJob[] => {
         ...base,
         image: buildConfig.audit?.jobImage ?? buildConfig.jobImage,
         cache: undefined,
-        script: [...(ensureArray(buildConfig.audit?.command) ?? [])],
+        script: [...ensureArray(buildConfig.audit?.command)],
         allow_failure: true,
         ...createArtifactsConfig(
-          context.componentConfig.dir,
+          context.build.dir,
           buildConfig.audit?.artifactsReports,
-          buildConfig.audit?.artifacts
+          buildConfig.audit?.artifacts,
         ),
       }
     : null;
@@ -59,11 +61,11 @@ export const createCustomTestJobs = (context: Context): CatladderJob[] => {
 
         ...base,
         image: buildConfig.lint?.jobImage ?? buildConfig.jobImage,
-        script: [...(ensureArray(buildConfig.lint?.command) ?? [])],
+        script: [...ensureArray(buildConfig.lint?.command)],
         ...createArtifactsConfig(
-          context.componentConfig.dir,
+          context.build.dir,
           buildConfig.lint?.artifactsReports,
-          buildConfig.lint?.artifacts
+          buildConfig.lint?.artifacts,
         ),
       }
     : null;
@@ -73,11 +75,11 @@ export const createCustomTestJobs = (context: Context): CatladderJob[] => {
 
         ...base,
         image: buildConfig.test?.jobImage ?? buildConfig.jobImage,
-        script: [...(ensureArray(buildConfig.test?.command) ?? [])],
+        script: [...ensureArray(buildConfig.test?.command)],
         ...createArtifactsConfig(
-          context.componentConfig.dir,
+          context.build.dir,
           buildConfig.test?.artifactsReports,
-          buildConfig.test?.artifacts
+          buildConfig.test?.artifacts,
         ),
       }
     : null;