@catladder/pipeline 1.153.0 → 1.154.0

package/src/build/base/createAppBuildJob.ts

@@ -1,9 +1,13 @@
 import { merge } from "lodash";
-import { join } from "path";
-import type { ComponentContext } from "../..";
+import type {
+  BuildContextStandalone,
+  ComponentContext,
+  WorkspaceContext,
+} from "../..";
 import { getRunnerImage } from "../..";
 import type { CatladderJob } from "../../types/jobs";
 import { ensureArray } from "../../utils";
+import { createBuildJobArtifacts } from "../artifacts/createBuildJobArtifact";
 import { ensureNodeVersion } from "../node/yarn";
 import {
   APP_BUILD_JOB_NAME,
@@ -15,9 +19,9 @@ import {
   writeDotEnv,
 } from "./writeDotEnv";
 
-export type AppBuildJobDefinition = Partial<CatladderJob>;
+export type AppBuildJobDefinition = Partial<Omit<CatladderJob, "artifacts">>;
 export const createAppBuildJob = (
-  context: ComponentContext,
+  context: ComponentContext<BuildContextStandalone> | WorkspaceContext,
   { script, variables, runnerVariables, ...def }: AppBuildJobDefinition,
 ): CatladderJob => {
   return merge(
@@ -30,9 +34,13 @@ export const createAppBuildJob = (
       cache: [],
       variables: {
         ...(variables ?? {}),
-        ...context.environment.envVars,
-        ...context.environment.jobOnlyVars.build.envVars,
-        ...(context.build.config.extraVars ?? {}),
+        ...(context.type === "component"
+          ? {
+              ...context.environment.envVars,
+              ...context.environment.jobOnlyVars.build.envVars,
+              ...(context.build.config.extraVars ?? {}),
+            }
+          : {}),
       },
       runnerVariables: {
         ...RUNNER_BUILD_RESOURCE_VARIABLES,
@@ -41,25 +49,21 @@ export const createAppBuildJob = (
       },
 
       script: [
-        ...(componentContextNeedsBuildTimeDotEnv(context)
-          ? writeDotEnv(context)
-          : []),
-        ...writeBuildInfo(context),
+        ...(context.type === "component"
+          ? componentContextNeedsBuildTimeDotEnv(context)
+            ? writeDotEnv(context)
+            : []
+          : context.type === "workspace"
+            ? context.components
+                .filter((c) => componentContextNeedsBuildTimeDotEnv(c))
+                .flatMap((c) => writeDotEnv(c))
+            : []),
+        ...(context.type === "component" ? writeBuildInfo(context) : []),
         ...ensureNodeVersion(context), // in pure node repos, we might want to have the nvmrc file in top-level
         `cd ${context.build.dir}`,
         ...(ensureArray(script) ?? []),
       ],
-      artifacts: {
-        paths: [join(context.build.dir, "__build_info.json")],
-        ...(componentContextNeedsBuildTimeDotEnv(context)
-          ? { exclude: [join(context.build.dir, ".env")] }
-          : {}),
-        reports: {
-          junit: context.build.config.artifactsReports?.junit?.map((p) =>
-            join(context.build.dir, p),
-          ),
-        },
-      },
+      artifacts: createBuildJobArtifacts(context),
     },
     def,
   );

package/src/build/base/index.ts

@@ -1,5 +1,10 @@
 import { sbomDeactivated } from "../../deploy/sbom";
-import type { ComponentContext } from "../../types/context";
+import type { WorkspaceContext } from "../../types/context";
+import {
+  componentContextHasWorkspaceBuild,
+  componentContextIsStandaloneBuild,
+  type ComponentContext,
+} from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
 import type { DockerBuildJobDefinition } from "../docker";
 import { createDockerBuildJobBase, requiresDockerBuild } from "../docker";
@@ -8,7 +13,7 @@ import { APP_BUILD_JOB_NAME } from "./constants";
 import type { AppBuildJobDefinition } from "./createAppBuildJob";
 import { createAppBuildJob } from "./createAppBuildJob";
 
-export const createBuildJobs = (
+export const createComponentBuildJobs = (
   context: ComponentContext,
   definitions: {
     appBuild?: AppBuildJobDefinition;
@@ -16,7 +21,7 @@ export const createBuildJobs = (
   },
 ): CatladderJob[] => {
   return [
-    ...(definitions.appBuild
+    ...(definitions.appBuild && componentContextIsStandaloneBuild(context)
       ? [createAppBuildJob(context, definitions.appBuild)]
       : []),
     ...(requiresDockerBuild(context)
@@ -25,7 +30,18 @@ export const createBuildJobs = (
             ...definitions.dockerBuild,
             needs: [
               ...(definitions.dockerBuild.needs ?? []),
-              ...(definitions.appBuild ? [APP_BUILD_JOB_NAME] : []),
+              ...(definitions.appBuild &&
+              componentContextIsStandaloneBuild(context)
+                ? [APP_BUILD_JOB_NAME]
+                : componentContextHasWorkspaceBuild(context)
+                  ? [
+                      {
+                        job: APP_BUILD_JOB_NAME,
+                        artifacts: true,
+                        workspaceName: context.build.workspaceName,
+                      },
+                    ]
+                  : []),
             ],
           }),
         ]
@@ -33,3 +49,14 @@ export const createBuildJobs = (
     ...(sbomDeactivated(context) ? [] : [createSbomBuildJob(context)]),
   ];
 };
+
+export const createWorkspaceBuildJobs = (
+  context: WorkspaceContext,
+  definitions: {
+    appBuild?: AppBuildJobDefinition;
+  },
+): CatladderJob[] => {
+  return definitions.appBuild
+    ? [createAppBuildJob(context, definitions.appBuild)]
+    : [];
+};

package/src/build/base/writeDotEnv.ts

@@ -1,5 +1,6 @@
 import { isNil } from "lodash";
 import type { ComponentContext } from "../../types";
+import { collapseableSection } from "../../utils/gitlab";
 
 /**
  * writes a .env file in the components folder
@@ -18,11 +19,14 @@ export const writeDotEnv = (context: ComponentContext) => {
     )
     .join("\n");
 
-  return [
+  return collapseableSection(
+    "write-dotenv-" + context.name,
+    "write dot env",
+  )([
     `cat <<EOF > ${context.build.dir}/.env
 ${keyValueString}
 EOF`,
-  ];
+  ]);
 };
 
 export const componentContextNeedsBuildTimeDotEnv = (

package/src/build/custom/__tests__/testJob.test.ts

@@ -1,9 +1,9 @@
-import type { ComponentContext } from "../../..";
+import type { BuildContextStandalone, ComponentContext } from "../../..";
 import { createCustomTestJobs } from "../testJob";
 
 describe("createCustomTestJobs", () => {
-  const baseContext: ComponentContext = {
-    componentName: "testComponent",
+  const baseContext: ComponentContext<BuildContextStandalone> = {
+    name: "testComponent",
     environment: {
       jobOnlyVars: {
         build: {},
@@ -12,7 +12,7 @@ describe("createCustomTestJobs", () => {
     build: {
       config: { type: "custom", docker: { type: "custom" }, jobImage: "" },
     },
-  } as ComponentContext;
+  } as ComponentContext<BuildContextStandalone>;
 
   it("throws error when not build type custom", () => {
     baseContext.build.config.type = "node";

package/src/build/custom/buildJob.ts

@@ -1,11 +1,10 @@
-import { join } from "path";
 import type { ComponentContext } from "../../types/context";
 import { ensureArray } from "../../utils";
 import { getDockerBuildScriptWithBuiltInDockerFile } from "../docker";
 import { isOfBuildType } from "../types";
 
 import type { CatladderJob } from "../../types/jobs";
-import { createBuildJobs } from "../base";
+import { createComponentBuildJobs } from "../base";
 
 const RUNNER_BUILD_VARIABLES = {
   KUBERNETES_CPU_REQUEST: "0.5",
@@ -22,7 +21,7 @@ export const createCustomBuildJobs = (
     throw new Error("deploy config is not custom");
   }
 
-  return createBuildJobs(context, {
+  return createComponentBuildJobs(context, {
     appBuild:
       buildConfig.buildCommand !== null
         ? {
@@ -31,16 +30,6 @@ export const createCustomBuildJobs = (
             cache: buildConfig.jobCache,
             services: buildConfig.jobServices,
             script: [...(ensureArray(buildConfig.buildCommand) ?? [])],
-            artifacts: {
-              paths: [
-                join(context.build.dir, "__build_info.json"),
-                join(context.build.dir, "dist"),
-                ...(buildConfig.artifactsPaths?.map((path) =>
-                  join(context.build.dir, path),
-                ) ?? []),
-              ],
-              expire_in: "1 day",
-            },
           }
         : undefined,
     dockerBuild: {

package/src/build/docker.ts

@@ -33,11 +33,9 @@ export const getDockerImageVariables = (context: ComponentContext) => {
         {
           DOCKER_REGISTRY: "$CI_REGISTRY",
 
-          DOCKER_CACHE_IMAGE:
-            "$CI_REGISTRY_IMAGE/caches/" + context.componentName,
+          DOCKER_CACHE_IMAGE: "$CI_REGISTRY_IMAGE/caches/" + context.name,
           // ONLY USED IN KUBERNETES
-          DOCKER_IMAGE_NAME:
-            context.environment.shortName + "/" + context.componentName,
+          DOCKER_IMAGE_NAME: context.env + "/" + context.name,
           DOCKER_IMAGE: "$CI_REGISTRY_IMAGE/$DOCKER_IMAGE_NAME",
         }),
 
@@ -162,10 +160,10 @@ export const getDockerBuildScriptWithBuiltInDockerFile = (
   defaultType?: BuildConfigDocker["type"],
 ) => {
   const type =
-    "docker" in context.componentConfig.build &&
-    context.componentConfig.build.docker &&
-    "type" in context.componentConfig.build.docker
-      ? context.componentConfig.build.docker?.type
+    "docker" in context.build.config &&
+    context.build.config.docker &&
+    "type" in context.build.config.docker
+      ? context.build.config.docker?.type
       : defaultType;
   return getDockerBuildDefaultScript(
     context,

package/src/build/index.ts

@@ -1,20 +1,24 @@
-import type { ComponentContext } from "../types/context";
+import type { ComponentContext, WorkspaceContext } from "../types/context";
 import type { EnvironmentContext } from "../types/environmentContext";
 
 import type { CatladderJob } from "../types/jobs";
 import { createCustomJobs } from "./custom";
-import { createNodeJobs, createStorybookJobs, createMeteorJobs } from "./node";
+import { createMeteorJobs, createNodeJobs, createStorybookJobs } from "./node";
 import { createRailsJobs } from "./rails";
 
-import type { BuildConfig, BuildConfigType } from "./types";
-export * from "./types";
+import type {
+  BuildConfig,
+  BuildConfigStandaloneType,
+  WorkspaceBuildConfig,
+} from "./types";
 export * from "./node";
+export * from "./types";
 
 export type BuildTypes = {
-  [type in BuildConfigType]: {
+  [type in BuildConfigStandaloneType]: {
     jobs: (context: ComponentContext) => CatladderJob[];
     defaults: (
-      envContext: EnvironmentContext<BuildConfigType, any>,
+      envContext: EnvironmentContext,
     ) => Partial<Extract<BuildConfig, { type: type }>>;
   };
 };
@@ -25,29 +29,35 @@ export const BUILD_TYPES: BuildTypes = {
     defaults: () => ({
       buildCommand: "yarn build",
       startCommand: "yarn start",
+      artifactsPaths: ["dist", ".next"],
     }),
   },
   "node-static": {
     jobs: createNodeJobs,
     defaults: () => ({
       buildCommand: "yarn build",
+      artifactsPaths: ["dist"],
     }),
   },
   storybook: {
     jobs: createStorybookJobs,
     defaults: () => ({
       buildCommand: ["yarn build-storybook --quiet -o ./dist"],
+      artifactsPaths: ["dist"],
     }),
   },
   meteor: {
     jobs: createMeteorJobs,
     defaults: () => ({
       startCommand: "node main.js",
+      artifactsPaths: ["dist"],
     }),
   },
   custom: {
     jobs: createCustomJobs,
-    defaults: () => ({}),
+    defaults: () => ({
+      artifactsPaths: ["dist"],
+    }),
   },
   rails: {
     jobs: createRailsJobs,
@@ -60,3 +70,25 @@ export const BUILD_TYPES: BuildTypes = {
     }),
   },
 };
+
+export type WorkspaceBuildTypes = {
+  [type in WorkspaceBuildConfig["type"]]: {
+    jobs: (context: WorkspaceContext) => CatladderJob[];
+    defaults: () => Partial<Extract<WorkspaceBuildConfig, { type: type }>>;
+  };
+};
+
+export const WORKSPACE_BUILD_TYPES: WorkspaceBuildTypes = {
+  node: {
+    jobs: createNodeJobs,
+    defaults: () => ({
+      buildCommand: "yarn build",
+      lint: {
+        command: "yarn lint",
+      },
+      test: {
+        command: "yarn test",
+      },
+    }),
+  },
+};

package/src/build/node/buildJob.ts

@@ -1,79 +1,91 @@
-import { join } from "path";
 import { getRunnerImage } from "../../runner";
-import type { ComponentContext } from "../../types/context";
+import type {
+  BuildContextStandalone,
+  WorkspaceContext,
+} from "../../types/context";
+import {
+  componentContextIsStandaloneBuild,
+  type ComponentContext,
+} from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
 import { ensureArray } from "../../utils";
-import { createBuildJobs } from "../base";
+import { createComponentBuildJobs, createWorkspaceBuildJobs } from "../base";
+import type { AppBuildJobDefinition } from "../base/createAppBuildJob";
+import type { DockerBuildJobDefinition } from "../docker";
 import { getDockerBuildScriptWithBuiltInDockerFile } from "../docker";
+import type { BuildConfigDocker } from "../types";
 import { isOfBuildType } from "../types";
 import { getNextCache, getNodeCache, getYarnCache } from "./cache";
 import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
 import { getDockerAppCopyAndBuildScript, getYarnInstall } from "./yarn";
 
 export const createNodeBuildJobs = (
-  context: ComponentContext,
+  context: ComponentContext | WorkspaceContext,
 ): CatladderJob[] => {
+  if (context.type === "workspace") {
+    return createWorkspaceBuildJobs(context, {
+      appBuild: createNodeBuildJobDefinition(context),
+    });
+  }
+  return createComponentBuildJobs(context, {
+    appBuild: componentContextIsStandaloneBuild(context)
+      ? createNodeBuildJobDefinition(context)
+      : undefined,
+
+    dockerBuild: createNodeDockerJobDefinition(context),
+  });
+};
+
+export const createNodeBuildJobDefinition = (
+  context: ComponentContext<BuildContextStandalone> | WorkspaceContext,
+): AppBuildJobDefinition | undefined => {
   const buildConfig = context.build.config;
 
   if (!isOfBuildType(buildConfig, "node", "node-static", "storybook")) {
     throw new Error("deploy config is not node, node-static or storybook");
   }
 
+  if (buildConfig.buildCommand === null) return undefined;
+
   const defaultImage = getRunnerImage("jobs-default");
   const yarnInstall = getYarnInstall(context);
 
-  return createBuildJobs(context, {
-    appBuild:
-      buildConfig.buildCommand !== null
-        ? {
-            image: buildConfig.jobImage ?? defaultImage,
-            runnerVariables: NODE_RUNNER_BUILD_VARIABLES,
-            cache: [
-              ...(ensureArray(buildConfig.jobCache) ?? []),
-              ...getNodeCache(context),
-              ...getNextCache(context),
-            ],
-            script: [
-              ...yarnInstall,
-              ...(ensureArray(buildConfig.buildCommand) ?? []),
-            ],
-            artifacts: {
-              paths: [
-                context.build.dir,
-                // also copy workspace dependencies in monorepo if packages are shared and they create build artifacts
-                ...(context.packageManagerInfo.currentWorkspaceDependencies ??
-                  []),
-              ].flatMap((dir) => [
-                join(dir, "__build_info.json"),
-                join(dir, "dist"),
-                join(dir, ".next"),
-                ...(buildConfig.artifactsPaths?.map((path) =>
-                  join(dir, path),
-                ) ?? []),
-              ]),
-              expire_in: "1 day",
-              when: "always",
-            },
-            jobTags: buildConfig.jobTags,
-          }
-        : undefined,
+  return {
+    image: buildConfig.jobImage ?? defaultImage,
+    runnerVariables: NODE_RUNNER_BUILD_VARIABLES,
+    cache: [
+      ...(ensureArray(buildConfig.jobCache) ?? []),
+      ...getNodeCache(context),
+      ...getNextCache(context),
+    ],
+    script: [...yarnInstall, ...(ensureArray(buildConfig.buildCommand) ?? [])],
+    jobTags: buildConfig.jobTags,
+  };
+};
 
-    dockerBuild: {
-      script: getDockerBuildScriptWithBuiltInDockerFile(
-        context,
-        buildConfig.type === "node-static" || buildConfig.type === "storybook"
-          ? "nginx"
-          : "node",
-      ),
-      cache: [...getYarnCache(context, "pull")],
-      variables: {
-        // only required for non static
-        DOCKER_COPY_AND_INSTALL_APP: getDockerAppCopyAndBuildScript(context),
-        DOCKER_COPY_WORKSPACE_FILES:
-          context.packageManagerInfo.pathsToCopyInDocker
-            .map((dir) => `COPY --chown=node:node ${dir} /app/${dir}`)
-            ?.join("\n"),
-      },
+export const createNodeDockerJobDefinition = (
+  context: ComponentContext,
+): DockerBuildJobDefinition => {
+  // get the default docker built-in type based on the build type
+  const dockerDefaultBuiltIn: BuildConfigDocker["type"] =
+    context.build.buildType === "node-static" ||
+    context.build.buildType === "storybook"
+      ? "nginx"
+      : "node";
+
+  return {
+    script: getDockerBuildScriptWithBuiltInDockerFile(
+      context,
+      dockerDefaultBuiltIn,
+    ),
+    cache: [...getYarnCache(context, "pull")],
+    variables: {
+      // only required for non static
+      DOCKER_COPY_AND_INSTALL_APP: getDockerAppCopyAndBuildScript(context),
+      DOCKER_COPY_WORKSPACE_FILES:
+        context.packageManagerInfo?.pathsToCopyInDocker
+          .map((dir) => `COPY --chown=node:node ${dir} /app/${dir}`)
+          ?.join("\n"),
     },
-  });
+  };
 };

package/src/build/node/cache.ts

@@ -1,7 +1,7 @@
 import { uniq } from "lodash";
 import { join } from "path";
 import slugify from "slugify";
-import type { ComponentContext, Context } from "../../types/context";
+import type { Context } from "../../types/context";
 import type { GitlabJobCache } from "../../types/gitlab-types";
 
 export const getYarnCache = (
@@ -9,6 +9,7 @@ export const getYarnCache = (
   policy = "pull-push",
 ): GitlabJobCache[] => {
   const componentIsInWorkspace =
+    context.type === "component" &&
     context.packageManagerInfo.componentIsInWorkspace;
   return [
     componentIsInWorkspace
@@ -30,6 +31,7 @@ export const getNodeModulesCache = (
   policy = "pull-push",
 ): GitlabJobCache[] => {
   const componentIsInWorkspace =
+    context.type === "component" &&
     context.packageManagerInfo.componentIsInWorkspace;
 
   // We intentionally do not use the contents of yarn.lock as a cache key, as yarn install should always guarantee that the files are updated, but it can still use part of the cache if not all packages are up-to-date.
@@ -64,10 +66,17 @@ export const getNodeCache = (
   ];
 };
 
-export const getNextCache = (context: ComponentContext): GitlabJobCache[] => [
-  {
-    key: context.componentName + "-next-cache",
-    policy: "pull-push",
-    paths: [context.build.dir + "/.next/cache/"],
-  },
-];
+export const getNextCache = (context: Context): GitlabJobCache[] => {
+  const key = context.name + "-next-cache";
+  const paths = context.build
+    .getComponentDirs("direct")
+    .map((c) => join(c, ".next/cache"));
+
+  return [
+    {
+      key,
+      policy: "pull-push",
+      paths,
+    },
+  ];
+};

package/src/build/node/index.ts

@@ -1,10 +1,12 @@
-import type { ComponentContext } from "../../types/context";
+import type { ComponentContext, WorkspaceContext } from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
 import { createNodeBuildJobs } from "./buildJob";
 import { createMeteorBuildJobs } from "./meteor";
 import { createNodeTestJobs } from "./testJob";
 
-export const createNodeJobs = (context: ComponentContext): CatladderJob[] => {
+export const createNodeJobs = (
+  context: ComponentContext | WorkspaceContext,
+): CatladderJob[] => {
   return [...createNodeTestJobs(context), ...createNodeBuildJobs(context)];
 };
 

package/src/build/node/meteor.ts

@@ -5,7 +5,7 @@ import type { ComponentContext } from "../../types/context";
 
 import type { CatladderJob } from "../../types/jobs";
 
-import { createBuildJobs } from "../base";
+import { createComponentBuildJobs } from "../base";
 import { getDockerBuildScriptWithBuiltInDockerFile } from "../docker";
 import { isOfBuildType } from "../types";
 import { getNodeCache } from "./cache";
@@ -13,7 +13,7 @@ import { getYarnInstall } from "./yarn";
 
 const getMeteorCache = (context: ComponentContext): GitlabJobCache[] => [
   {
-    key: context.componentName + "meteor-build-cache",
+    key: context.name + "meteor-build-cache",
     policy: "pull-push",
     paths: [
       join(context.build.dir, ".meteor/local/resolver-result-cache.json"),
@@ -34,7 +34,7 @@ export const createMeteorBuildJobs = (
 
   const yarnInstall = getYarnInstall(context);
 
-  return createBuildJobs(context, {
+  return createComponentBuildJobs(context, {
     appBuild:
       buildConfig.buildCommand !== null
         ? {
@@ -54,12 +54,6 @@ export const createMeteorBuildJobs = (
 
               "cp ./__build_info.json ./dist/bundle/programs/server",
             ],
-            artifacts: {
-              paths: [
-                context.build.dir + "/__build_info.json",
-                context.build.dir + "/dist",
-              ],
-            },
           }
         : undefined,
     dockerBuild: {

package/src/build/node/testJob.ts

@@ -1,35 +1,49 @@
 import { getRunnerImage } from "../../runner";
-import type { ComponentContext } from "../../types/context";
+import type { WorkspaceContext } from "../../types/context";
+import {
+  componentContextIsStandaloneBuild,
+  type ComponentContext,
+} from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
 import { ensureArray, notNil } from "../../utils";
+import { createArtifactsConfig } from "../base/createArtifactsConfig";
 import { getNodeCache } from "./cache";
 import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
 import { ensureNodeVersion, getYarnInstall } from "./yarn";
-import { createArtifactsConfig } from "../base/createArtifactsConfig";
 
 export const createNodeTestJobs = (
-  context: ComponentContext,
+  context: ComponentContext | WorkspaceContext,
 ): CatladderJob[] => {
   // don't run tests after release
   // TODO: this will be replaced by using rules
   if (context.trigger === "taggedRelease") {
     return [];
   }
-
-  const buildConfig = context.build.config;
+  // if its not a standalone build, we don't need to run tests
+  if (
+    context.type === "component" &&
+    !componentContextIsStandaloneBuild(context)
+  ) {
+    return [];
+  }
 
   const defaultImage = getRunnerImage("jobs-default");
   const base: Omit<CatladderJob, "script" | "name"> = {
     variables: {
       APP_PATH: context.build.dir,
-      ...context.environment.jobOnlyVars.build.envVars,
-      ...(buildConfig.extraVars ?? {}),
+      ...(context.type === "component"
+        ? {
+            ...context.environment.jobOnlyVars.build.envVars,
+            ...(context.build.config.extraVars ?? {}),
+          }
+        : {}),
     },
     runnerVariables: NODE_RUNNER_BUILD_VARIABLES,
     stage: "test",
     needs: [],
     envMode: "none",
   };
+  const buildConfig = context.build.config;
   const yarnInstall = getYarnInstall(context);
   const auditJob: CatladderJob | null =
     buildConfig.audit !== false