@catladder/pipeline 0.0.0-node16-ddfae617 → 0.0.0-perf-cli-reduce-bundle-size-with-tsconfig-optimization-3556aaa4

package/src/build/artifacts/createBuildJobArtifact.ts

@@ -0,0 +1,61 @@
+import { join } from "path";
+import {
+  componentContextIsStandaloneBuild,
+  type ComponentContext,
+  type Context,
+} from "../../types";
+import type { CatladderJob } from "../../types/jobs";
+import { uniq } from "lodash";
+import { componentContextNeedsBuildTimeDotEnv } from "../base/writeDotEnv";
+
+export const createBuildJobArtifacts = (
+  context: Context,
+): CatladderJob["artifacts"] => {
+  const paths =
+    context.type === "workspace"
+      ? context.components.flatMap((c) => getArtifactsPathForComponent(c))
+      : getArtifactsPathForComponent(context, ["__build_info.json"]);
+  // esxclude .env files if generated
+  const exclude =
+    context.type === "workspace"
+      ? context.components.flatMap((c) => getDotEnvPathsForComponent(c))
+      : getDotEnvPathsForComponent(context);
+  return {
+    paths: uniq(paths).sort((a, b) => a.localeCompare(b)),
+    ...(exclude.length > 0 ? { exclude } : {}),
+    expire_in: "1 day",
+    when: "always",
+    reports:
+      // TODO: support for junit reports in other builds
+      context.type === "component" && componentContextIsStandaloneBuild(context)
+        ? {
+            junit: context.build.config.artifactsReports?.junit?.map((p) =>
+              join(context.build.dir, p),
+            ),
+          }
+        : {},
+  };
+};
+const getArtifactsPathForComponent = (
+  c: ComponentContext,
+  additionalPaths?: string[],
+): string[] => {
+  return [
+    ...(c.build.config.artifactsPaths ?? []),
+    ...(additionalPaths ?? []),
+  ]?.flatMap((artifact) =>
+    c.build
+      // in theory, we only need "direct",
+      // but in some cases project may have packages in the workspace that create build artifacts, which aren't components
+      // this highly depends on the build tool. To be safe, we get all
+      .getComponentDirs("all")
+      .flatMap((cDir) => join(cDir, artifact)),
+  );
+};
+
+const getDotEnvPathsForComponent = (c: ComponentContext): string[] => {
+  if (componentContextNeedsBuildTimeDotEnv(c)) {
+    return [join(c.build.dir, ".env")];
+  }
+  return [];
+};

package/src/build/base/__tests__/__snapshots__/createArtifactsConfig.test.ts.snap

@@ -0,0 +1,49 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`createArtifactsConfig merges junit files if both given 1`] = `
+{
+  "artifacts": {
+    "paths": [
+      "testpath",
+    ],
+    "reports": {
+      "junit": [
+        "partial-report.xml",
+        "full-report.xml",
+      ],
+      "terraform": "something",
+    },
+  },
+}
+`;
+
+exports[`createArtifactsConfig returns artifacts if no artifactsReports 1`] = `
+{
+  "artifacts": {
+    "paths": [
+      "testpath",
+    ],
+    "reports": {
+      "coverage_report": {
+        "coverage_format": "cobertura",
+        "path": "some",
+      },
+      "junit": [],
+      "terraform": "something",
+    },
+  },
+}
+`;
+
+exports[`createArtifactsConfig returns junit parts of artifactsReports 1`] = `
+{
+  "artifacts": {
+    "paths": [],
+    "reports": {
+      "junit": [
+        "report.xml",
+      ],
+    },
+  },
+}
+`;

package/src/build/base/__tests__/createArtifactsConfig.test.ts

@@ -0,0 +1,41 @@
+import { createArtifactsConfig } from "../createArtifactsConfig";
+
+describe("createArtifactsConfig", () => {
+  it("returns undefined if no artifacts or artifactsReports", () => {
+    expect(createArtifactsConfig(".")).toBeUndefined();
+  });
+
+  it("returns artifacts if no artifactsReports", () => {
+    expect(
+      createArtifactsConfig(".", undefined, {
+        paths: ["testpath"],
+        reports: {
+          coverage_report: { coverage_format: "cobertura", path: "some" },
+          terraform: "something",
+        },
+      }),
+    ).toMatchSnapshot();
+  });
+
+  it("returns junit parts of artifactsReports", () => {
+    expect(
+      createArtifactsConfig(".", { junit: ["report.xml"] }),
+    ).toMatchSnapshot();
+  });
+
+  it("merges junit files if both given", () => {
+    const result = createArtifactsConfig(
+      ".",
+      { junit: ["partial-report.xml"] },
+      {
+        paths: ["testpath"],
+        reports: {
+          junit: "full-report.xml",
+          terraform: "something",
+        },
+      },
+    );
+    expect(result).toMatchSnapshot();
+    expect(result?.artifacts?.reports?.junit).toHaveLength(2);
+  });
+});

package/src/build/base/constants.ts

@@ -1,7 +1,7 @@
 export const APP_BUILD_JOB_NAME = "🔨 app";
 export const RUNNER_BUILD_RESOURCE_VARIABLES = {
-  KUBERNETES_CPU_REQUEST: "0.5",
-  KUBERNETES_CPU_LIMIT: "2",
+  KUBERNETES_CPU_REQUEST: "0.45",
+
   KUBERNETES_MEMORY_REQUEST: "1Gi",
   KUBERNETES_MEMORY_LIMIT: "4Gi",
 };

package/src/build/base/createAppBuildJob.ts

@@ -0,0 +1,70 @@
+import { merge } from "lodash";
+import type {
+  BuildContextStandalone,
+  ComponentContext,
+  WorkspaceContext,
+} from "../..";
+import { getRunnerImage } from "../..";
+import type { CatladderJob } from "../../types/jobs";
+import { ensureArray } from "../../utils";
+import { createBuildJobArtifacts } from "../artifacts/createBuildJobArtifact";
+import { ensureNodeVersion } from "../node/yarn";
+import {
+  APP_BUILD_JOB_NAME,
+  RUNNER_BUILD_RESOURCE_VARIABLES,
+} from "./constants";
+import { writeBuildInfo } from "./writeBuildInfo";
+import {
+  componentContextNeedsBuildTimeDotEnv,
+  writeDotEnv,
+} from "./writeDotEnv";
+
+export type AppBuildJobDefinition = Partial<Omit<CatladderJob, "artifacts">>;
+export const createAppBuildJob = (
+  context: ComponentContext<BuildContextStandalone> | WorkspaceContext,
+  { script, variables, runnerVariables, ...def }: AppBuildJobDefinition,
+): CatladderJob => {
+  return merge(
+    {
+      name: APP_BUILD_JOB_NAME,
+      envMode: "jobPerEnv",
+      stage: "build",
+      image: getRunnerImage("jobs-default"),
+      needs: [],
+      cache: [],
+      variables: {
+        ...(variables ?? {}),
+        ...(context.type === "component"
+          ? {
+              ...context.environment.envVars,
+              ...context.environment.jobOnlyVars.build.envVars,
+              ...(context.build.config.extraVars ?? {}),
+            }
+          : {}),
+      },
+      runnerVariables: {
+        ...RUNNER_BUILD_RESOURCE_VARIABLES,
+        ...(runnerVariables ?? {}),
+        ...(context.build.config.runnerVariables ?? {}),
+      },
+
+      script: [
+        ...(context.type === "component"
+          ? componentContextNeedsBuildTimeDotEnv(context)
+            ? writeDotEnv(context)
+            : []
+          : context.type === "workspace"
+            ? context.components
+                .filter((c) => componentContextNeedsBuildTimeDotEnv(c))
+                .flatMap((c) => writeDotEnv(c))
+            : []),
+        ...(context.type === "component" ? writeBuildInfo(context) : []),
+        ...ensureNodeVersion(context), // in pure node repos, we might want to have the nvmrc file in top-level
+        `cd ${context.build.dir}`,
+        ...(ensureArray(script) ?? []),
+      ],
+      artifacts: createBuildJobArtifacts(context),
+    },
+    def,
+  );
+};

package/src/build/base/createArtifactsConfig.ts

@@ -0,0 +1,34 @@
+import { join } from "path";
+import type { BuildConfigArtifactsReports } from "../types";
+import type { Artifacts } from "../../types";
+import type { OptionalArtifacts } from "../custom/testJob";
+
+export const createArtifactsConfig = (
+  rootPath: string,
+  artifactsReports?: BuildConfigArtifactsReports,
+  artifacts?: Artifacts,
+): OptionalArtifacts =>
+  artifactsReports || artifacts
+    ? {
+        artifacts: {
+          paths: [],
+          ...artifacts,
+          reports: {
+            ...artifacts?.reports,
+            junit: [
+              artifactsReports?.junit,
+              Array.isArray(artifacts?.reports?.junit) &&
+                artifacts?.reports?.junit,
+              typeof artifacts?.reports?.junit === "string" && [
+                artifacts?.reports?.junit,
+              ],
+            ]
+              .filter(Array.isArray)
+              .reduce(
+                (acc, curr) => [...acc, ...curr.map((p) => join(rootPath, p))],
+                [],
+              ),
+          },
+        },
+      }
+    : undefined;

package/src/build/base/index.ts

@@ -0,0 +1,62 @@
+import { sbomDeactivated } from "../../deploy/sbom";
+import type { WorkspaceContext } from "../../types/context";
+import {
+  componentContextHasWorkspaceBuild,
+  componentContextIsStandaloneBuild,
+  type ComponentContext,
+} from "../../types/context";
+import type { CatladderJob } from "../../types/jobs";
+import type { DockerBuildJobDefinition } from "../docker";
+import { createDockerBuildJobBase, requiresDockerBuild } from "../docker";
+import { createSbomBuildJob } from "../sbom";
+import { APP_BUILD_JOB_NAME } from "./constants";
+import type { AppBuildJobDefinition } from "./createAppBuildJob";
+import { createAppBuildJob } from "./createAppBuildJob";
+
+export const createComponentBuildJobs = (
+  context: ComponentContext,
+  definitions: {
+    appBuild?: AppBuildJobDefinition;
+    dockerBuild: DockerBuildJobDefinition;
+  },
+): CatladderJob[] => {
+  return [
+    ...(definitions.appBuild && componentContextIsStandaloneBuild(context)
+      ? [createAppBuildJob(context, definitions.appBuild)]
+      : []),
+    ...(requiresDockerBuild(context)
+      ? [
+          createDockerBuildJobBase(context, {
+            ...definitions.dockerBuild,
+            needs: [
+              ...(definitions.dockerBuild.needs ?? []),
+              ...(definitions.appBuild &&
+              componentContextIsStandaloneBuild(context)
+                ? [APP_BUILD_JOB_NAME]
+                : componentContextHasWorkspaceBuild(context)
+                  ? [
+                      {
+                        job: APP_BUILD_JOB_NAME,
+                        artifacts: true,
+                        workspaceName: context.build.workspaceName,
+                      },
+                    ]
+                  : []),
+            ],
+          }),
+        ]
+      : []),
+    ...(sbomDeactivated(context) ? [] : [createSbomBuildJob(context)]),
+  ];
+};
+
+export const createWorkspaceBuildJobs = (
+  context: WorkspaceContext,
+  definitions: {
+    appBuild?: AppBuildJobDefinition;
+  },
+): CatladderJob[] => {
+  return definitions.appBuild
+    ? [createAppBuildJob(context, definitions.appBuild)]
+    : [];
+};

package/src/build/base/writeBuildInfo.ts

@@ -0,0 +1,7 @@
+import type { ComponentContext } from "../../types";
+
+export const writeBuildInfo = (context: ComponentContext) => {
+  return [
+    `echo '{"id":"${context.environment.envVars.BUILD_INFO_BUILD_ID}","time":"${context.environment.envVars.BUILD_INFO_BUILD_TIME}"}' > ${context.build.dir}/__build_info.json`,
+  ];
+};

package/src/build/base/writeDotEnv.ts

@@ -0,0 +1,40 @@
+import { isNil } from "lodash";
+import type { ComponentContext } from "../../types";
+import { collapseableSection } from "../../utils/gitlab";
+import { ALL_BUILD_VARIABLES } from "../../context/getBuildInfoVariables";
+
+/**
+ * writes a .env file in the components folder
+ * @param context
+ * @returns
+ */
+export const writeDotEnv = (context: ComponentContext) => {
+  const envVars = context.environment.envVars;
+
+  // make key=value and sanitize multiline
+  const keyValueString = Object.entries(envVars)
+    // filter out null and undefined values
+    .filter(([, value]) => !isNil(value))
+    // filter out build variables, since they may interfer with caching like turbo
+    // build variables are rarely used anyway and we may treat them differently in the future
+    .filter(([key]) => !ALL_BUILD_VARIABLES.includes(key))
+    .map(
+      ([key, value]) => `${key}=${value?.toString().replaceAll("\n", "\\n")}`,
+    )
+    .join("\n");
+
+  return collapseableSection(
+    "write-dotenv-" + context.name,
+    "write dot env",
+  )([
+    `cat <<EOF > ${context.build.dir}/.env
+${keyValueString}
+EOF`,
+  ]);
+};
+
+export const componentContextNeedsBuildTimeDotEnv = (
+  context: ComponentContext,
+) => {
+  return context.componentConfig.dotEnv === true; // don't build when set to `local`
+};

package/src/build/custom/__tests__/testJob.test.ts

@@ -0,0 +1,29 @@
+import type { BuildContextStandalone, ComponentContext } from "../../..";
+import { createCustomTestJobs } from "../testJob";
+
+describe("createCustomTestJobs", () => {
+  const baseContext: ComponentContext<BuildContextStandalone> = {
+    name: "testComponent",
+    environment: {
+      jobOnlyVars: {
+        build: {},
+      },
+    },
+    build: {
+      config: { type: "custom", docker: { type: "custom" }, jobImage: "" },
+    },
+  } as ComponentContext<BuildContextStandalone>;
+
+  it("throws error when not build type custom", () => {
+    baseContext.build.config.type = "node";
+    expect(() => createCustomTestJobs(baseContext)).toThrowError(
+      "deploy config is not custom",
+    );
+    baseContext.build.config.type = "custom";
+  });
+
+  it("returns empty array if no audit, lint, and test definition", () => {
+    const jobs = createCustomTestJobs(baseContext);
+    expect(jobs).toEqual([]);
+  });
+});

package/src/build/custom/buildJob.ts

@@ -0,0 +1,41 @@
+import type { ComponentContext } from "../../types/context";
+import { ensureArray } from "../../utils";
+import { getDockerBuildScriptWithBuiltInDockerFile } from "../docker";
+import { isOfBuildType } from "../types";
+
+import type { CatladderJob } from "../../types/jobs";
+import { createComponentBuildJobs } from "../base";
+
+const RUNNER_BUILD_VARIABLES = {
+  KUBERNETES_CPU_REQUEST: "0.45",
+  KUBERNETES_MEMORY_REQUEST: "1Gi",
+  KUBERNETES_MEMORY_LIMIT: "4Gi",
+};
+
+export const createCustomBuildJobs = (
+  context: ComponentContext,
+): CatladderJob[] => {
+  const buildConfig = context.build.config;
+
+  if (!isOfBuildType(buildConfig, "custom")) {
+    throw new Error("deploy config is not custom");
+  }
+
+  return createComponentBuildJobs(context, {
+    appBuild:
+      buildConfig.buildCommand !== null
+        ? {
+            image: buildConfig.jobImage,
+            runnerVariables: RUNNER_BUILD_VARIABLES,
+            cache: buildConfig.jobCache,
+            services: buildConfig.jobServices,
+            script: [...(ensureArray(buildConfig.buildCommand) ?? [])],
+          }
+        : undefined,
+    dockerBuild: {
+      script: getDockerBuildScriptWithBuiltInDockerFile(context),
+
+      variables: {},
+    },
+  });
+};

package/src/build/custom/index.ts

@@ -0,0 +1,8 @@
+import type { ComponentContext } from "../../types/context";
+import type { CatladderJob } from "../../types/jobs";
+import { createCustomBuildJobs } from "./buildJob";
+import { createCustomTestJobs } from "./testJob";
+
+export const createCustomJobs = (context: ComponentContext): CatladderJob[] => {
+  return [...createCustomTestJobs(context), ...createCustomBuildJobs(context)];
+};

package/src/build/custom/testJob.ts

@@ -0,0 +1,93 @@
+import type { ComponentContext } from "../../types/context";
+import type { CatladderJob } from "../../types/jobs";
+import { ensureArray, notNil } from "../../utils";
+import { isOfBuildType } from "../types";
+import type { Artifacts } from "../../types";
+import { createArtifactsConfig } from "../base/createArtifactsConfig";
+
+const RUNNER_CUSTOM_TEST_VARIABLES = {
+  KUBERNETES_CPU_REQUEST: "0.45",
+  KUBERNETES_MEMORY_REQUEST: "1Gi",
+  KUBERNETES_MEMORY_LIMIT: "4Gi",
+};
+
+export const createCustomTestJobs = (
+  context: ComponentContext,
+): CatladderJob[] => {
+  // don't run tests after release
+  // TODO: this will be replaced by using rules
+  if (context.trigger === "taggedRelease") {
+    return [];
+  }
+
+  const buildConfig = context.build.config;
+
+  if (!isOfBuildType(buildConfig, "custom")) {
+    throw new Error("deploy config is not custom");
+  }
+
+  const base: Omit<CatladderJob, "script" | "name"> = {
+    variables: {
+      APP_PATH: context.build.dir,
+      ...context.environment.jobOnlyVars.build.envVars,
+      ...(buildConfig.extraVars ?? {}),
+    },
+    runnerVariables: RUNNER_CUSTOM_TEST_VARIABLES,
+    services: buildConfig.jobServices,
+    cache: buildConfig.jobCache,
+    stage: "test",
+    needs: [],
+    envMode: "none",
+  };
+  const auditJob: CatladderJob | null = buildConfig.audit
+    ? {
+        name: "🛡 audit",
+        ...base,
+        image: buildConfig.audit?.jobImage ?? buildConfig.jobImage,
+        cache: undefined,
+        script: [...(ensureArray(buildConfig.audit?.command) ?? [])],
+        allow_failure: true,
+        ...createArtifactsConfig(
+          context.build.dir,
+          buildConfig.audit?.artifactsReports,
+          buildConfig.audit?.artifacts,
+        ),
+      }
+    : null;
+
+  const lintJob: CatladderJob | null = buildConfig.lint
+    ? {
+        name: "👮 lint",
+
+        ...base,
+        image: buildConfig.lint?.jobImage ?? buildConfig.jobImage,
+        script: [...(ensureArray(buildConfig.lint?.command) ?? [])],
+        ...createArtifactsConfig(
+          context.build.dir,
+          buildConfig.lint?.artifactsReports,
+          buildConfig.lint?.artifacts,
+        ),
+      }
+    : null;
+  const testJob: CatladderJob | null = buildConfig.test
+    ? {
+        name: "🧪 test",
+
+        ...base,
+        image: buildConfig.test?.jobImage ?? buildConfig.jobImage,
+        script: [...(ensureArray(buildConfig.test?.command) ?? [])],
+        ...createArtifactsConfig(
+          context.build.dir,
+          buildConfig.test?.artifactsReports,
+          buildConfig.test?.artifacts,
+        ),
+      }
+    : null;
+  return [auditJob, lintJob, testJob].filter(notNil);
+};
+
+export type OptionalArtifacts =
+  | {
+      artifacts: Artifacts;
+    }
+  | undefined;