@catladder/pipeline 0.0.0-node16-ddfae617 → 0.0.0-perf-cli-reduce-bundle-size-with-tsconfig-optimization-3556aaa4

package/src/build/node/index.ts

@@ -1,17 +1,21 @@
-import type { Context } from "../../types/context";
-import { CatladderJob } from "../../types/jobs";
+import type { ComponentContext, WorkspaceContext } from "../../types/context";
+import type { CatladderJob } from "../../types/jobs";
 import { createNodeBuildJobs } from "./buildJob";
 import { createMeteorBuildJobs } from "./meteor";
 import { createNodeTestJobs } from "./testJob";
 
-export const createNodeJobs = (context: Context): CatladderJob[] => {
+export const createNodeJobs = (
+  context: ComponentContext | WorkspaceContext,
+): CatladderJob[] => {
   return [...createNodeTestJobs(context), ...createNodeBuildJobs(context)];
 };
 
-export const createStorybookJobs = (context: Context): CatladderJob[] => {
+export const createStorybookJobs = (
+  context: ComponentContext,
+): CatladderJob[] => {
   return [...createNodeBuildJobs(context)];
 };
 
-export const createMeteorJobs = (context: Context): CatladderJob[] => {
+export const createMeteorJobs = (context: ComponentContext): CatladderJob[] => {
   return [...createNodeTestJobs(context), ...createMeteorBuildJobs(context)];
 };

package/src/build/node/meteor.ts

@@ -1,69 +1,66 @@
 import { join } from "path";
 import { getRunnerImage } from "../../runner";
-import { GitlabJobCache } from "../../types";
-import type { Context } from "../../types/context";
+import type { GitlabJobCache } from "../../types";
+import type { ComponentContext } from "../../types/context";
 
-import { CatladderJob } from "../../types/jobs";
+import type { CatladderJob } from "../../types/jobs";
 
-import { APP_BUILD_JOB_NAME } from "../base/constants";
-import { createBuildJob } from "../base/createBuildJob";
-import { createDockerBuildJobDefault } from "../docker";
+import { createComponentBuildJobs } from "../base";
+import { getDockerBuildScriptWithBuiltInDockerFile } from "../docker";
 import { isOfBuildType } from "../types";
 import { getNodeCache } from "./cache";
 import { getYarnInstall } from "./yarn";
 
-const getMeteorCache = (context: Context): GitlabJobCache[] => [
+const getMeteorCache = (context: ComponentContext): GitlabJobCache[] => [
   {
-    key: context.componentName + "meteor-build-cache",
+    key: context.name + "meteor-build-cache",
     policy: "pull-push",
     paths: [
-      join(
-        context.componentConfig.dir,
-        ".meteor/local/resolver-result-cache.json"
-      ),
-      join(context.componentConfig.dir, ".meteor/local/plugin-cache"),
-      join(context.componentConfig.dir, ".meteor/local/isopacks"),
-      join(context.componentConfig.dir, ".meteor/local/bundler-cache/scanner"),
+      join(context.build.dir, ".meteor/local/resolver-result-cache.json"),
+      join(context.build.dir, ".meteor/local/plugin-cache"),
+      join(context.build.dir, ".meteor/local/isopacks"),
+      join(context.build.dir, ".meteor/local/bundler-cache/scanner"),
     ],
   },
 ];
-export const createMeteorBuildJobs = (context: Context): CatladderJob[] => {
-  const buildConfig = context.componentConfig.build;
+export const createMeteorBuildJobs = (
+  context: ComponentContext,
+): CatladderJob[] => {
+  const buildConfig = context.build.config;
 
   if (!isOfBuildType(buildConfig, "meteor")) {
     throw new Error("deploy config is not meteor");
   }
 
   const yarnInstall = getYarnInstall(context);
-  const appBuildJob: CatladderJob | null =
-    buildConfig.buildCommand !== null
-      ? createBuildJob(context, {
-          cache: [...getNodeCache(context), ...getMeteorCache(context)],
-          image: getRunnerImage("jobs-meteor"),
-          script: [
-            ...yarnInstall,
 
-            'echo "add healthcheck package"',
-            "meteor add panter:healthroute --allow-superuser",
-            "meteor add qualia:prod-shell --allow-superuser",
+  return createComponentBuildJobs(context, {
+    appBuild:
+      buildConfig.buildCommand !== null
+        ? {
+            cache: [...getNodeCache(context), ...getMeteorCache(context)],
+            image: getRunnerImage("jobs-meteor"),
+            variables: {
+              METEOR_DISABLE_OPTIMISTIC_CACHING: "1", // see https://forums.meteor.com/t/veeery-long-building-time-inside-docker-container/58673/17?u=macrozone
+            },
+            script: [
+              ...yarnInstall,
 
-            'TOOL_NODE_FLAGS="--max_old_space_size=3584 --min_semi_space_size=8 --max_semi_space_size=256 --optimize_for_size" meteor build ./dist --architecture os.linux.x86_64 --allow-superuser --server-only --directory',
+              'echo "add healthcheck package"',
+              "meteor add panter:healthroute --allow-superuser",
+              "meteor add qualia:prod-shell --allow-superuser",
 
-            "cp ./__build_info.json ./dist/bundle/programs/server",
-          ],
-          artifacts: {
-            paths: [
-              context.componentConfig.dir + "/__build_info.json",
-              context.componentConfig.dir + "/dist",
+              'TOOL_NODE_FLAGS="--max_old_space_size=3584 --min_semi_space_size=8 --max_semi_space_size=256 --optimize_for_size" meteor build ./dist --architecture os.linux.x86_64 --allow-superuser --server-only --directory',
+
+              "cp ./__build_info.json ./dist/bundle/programs/server",
             ],
-          },
-        })
-      : null;
-  return [
-    ...(appBuildJob ? [appBuildJob] : []),
-    createDockerBuildJobDefault(context, {
-      script: ["ensureMeteorDockerfile"],
-      needs: appBuildJob ? [APP_BUILD_JOB_NAME] : [],
-    }),
-  ];
+          }
+        : undefined,
+    dockerBuild: {
+      script: getDockerBuildScriptWithBuiltInDockerFile(context, "meteor"),
+      variables: {
+        METEOR_INSTALL_SCRIPTS: buildConfig.installScripts ? "true" : "",
+      },
+    },
+  });
 };

package/src/build/node/testJob.ts

@@ -1,45 +1,72 @@
-import { Context } from "../../types/context";
-import { CatladderJob } from "../../types/jobs";
+import { getRunnerImage } from "../../runner";
+import type { WorkspaceContext } from "../../types/context";
+import {
+  componentContextIsStandaloneBuild,
+  type ComponentContext,
+} from "../../types/context";
+import type { CatladderJob } from "../../types/jobs";
 import { ensureArray, notNil } from "../../utils";
+import { createArtifactsConfig } from "../base/createArtifactsConfig";
 import { getNodeCache } from "./cache";
 import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
-import { getYarnInstall } from "./yarn";
+import { ensureNodeVersion, getYarnInstall } from "./yarn";
 
-export const createNodeTestJobs = (context: Context): CatladderJob[] => {
+export const createNodeTestJobs = (
+  context: ComponentContext | WorkspaceContext,
+): CatladderJob[] => {
   // don't run tests after release
-  if (context.commitInfo?.trigger === "taggedRelease") {
+  // TODO: this will be replaced by using rules
+  if (context.trigger === "taggedRelease") {
+    return [];
+  }
+  // if its not a standalone build, we don't need to run tests
+  if (
+    context.type === "component" &&
+    !componentContextIsStandaloneBuild(context)
+  ) {
     return [];
   }
 
-  const buildConfig = context.componentConfig.build;
-
+  const defaultImage = getRunnerImage("jobs-default");
   const base: Omit<CatladderJob, "script" | "name"> = {
     variables: {
-      APP_PATH: context.componentConfig.dir,
-      ...NODE_RUNNER_BUILD_VARIABLES,
-      ...(buildConfig.extraVars ?? {}),
+      APP_PATH: context.build.dir,
+      ...(context.type === "component"
+        ? {
+            ...context.environment.jobOnlyVars.build.envVars,
+            ...(context.build.config.extraVars ?? {}),
+          }
+        : {}),
     },
+
+    runnerVariables: NODE_RUNNER_BUILD_VARIABLES,
     stage: "test",
     needs: [],
     envMode: "none",
   };
+  const buildConfig = context.build.config;
   const yarnInstall = getYarnInstall(context);
   const auditJob: CatladderJob | null =
     buildConfig.audit !== false
       ? {
           name: "🛡 audit",
-
           ...base,
+          image: buildConfig.audit?.jobImage ?? defaultImage,
           cache: undefined, // audit does not need yarn install and no cache
           script: [
-            `cd ${context.componentConfig.dir}`,
+            `cd ${context.build.dir}`,
             ...(ensureArray(buildConfig.audit?.command) ?? [
-              context.packageManagerInfo?.isClassic
+              context.packageManagerInfo.isClassic
                 ? "yarn audit"
                 : "yarn npm audit --environment production", // yarn 2
             ]),
           ],
           allow_failure: true,
+          ...createArtifactsConfig(
+            context.build.dir,
+            buildConfig.audit?.artifactsReports,
+            buildConfig.audit?.artifacts,
+          ),
         }
       : null;
 
@@ -47,14 +74,20 @@ export const createNodeTestJobs = (context: Context): CatladderJob[] => {
     buildConfig.lint !== false
       ? {
           name: "👮 lint",
-
           ...base,
+          image: buildConfig.lint?.jobImage ?? defaultImage,
           cache: getNodeCache(context),
           script: [
-            `cd ${context.componentConfig.dir}`,
+            ...ensureNodeVersion(context),
+            `cd ${context.build.dir}`,
             ...yarnInstall,
             ...(ensureArray(buildConfig.lint?.command) ?? ["yarn lint"]),
           ],
+          ...createArtifactsConfig(
+            context.build.dir,
+            buildConfig.lint?.artifactsReports,
+            buildConfig.lint?.artifacts,
+          ),
         }
       : null;
   const testJob: CatladderJob | null =
@@ -63,12 +96,20 @@ export const createNodeTestJobs = (context: Context): CatladderJob[] => {
           name: "🧪 test",
 
           ...base,
+          image:
+            buildConfig.test?.jobImage ?? getRunnerImage("jobs-testing-chrome"),
           cache: getNodeCache(context),
           script: [
-            `cd ${context.componentConfig.dir}`,
+            ...ensureNodeVersion(context),
+            `cd ${context.build.dir}`,
             ...yarnInstall,
             ...(ensureArray(buildConfig.test?.command) ?? ["yarn test"]),
           ],
+          ...createArtifactsConfig(
+            context.build.dir,
+            buildConfig.test?.artifactsReports,
+            buildConfig.test?.artifacts,
+          ),
         }
       : null;
   return [auditJob, lintJob, testJob].filter(notNil);

package/src/build/node/yarn.ts

@@ -1,29 +1,85 @@
-import { Context } from "../../types";
+import { BashExpression } from "../../bash/BashExpression";
+import type { Context } from "../../types";
+import { ensureArray } from "../../utils";
+import { collapseableSection } from "../../utils/gitlab";
 
-export const getYarnInstallCommand = (
+const YARN_INSTALL_CLASSIC = `yarn install --frozen-lockfile`;
+
+// FIXME: check why and when rebuild is needed
+const YARN_BERRY_PROD_REBUILD = `yarn workspaces focus --production && yarn rebuild`;
+
+const getYarnInstallCommand = (context: Context) => {
+  if (context.packageManagerInfo.isClassic) {
+    return YARN_INSTALL_CLASSIC;
+  }
+
+  return `yarn install --immutable`;
+};
+
+export const ensureNodeVersion = (context: Context) =>
+  collapseableSection(
+    "nodeinstall",
+    "Ensure node version",
+  )([
+    "if [ -f ~/.nvm/nvm.sh ];  then source ~/.nvm/nvm.sh; fi",
+    "if command -v nvm &> /dev/null && [ -f ./.nvmrc ]; then nvm install; fi",
+  ]);
+
+export const getYarnInstall = (
   context: Context,
   options?: {
-    prodOnly?: boolean;
-    noScripts?: boolean;
+    noCustomPostInstall: boolean;
+  },
+) => {
+  const postInstall =
+    context.type !== "workspace" && "postInstall" in context.build.config
+      ? context.build.config.postInstall
+      : null;
+  return [
+    ...ensureNodeVersion(context),
+    ...collapseableSection(
+      "yarninstall",
+      "Yarn install",
+    )([getYarnInstallCommand(context)]),
+    ...(postInstall && !options?.noCustomPostInstall
+      ? collapseableSection(
+          "postinstall",
+          "Custom post install",
+        )(ensureArray(postInstall) ?? [])
+      : []),
+  ];
+};
+
+const DOCKER_COPY_FILES = `COPY --chown=node:node $APP_DIR .`;
+
+export const getDockerAppCopyAndBuildScript = (context: Context) => {
+  if (context.packageManagerInfo.isClassic) {
+    return new BashExpression(
+      `
+RUN ${YARN_INSTALL_CLASSIC} --production --ignore-scripts
+${DOCKER_COPY_FILES}
+RUN ${YARN_INSTALL_CLASSIC} --production 
+    `.trim(),
+    );
   }
-) =>
-  context.packageManagerInfo?.isClassic
-    ? `yarn install --frozen-lockfile ${
-        options?.prodOnly ? "--production" : ""
-      } ${options?.noScripts ? "--ignore-scripts" : ""}`
-    : options?.prodOnly
-    ? `${
-        options?.noScripts ? "YARN_ENABLE_SCRIPTS=false " : ""
-      }yarn plugin import workspace-tools && yarn workspaces focus --production` // needs yarn plugin import workspace-tools
-    : `${
-        options?.noScripts ? "YARN_ENABLE_SCRIPTS=false " : ""
-      }yarn install --immutable`;
-
-export const ensureNodeVersion = (context: Context) => [
-  "if [ -f ./.nvmrc ]; then source /root/.nvm/nvm.sh && nvm install <<< .nvmrc; fi",
-];
-
-export const getYarnInstall = (context: Context) => [
-  ...ensureNodeVersion(context),
-  getYarnInstallCommand(context),
-];
+
+  // yarn >= 4 ships with build in plugins, see https://github.com/yarnpkg/berry/pull/4253
+  // trying to import those fail on this version
+  const doesNotShipWithBuiltInPlugins = ["2", "3"].some((v) =>
+    context.packageManagerInfo.version.startsWith(v),
+  );
+  const maybeAddWorkspaceToolsCommand = doesNotShipWithBuiltInPlugins
+    ? "RUN yarn plugin import workspace-tools"
+    : "";
+
+  // copy first everything and then install
+  // rebuild first does not work as it will run postinstall and that might require files in the app
+  return new BashExpression(
+    `
+${DOCKER_COPY_FILES}
+${maybeAddWorkspaceToolsCommand}
+RUN ${YARN_BERRY_PROD_REBUILD}
+
+    `.trim(),
+  );
+};

package/src/build/rails/build.ts

@@ -0,0 +1,69 @@
+import {
+  componentContextIsStandaloneBuild,
+  type BuildConfigRails,
+  type BuildContextStandalone,
+  type ComponentContext,
+} from "../..";
+import type { CatladderJob } from "../../types/jobs";
+import { createComponentBuildJobs } from "../base";
+import {
+  getDockerBuildDefaultScript,
+  gitlabDockerLogin,
+  hasDockerfile,
+} from "../docker";
+import { isOfBuildType } from "../types";
+
+export const createRailsBuildJobs = (
+  context: ComponentContext,
+): CatladderJob[] => {
+  const buildConfig = context.build.config;
+  if (!isOfBuildType(buildConfig, "rails")) {
+    // should not happen
+    throw new Error("build type is not rails");
+  }
+
+  // if its not a standalone build, we don't need to run tests
+  if (!componentContextIsStandaloneBuild(context)) {
+    throw new Error("workspace builds are not supported for rails apps");
+  }
+
+  if (hasDockerfile(context)) {
+    return createComponentBuildJobs(context, {
+      appBuild: undefined,
+      dockerBuild: {
+        script: getDockerBuildDefaultScript(context),
+        variables: {},
+      },
+    });
+  }
+
+  const cnbConf = buildConfig.cnbBuilder;
+
+  const packEnvArgs = Object.entries(cnbConf?.buildVars ?? {})
+    .map(([k, v]) => `--env '${k}${v ? `=${v}` : ""}'`)
+    .join(" ");
+  return createComponentBuildJobs(context, {
+    appBuild: undefined,
+    dockerBuild: {
+      variables: {
+        ...context.environment.jobOnlyVars.build.envVars,
+        ...context.build.config.extraVars,
+      },
+      // custom script
+      script: [
+        ...gitlabDockerLogin(context),
+        `cd ${context.build.dir}`,
+        `docker pull $DOCKER_CACHE_IMAGE || true`,
+        `wget --output-document=- https://github.com/buildpacks/pack/releases/download/v${cnbConf?.packVersion}/pack-v${cnbConf?.packVersion}-linux.tgz | tar -zx --directory /usr/local/bin pack`,
+        `chmod +x /usr/local/bin/pack`,
+        //  replace private git ssh gem sources with https to make bundler with credentials via env var work
+        `sed --in-place 's|git@\\([^:]*\\):|https://\\1/|g' Gemfile Gemfile.lock`,
+        `pack build "$DOCKER_IMAGE:$DOCKER_IMAGE_TAG" --builder '${
+          cnbConf?.image
+        }' --publish --cache-image "$DOCKER_CACHE_IMAGE" ${packEnvArgs} ${
+          cnbConf?.packExtraArgs?.join(" ") ?? ""
+        }`,
+      ],
+    },
+  });
+};

package/src/build/rails/index.ts

@@ -0,0 +1,8 @@
+import type { ComponentContext } from "../../types";
+import type { CatladderJob } from "../../types/jobs";
+import { createRailsBuildJobs } from "./build";
+import { createRailsTestJobs } from "./test";
+
+export const createRailsJobs = (context: ComponentContext): CatladderJob[] => {
+  return [...createRailsTestJobs(context), ...createRailsBuildJobs(context)];
+};

package/src/build/rails/test.ts

@@ -0,0 +1,103 @@
+import {
+  componentContextIsStandaloneBuild,
+  type ComponentContext,
+} from "../..";
+import type { CatladderJob } from "../../types/jobs";
+import { ensureArray, notNil } from "../../utils";
+
+export const createRailsTestJobs = (
+  context: ComponentContext,
+): CatladderJob[] => {
+  // don't run tests after release
+  // TODO: this will be replaced by using rules
+  if (context.trigger === "taggedRelease") {
+    return [];
+  }
+
+  // if its not a standalone build, we don't need to run tests
+  if (!componentContextIsStandaloneBuild(context)) {
+    return [];
+  }
+
+  const buildConfig = context.build.config;
+
+  const base: Omit<CatladderJob, "script" | "name"> = {
+    variables: {
+      ...context.environment.jobOnlyVars.build.envVars,
+      ...(buildConfig.extraVars ?? {}),
+    },
+    stage: "test",
+    needs: [],
+    envMode: "none",
+  };
+  const defaultImage = "docker.io/ruby";
+  const bundlerCacheDir = "tmp/cache";
+  const bundlerInstall = [
+    `bundle config set path '${bundlerCacheDir}'`,
+    "bundle install -j $(nproc)",
+  ];
+  const bundlerCache = {
+    key: {
+      files: ["Gemfile.lock"],
+      prefix: "$CI_JOB_IMAGE", // a changed image might have different OS libraries which no longer work with the cached gems
+    },
+    paths: [bundlerCacheDir],
+  };
+  const auditJob: CatladderJob | null =
+    buildConfig.audit !== false
+      ? {
+          name: "🛡 audit",
+
+          ...base,
+          cache: undefined, // audit does not need bundle install and no cache
+          image:
+            buildConfig.audit?.jobImage ?? buildConfig.jobImage ?? defaultImage,
+          script: [
+            `cd ${context.build.dir}`,
+            ...(ensureArray(buildConfig.audit?.command) ?? [
+              "gem install bundler-audit",
+              "bundle audit check",
+            ]),
+          ],
+          allow_failure: true,
+        }
+      : null;
+
+  const lintJob: CatladderJob | null =
+    buildConfig.lint !== false
+      ? {
+          name: "👮 lint",
+
+          ...base,
+          cache: bundlerCache,
+          image:
+            buildConfig.lint?.jobImage ?? buildConfig.jobImage ?? defaultImage,
+          script: [
+            `cd ${context.build.dir}`,
+            ...bundlerInstall,
+            ...(ensureArray(buildConfig.lint?.command) ?? [
+              "bundle exec rubocop",
+            ]),
+          ],
+        }
+      : null;
+  const testJob: CatladderJob | null =
+    buildConfig.test !== false
+      ? {
+          name: "🧪 test",
+
+          ...base,
+          cache: bundlerCache,
+          image:
+            buildConfig.test?.jobImage ?? buildConfig.jobImage ?? defaultImage,
+          script: [
+            `cd ${context.build.dir}`,
+            ...bundlerInstall,
+            ...(ensureArray(buildConfig.test?.command) ?? [
+              "bundle exec rspec",
+            ]),
+          ],
+        }
+      : null;
+  return [auditJob, lintJob, testJob].filter(notNil);
+};

package/src/build/sbom.ts

@@ -0,0 +1,48 @@
+import { isStandaloneBuildConfig } from ".";
+import type { ComponentContext } from "../types/context";
+import type { CatladderJob } from "../types/jobs";
+import { ensureArray } from "../utils";
+
+export const SBOM_BUILD_JOB_NAME = "🧾 sbom";
+export const SBOM_FILE = "__sbom.json";
+
+export const createSbomBuildJob = (context: ComponentContext): CatladderJob => {
+  const buildConfig = context.build.config;
+
+  const defaultImage = "aquasec/trivy:0.38.3";
+  const defaultScript = [
+    `trivy fs --quiet --format cyclonedx --output "${SBOM_FILE}" ${
+      context.packageManagerInfo.componentIsInWorkspace
+        ? "."
+        : context.build.dir
+    }`,
+  ];
+
+  const image =
+    isStandaloneBuildConfig(buildConfig) &&
+    buildConfig.type === "custom" &&
+    buildConfig.sbom !== false
+      ? buildConfig.sbom?.jobImage ?? defaultImage
+      : defaultImage;
+
+  const script =
+    isStandaloneBuildConfig(buildConfig) &&
+    buildConfig.type === "custom" &&
+    buildConfig.sbom !== false
+      ? ensureArray(buildConfig.sbom?.command) ?? defaultScript
+      : defaultScript;
+
+  return {
+    name: SBOM_BUILD_JOB_NAME,
+    stage: "build",
+    envMode: "jobPerEnv",
+    variables: {},
+    cache: undefined,
+    image,
+    script,
+    allow_failure: true,
+    artifacts: {
+      paths: [SBOM_FILE],
+    },
+  };
+};