@catladder/pipeline 0.0.0-more-collapse-deeb54d7 → 0.0.0-multiple-gitlab-includes-3df5566d

package/src/build/node/meteor.ts

@@ -1,33 +1,31 @@
-import { join } from "path";
 import { getRunnerImage } from "../../runner";
-import type { GitlabJobCache } from "../../types";
-import type { Context } from "../../types/context";
+import type { ComponentContext } from "../../types/context";
 
 import type { CatladderJob } from "../../types/jobs";
 
-import { createBuildJobs } from "../base";
-import { getDockerBuildDefaultScript } from "../docker";
+import { createComponentBuildJobs } from "../base";
+import { getDockerBuildScriptWithBuiltInDockerFile } from "../docker";
+import type { CacheConfig } from "../types";
 import { isOfBuildType } from "../types";
 import { getNodeCache } from "./cache";
 import { getYarnInstall } from "./yarn";
 
-const getMeteorCache = (context: Context): GitlabJobCache[] => [
+const getMeteorCache = (context: ComponentContext): CacheConfig[] => [
   {
-    key: context.componentName + "meteor-build-cache",
+    key: "meteor-build-cache",
     policy: "pull-push",
     paths: [
-      join(
-        context.componentConfig.dir,
-        ".meteor/local/resolver-result-cache.json"
-      ),
-      join(context.componentConfig.dir, ".meteor/local/plugin-cache"),
-      join(context.componentConfig.dir, ".meteor/local/isopacks"),
-      join(context.componentConfig.dir, ".meteor/local/bundler-cache/scanner"),
+      ".meteor/local/resolver-result-cache.json",
+      ".meteor/local/plugin-cache",
+      ".meteor/local/isopacks",
+      ".meteor/local/bundler-cache/scanner",
     ],
   },
 ];
-export const createMeteorBuildJobs = (context: Context): CatladderJob[] => {
-  const buildConfig = context.componentConfig.build;
+export const createMeteorBuildJobs = (
+  context: ComponentContext,
+): CatladderJob[] => {
+  const buildConfig = context.build.config;
 
   if (!isOfBuildType(buildConfig, "meteor")) {
     throw new Error("deploy config is not meteor");
@@ -35,7 +33,7 @@ export const createMeteorBuildJobs = (context: Context): CatladderJob[] => {
 
   const yarnInstall = getYarnInstall(context);
 
-  return createBuildJobs(context, {
+  return createComponentBuildJobs(context, {
     appBuild:
       buildConfig.buildCommand !== null
         ? {
@@ -55,16 +53,10 @@ export const createMeteorBuildJobs = (context: Context): CatladderJob[] => {
 
               "cp ./__build_info.json ./dist/bundle/programs/server",
             ],
-            artifacts: {
-              paths: [
-                context.componentConfig.dir + "/__build_info.json",
-                context.componentConfig.dir + "/dist",
-              ],
-            },
           }
         : undefined,
     dockerBuild: {
-      script: getDockerBuildDefaultScript("ensureMeteorDockerfile"),
+      script: getDockerBuildScriptWithBuiltInDockerFile(context, "meteor"),
       variables: {
         METEOR_INSTALL_SCRIPTS: buildConfig.installScripts ? "true" : "",
       },

package/src/build/node/testJob.ts

@@ -1,31 +1,50 @@
 import { getRunnerImage } from "../../runner";
-import type { Context } from "../../types/context";
+import type { WorkspaceContext } from "../../types/context";
+import {
+  componentContextIsStandaloneBuild,
+  type ComponentContext,
+} from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
-import { ensureArray, notNil } from "../../utils";
+import { ensureArrayOrNull, notNil } from "../../utils";
+import { createArtifactsConfig } from "../base/createArtifactsConfig";
 import { getNodeCache } from "./cache";
 import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
-import { getYarnInstall } from "./yarn";
-import { createArtifactsConfig } from "../base/createArtifactsConfig";
+import { ensureNodeVersion, getYarnInstall } from "./yarn";
+import { createJobCacheFromCacheConfigs } from "../cache/createJobCache";
 
-export const createNodeTestJobs = (context: Context): CatladderJob[] => {
+export const createNodeTestJobs = (
+  context: ComponentContext | WorkspaceContext,
+): CatladderJob[] => {
   // don't run tests after release
-  if (context.commitInfo?.trigger === "taggedRelease") {
+  // TODO: this will be replaced by using rules
+  if (context.trigger === "taggedRelease") {
+    return [];
+  }
+  // if its not a standalone build, we don't need to run tests
+  if (
+    context.type === "component" &&
+    !componentContextIsStandaloneBuild(context)
+  ) {
     return [];
   }
 
-  const buildConfig = context.componentConfig.build;
   const defaultImage = getRunnerImage("jobs-default");
   const base: Omit<CatladderJob, "script" | "name"> = {
     variables: {
-      APP_PATH: context.componentConfig.dir,
-      ...NODE_RUNNER_BUILD_VARIABLES,
-      ...context.environment.jobOnlyVars.build.envVars,
-      ...(buildConfig.extraVars ?? {}),
+      APP_PATH: context.build.dir,
+      ...(context.type === "component"
+        ? {
+            ...context.environment.jobOnlyVars.build.envVars,
+          }
+        : {}),
     },
+
+    runnerVariables: NODE_RUNNER_BUILD_VARIABLES,
     stage: "test",
     needs: [],
     envMode: "none",
   };
+  const buildConfig = context.build.config;
   const yarnInstall = getYarnInstall(context);
   const auditJob: CatladderJob | null =
     buildConfig.audit !== false
@@ -35,18 +54,18 @@ export const createNodeTestJobs = (context: Context): CatladderJob[] => {
           image: buildConfig.audit?.jobImage ?? defaultImage,
           cache: undefined, // audit does not need yarn install and no cache
           script: [
-            `cd ${context.componentConfig.dir}`,
-            ...(ensureArray(buildConfig.audit?.command) ?? [
-              context.packageManagerInfo?.isClassic
+            `cd ${context.build.dir}`,
+            ...(ensureArrayOrNull(buildConfig.audit?.command) ?? [
+              context.packageManagerInfo.isClassic
                 ? "yarn audit"
                 : "yarn npm audit --environment production", // yarn 2
             ]),
           ],
           allow_failure: true,
           ...createArtifactsConfig(
-            context.componentConfig.dir,
+            context.build.dir,
             buildConfig.audit?.artifactsReports,
-            buildConfig.audit?.artifacts
+            buildConfig.audit?.artifacts,
           ),
         }
       : null;
@@ -57,16 +76,17 @@ export const createNodeTestJobs = (context: Context): CatladderJob[] => {
           name: "👮 lint",
           ...base,
           image: buildConfig.lint?.jobImage ?? defaultImage,
-          cache: getNodeCache(context),
+          cache: createJobCacheFromCacheConfigs(context, getNodeCache(context)),
           script: [
-            `cd ${context.componentConfig.dir}`,
+            ...ensureNodeVersion(context),
+            `cd ${context.build.dir}`,
             ...yarnInstall,
-            ...(ensureArray(buildConfig.lint?.command) ?? ["yarn lint"]),
+            ...(ensureArrayOrNull(buildConfig.lint?.command) ?? ["yarn lint"]),
           ],
           ...createArtifactsConfig(
-            context.componentConfig.dir,
+            context.build.dir,
             buildConfig.lint?.artifactsReports,
-            buildConfig.lint?.artifacts
+            buildConfig.lint?.artifacts,
           ),
         }
       : null;
@@ -78,16 +98,17 @@ export const createNodeTestJobs = (context: Context): CatladderJob[] => {
           ...base,
           image:
             buildConfig.test?.jobImage ?? getRunnerImage("jobs-testing-chrome"),
-          cache: getNodeCache(context),
+          cache: createJobCacheFromCacheConfigs(context, getNodeCache(context)),
           script: [
-            `cd ${context.componentConfig.dir}`,
+            ...ensureNodeVersion(context),
+            `cd ${context.build.dir}`,
             ...yarnInstall,
-            ...(ensureArray(buildConfig.test?.command) ?? ["yarn test"]),
+            ...(ensureArrayOrNull(buildConfig.test?.command) ?? ["yarn test"]),
           ],
           ...createArtifactsConfig(
-            context.componentConfig.dir,
+            context.build.dir,
             buildConfig.test?.artifactsReports,
-            buildConfig.test?.artifacts
+            buildConfig.test?.artifacts,
           ),
         }
       : null;

package/src/build/node/yarn.ts

@@ -1,5 +1,7 @@
+import { BashExpression } from "../../bash/BashExpression";
 import type { Context } from "../../types";
 import { ensureArray } from "../../utils";
+import { collapseableSection } from "../../utils/gitlab";
 
 const YARN_INSTALL_CLASSIC = `yarn install --frozen-lockfile`;
 
@@ -7,33 +9,43 @@ const YARN_INSTALL_CLASSIC = `yarn install --frozen-lockfile`;
 const YARN_BERRY_PROD_REBUILD = `yarn workspaces focus --production && yarn rebuild`;
 
 const getYarnInstallCommand = (context: Context) => {
-  if (context.packageManagerInfo?.isClassic) {
+  if (context.packageManagerInfo.isClassic) {
     return YARN_INSTALL_CLASSIC;
   }
 
   return `yarn install --immutable`;
 };
 
-export const ensureNodeVersion = (context: Context) => [
-  "if [ -f ~/.nvm/nvm.sh ];  then source ~/.nvm/nvm.sh; fi",
-  "if command -v nvm &> /dev/null && [ -f ./.nvmrc ]; then nvm install; fi",
-];
+export const ensureNodeVersion = (context: Context) =>
+  collapseableSection(
+    "nodeinstall",
+    "Ensure node version",
+  )([
+    "if [ -f ~/.nvm/nvm.sh ];  then source ~/.nvm/nvm.sh; fi",
+    "if command -v nvm &> /dev/null && [ -f ./.nvmrc ]; then nvm install; fi",
+  ]);
 
 export const getYarnInstall = (
   context: Context,
   options?: {
     noCustomPostInstall: boolean;
-  }
+  },
 ) => {
   const postInstall =
-    "postInstall" in context.componentConfig.build
-      ? context.componentConfig.build.postInstall
+    context.type !== "workspace" && "postInstall" in context.build.config
+      ? context.build.config.postInstall
       : null;
   return [
     ...ensureNodeVersion(context),
-    getYarnInstallCommand(context),
+    ...collapseableSection(
+      "yarninstall",
+      "Yarn install",
+    )([getYarnInstallCommand(context)]),
     ...(postInstall && !options?.noCustomPostInstall
-      ? ensureArray(postInstall) ?? []
+      ? collapseableSection(
+          "postinstall",
+          "Custom post install",
+        )(ensureArray(postInstall))
       : []),
   ];
 };
@@ -41,18 +53,20 @@ export const getYarnInstall = (
 const DOCKER_COPY_FILES = `COPY --chown=node:node $APP_DIR .`;
 
 export const getDockerAppCopyAndBuildScript = (context: Context) => {
-  if (context.packageManagerInfo?.isClassic) {
-    return `
+  if (context.packageManagerInfo.isClassic) {
+    return new BashExpression(
+      `
 RUN ${YARN_INSTALL_CLASSIC} --production --ignore-scripts
 ${DOCKER_COPY_FILES}
 RUN ${YARN_INSTALL_CLASSIC} --production 
-    `.trim();
+    `.trim(),
+    );
   }
 
   // yarn >= 4 ships with build in plugins, see https://github.com/yarnpkg/berry/pull/4253
   // trying to import those fail on this version
   const doesNotShipWithBuiltInPlugins = ["2", "3"].some((v) =>
-    context.packageManagerInfo?.version.startsWith(v)
+    context.packageManagerInfo.version.startsWith(v),
   );
   const maybeAddWorkspaceToolsCommand = doesNotShipWithBuiltInPlugins
     ? "RUN yarn plugin import workspace-tools"
@@ -60,10 +74,12 @@ RUN ${YARN_INSTALL_CLASSIC} --production
 
   // copy first everything and then install
   // rebuild first does not work as it will run postinstall and that might require files in the app
-  return `
+  return new BashExpression(
+    `
 ${DOCKER_COPY_FILES}
 ${maybeAddWorkspaceToolsCommand}
 RUN ${YARN_BERRY_PROD_REBUILD}
 
-    `.trim();
+    `.trim(),
+  );
 };

package/src/build/rails/build.ts

@@ -1,6 +1,11 @@
-import type { Context } from "../..";
+import {
+  componentContextIsStandaloneBuild,
+  type BuildConfigRails,
+  type BuildContextStandalone,
+  type ComponentContext,
+} from "../..";
 import type { CatladderJob } from "../../types/jobs";
-import { createBuildJobs } from "../base";
+import { createComponentBuildJobs } from "../base";
 import {
   getDockerBuildDefaultScript,
   gitlabDockerLogin,
@@ -8,18 +13,25 @@ import {
 } from "../docker";
 import { isOfBuildType } from "../types";
 
-export const createRailsBuildJobs = (context: Context): CatladderJob[] => {
-  const buildConfig = context.componentConfig.build;
+export const createRailsBuildJobs = (
+  context: ComponentContext,
+): CatladderJob[] => {
+  const buildConfig = context.build.config;
   if (!isOfBuildType(buildConfig, "rails")) {
     // should not happen
     throw new Error("build type is not rails");
   }
 
+  // if its not a standalone build, we don't need to run tests
+  if (!componentContextIsStandaloneBuild(context)) {
+    throw new Error("workspace builds are not supported for rails apps");
+  }
+
   if (hasDockerfile(context)) {
-    return createBuildJobs(context, {
+    return createComponentBuildJobs(context, {
       appBuild: undefined,
       dockerBuild: {
-        script: getDockerBuildDefaultScript(),
+        script: getDockerBuildDefaultScript(context),
         variables: {},
       },
     });
@@ -30,16 +42,16 @@ export const createRailsBuildJobs = (context: Context): CatladderJob[] => {
   const packEnvArgs = Object.entries(cnbConf?.buildVars ?? {})
     .map(([k, v]) => `--env '${k}${v ? `=${v}` : ""}'`)
     .join(" ");
-  return createBuildJobs(context, {
+  return createComponentBuildJobs(context, {
     appBuild: undefined,
     dockerBuild: {
       variables: {
         ...context.environment.jobOnlyVars.build.envVars,
-        ...context.componentConfig.build.extraVars,
       },
       // custom script
       script: [
-        gitlabDockerLogin,
+        ...gitlabDockerLogin(context),
+        `cd ${context.build.dir}`,
         `docker pull $DOCKER_CACHE_IMAGE || true`,
         `wget --output-document=- https://github.com/buildpacks/pack/releases/download/v${cnbConf?.packVersion}/pack-v${cnbConf?.packVersion}-linux.tgz | tar -zx --directory /usr/local/bin pack`,
         `chmod +x /usr/local/bin/pack`,

package/src/build/rails/index.ts

@@ -1,8 +1,8 @@
-import type { Context } from "../../types";
+import type { ComponentContext } from "../../types";
 import type { CatladderJob } from "../../types/jobs";
 import { createRailsBuildJobs } from "./build";
 import { createRailsTestJobs } from "./test";
 
-export const createRailsJobs = (context: Context): CatladderJob[] => {
+export const createRailsJobs = (context: ComponentContext): CatladderJob[] => {
   return [...createRailsTestJobs(context), ...createRailsBuildJobs(context)];
 };

package/src/build/rails/test.ts

@@ -1,19 +1,29 @@
-import type { Context } from "../..";
+import {
+  componentContextIsStandaloneBuild,
+  type ComponentContext,
+} from "../..";
 import type { CatladderJob } from "../../types/jobs";
-import { ensureArray, notNil } from "../../utils";
+import { ensureArrayOrNull, notNil } from "../../utils";
 
-export const createRailsTestJobs = (context: Context): CatladderJob[] => {
+export const createRailsTestJobs = (
+  context: ComponentContext,
+): CatladderJob[] => {
   // don't run tests after release
-  if (context.commitInfo?.trigger === "taggedRelease") {
+  // TODO: this will be replaced by using rules
+  if (context.trigger === "taggedRelease") {
     return [];
   }
 
-  const buildConfig = context.componentConfig.build;
+  // if its not a standalone build, we don't need to run tests
+  if (!componentContextIsStandaloneBuild(context)) {
+    return [];
+  }
+
+  const buildConfig = context.build.config;
 
   const base: Omit<CatladderJob, "script" | "name"> = {
     variables: {
       ...context.environment.jobOnlyVars.build.envVars,
-      ...(buildConfig.extraVars ?? {}),
     },
     stage: "test",
     needs: [],
@@ -42,8 +52,8 @@ export const createRailsTestJobs = (context: Context): CatladderJob[] => {
           image:
             buildConfig.audit?.jobImage ?? buildConfig.jobImage ?? defaultImage,
           script: [
-            `cd ${context.componentConfig.dir}`,
-            ...(ensureArray(buildConfig.audit?.command) ?? [
+            `cd ${context.build.dir}`,
+            ...(ensureArrayOrNull(buildConfig.audit?.command) ?? [
               "gem install bundler-audit",
               "bundle audit check",
             ]),
@@ -62,9 +72,9 @@ export const createRailsTestJobs = (context: Context): CatladderJob[] => {
           image:
             buildConfig.lint?.jobImage ?? buildConfig.jobImage ?? defaultImage,
           script: [
-            `cd ${context.componentConfig.dir}`,
+            `cd ${context.build.dir}`,
             ...bundlerInstall,
-            ...(ensureArray(buildConfig.lint?.command) ?? [
+            ...(ensureArrayOrNull(buildConfig.lint?.command) ?? [
               "bundle exec rubocop",
             ]),
           ],
@@ -80,9 +90,9 @@ export const createRailsTestJobs = (context: Context): CatladderJob[] => {
           image:
             buildConfig.test?.jobImage ?? buildConfig.jobImage ?? defaultImage,
           script: [
-            `cd ${context.componentConfig.dir}`,
+            `cd ${context.build.dir}`,
             ...bundlerInstall,
-            ...(ensureArray(buildConfig.test?.command) ?? [
+            ...(ensureArrayOrNull(buildConfig.test?.command) ?? [
               "bundle exec rspec",
             ]),
           ],

package/src/build/sbom.ts

@@ -1,30 +1,35 @@
-import type { Context } from "../types/context";
+import { isStandaloneBuildConfig } from ".";
+import type { ComponentContext } from "../types/context";
 import type { CatladderJob } from "../types/jobs";
-import { ensureArray } from "../utils";
+import { ensureArrayOrNull } from "../utils";
 
 export const SBOM_BUILD_JOB_NAME = "🧾 sbom";
 export const SBOM_FILE = "__sbom.json";
 
-export const createSbomBuildJob = (context: Context): CatladderJob => {
-  const buildConfig = context.componentConfig.build;
+export const createSbomBuildJob = (context: ComponentContext): CatladderJob => {
+  const buildConfig = context.build.config;
 
   const defaultImage = "aquasec/trivy:0.38.3";
   const defaultScript = [
     `trivy fs --quiet --format cyclonedx --output "${SBOM_FILE}" ${
-      context.packageManagerInfo?.componentIsInWorkspace
+      context.packageManagerInfo.componentIsInWorkspace
         ? "."
-        : context.componentConfig.dir
+        : context.build.dir
     }`,
   ];
 
   const image =
-    buildConfig.type === "custom" && buildConfig.sbom !== false
-      ? buildConfig.sbom?.jobImage ?? defaultImage
+    isStandaloneBuildConfig(buildConfig) &&
+    buildConfig.type === "custom" &&
+    buildConfig.sbom !== false
+      ? (buildConfig.sbom?.jobImage ?? defaultImage)
       : defaultImage;
 
   const script =
-    buildConfig.type === "custom" && buildConfig.sbom !== false
-      ? ensureArray(buildConfig.sbom?.command) ?? defaultScript
+    isStandaloneBuildConfig(buildConfig) &&
+    buildConfig.type === "custom" &&
+    buildConfig.sbom !== false
+      ? (ensureArrayOrNull(buildConfig.sbom?.command) ?? defaultScript)
       : defaultScript;
 
   return {