@catladder/pipeline 1.111.4 → 1.112.1

package/src/build/base/index.ts

@@ -0,0 +1,35 @@
+import { sbomDeactivated } from "../../deploy/sbom";
+import type { Context } from "../../types/context";
+import type { CatladderJob } from "../../types/jobs";
+import type { DockerBuildJobDefinition } from "../docker";
+import { createDockerBuildJobBase, requiresDockerBuild } from "../docker";
+import { createSbomBuildJob } from "../sbom";
+import { APP_BUILD_JOB_NAME } from "./constants";
+import type { AppBuildJobDefinition } from "./createAppBuildJob";
+import { createAppBuildJob } from "./createAppBuildJob";
+
+export const createBuildJobs = (
+  context: Context,
+  definitions: {
+    appBuild?: AppBuildJobDefinition;
+    dockerBuild: DockerBuildJobDefinition;
+  }
+): CatladderJob[] => {
+  return [
+    ...(definitions.appBuild
+      ? [createAppBuildJob(context, definitions.appBuild)]
+      : []),
+    ...(requiresDockerBuild(context)
+      ? [
+          createDockerBuildJobBase(context, {
+            ...definitions.dockerBuild,
+            needs: [
+              ...(definitions.dockerBuild.needs ?? []),
+              ...(definitions.appBuild ? [APP_BUILD_JOB_NAME] : []),
+            ],
+          }),
+        ]
+      : []),
+    ...(sbomDeactivated(context) ? [] : [createSbomBuildJob(context)]),
+  ];
+};

package/src/build/custom/buildJob.ts

@@ -1,14 +1,11 @@
 import { join } from "path";
 import type { Context } from "../../types/context";
 import { ensureArray } from "../../utils";
-import { APP_BUILD_JOB_NAME } from "../base/constants";
-import { createBuildJob } from "../base/createBuildJob";
-import { createDockerBuildJobDefault, requiresDockerBuild } from "../docker";
+import { getDockerBuildDefaultScript, requiresDockerBuild } from "../docker";
 import { isOfBuildType } from "../types";
 
 import type { CatladderJob } from "../../types/jobs";
-import { createSbomBuildJob } from "../sbom";
-import { sbomDeactivated } from "../../deploy/sbom";
+import { createBuildJobs } from "../base";
 
 const RUNNER_BUILD_VARIABLES = {
   KUBERNETES_CPU_REQUEST: "0.5",
@@ -24,45 +21,35 @@ export const createCustomBuildJobs = (context: Context): CatladderJob[] => {
     throw new Error("deploy config is not custom");
   }
 
-  const appBuildJob: CatladderJob | null =
-    buildConfig.buildCommand !== null
-      ? createBuildJob(context, {
-          image: buildConfig.jobImage,
-          variables: {
-            ...RUNNER_BUILD_VARIABLES,
-          },
-          cache: buildConfig.jobCache,
-          services: buildConfig.jobServices,
-          script: [...(ensureArray(buildConfig.buildCommand) ?? [])],
-          artifacts: {
-            paths: [
-              join(context.componentConfig.dir, "__build_info.json"),
-              join(context.componentConfig.dir, "dist"),
-              ...(buildConfig.artifactsPaths?.map((path) =>
-                join(context.componentConfig.dir, path)
-              ) ?? []),
-            ],
-            expire_in: "1 day",
-          },
-        })
-      : null;
-  return [
-    ...(appBuildJob ? [appBuildJob] : []),
-    ...(requiresDockerBuild(context)
-      ? [
-          createDockerBuildJobDefault(context, {
-            script: [
-              buildConfig.docker?.type === "nginx"
-                ? "ensureNginxDockerfile"
-                : "", // e.g. custom,
-            ].filter(Boolean),
+  return createBuildJobs(context, {
+    appBuild:
+      buildConfig.buildCommand !== null
+        ? {
+            image: buildConfig.jobImage,
+            variables: {
+              ...RUNNER_BUILD_VARIABLES,
+            },
+            cache: buildConfig.jobCache,
+            services: buildConfig.jobServices,
+            script: [...(ensureArray(buildConfig.buildCommand) ?? [])],
+            artifacts: {
+              paths: [
+                join(context.componentConfig.dir, "__build_info.json"),
+                join(context.componentConfig.dir, "dist"),
+                ...(buildConfig.artifactsPaths?.map((path) =>
+                  join(context.componentConfig.dir, path)
+                ) ?? []),
+              ],
+              expire_in: "1 day",
+            },
+          }
+        : undefined,
+    dockerBuild: {
+      script: getDockerBuildDefaultScript(
+        buildConfig.docker?.type === "nginx" ? "ensureNginxDockerfile" : ""
+      ),
 
-            variables: {},
-
-            needs: appBuildJob ? [APP_BUILD_JOB_NAME] : [],
-          }),
-        ]
-      : []),
-    ...(sbomDeactivated(context) ? [] : [createSbomBuildJob(context)]),
-  ];
+      variables: {},
+    },
+  });
 };

package/src/build/docker.ts

@@ -57,6 +57,7 @@ export const getDockerBuildVariables = (context: Context) => {
 };
 
 export const DOCKER_BUILD_JOB_NAME = "🔨 docker";
+
 export const getDockerJobBaseProps = (context: Context) => {
   return {
     image: getRunnerImage("docker-build"),
@@ -69,6 +70,8 @@ export const getDockerJobBaseProps = (context: Context) => {
     variables: getDockerBaseVariables(),
   };
 };
+
+export type DockerBuildJobDefinition = Partial<CatladderJob>;
 export const createDockerBuildJobBase = (
   context: Context,
   { script, ...def }: Partial<CatladderJob>
@@ -90,20 +93,14 @@ export const createDockerBuildJobBase = (
 
 export const gitlabDockerLogin =
   "docker login --username gitlab-ci-token --password $CI_JOB_TOKEN $CI_REGISTRY";
-export const createDockerBuildJobDefault = (
-  context: Context,
-  { script, ...def }: Partial<CatladderJob>
-): CatladderJob => {
-  return createDockerBuildJobBase(context, {
-    script: [
-      ...(script || []),
-      gitlabDockerLogin,
-      "docker version",
-      "docker build --network host --cache-from $DOCKER_CACHE_IMAGE --tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG -f $APP_DIR/Dockerfile . --build-arg BUILDKIT_INLINE_CACHE=1", //BUILDKIT_INLINE_CACHE,  see https://testdriven.io/blog/faster-ci-builds-with-docker-cache/
-      "docker push $DOCKER_IMAGE:$DOCKER_IMAGE_TAG",
-      "docker tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG $DOCKER_CACHE_IMAGE",
-      "docker push $DOCKER_CACHE_IMAGE",
-    ],
-    ...def,
-  });
-};
+
+export const getDockerBuildDefaultScript = (ensureDockerFileScript?: string) =>
+  [
+    ensureDockerFileScript,
+    gitlabDockerLogin,
+    "docker version",
+    "docker build --network host --cache-from $DOCKER_CACHE_IMAGE --tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG -f $APP_DIR/Dockerfile . --build-arg BUILDKIT_INLINE_CACHE=1", //BUILDKIT_INLINE_CACHE,  see https://testdriven.io/blog/faster-ci-builds-with-docker-cache/
+    "docker push $DOCKER_IMAGE:$DOCKER_IMAGE_TAG",
+    "docker tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG $DOCKER_CACHE_IMAGE",
+    "docker push $DOCKER_CACHE_IMAGE",
+  ].filter(Boolean);

package/src/build/node/buildJob.ts

@@ -1,17 +1,14 @@
+import { join } from "path";
+import { getRunnerImage } from "../../runner";
 import type { Context } from "../../types/context";
+import type { CatladderJob } from "../../types/jobs";
 import { ensureArray } from "../../utils";
-import { APP_BUILD_JOB_NAME } from "../base/constants";
-import { createBuildJob } from "../base/createBuildJob";
-import { createDockerBuildJobDefault, requiresDockerBuild } from "../docker";
-import { join } from "path";
+import { createBuildJobs } from "../base";
+import { getDockerBuildDefaultScript, requiresDockerBuild } from "../docker";
 import { isOfBuildType } from "../types";
 import { getNextCache, getNodeCache, getYarnCache } from "./cache";
 import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
 import { getDockerAppCopyAndBuildScript, getYarnInstall } from "./yarn";
-import type { CatladderJob } from "../../types/jobs";
-import { getRunnerImage } from "../../runner";
-import { createSbomBuildJob } from "../sbom";
-import { sbomDeactivated } from "../../deploy/sbom";
 
 export const createNodeBuildJobs = (context: Context): CatladderJob[] => {
   const buildConfig = context.componentConfig.build;
@@ -22,62 +19,54 @@ export const createNodeBuildJobs = (context: Context): CatladderJob[] => {
 
   const defaultImage = getRunnerImage("jobs-default");
   const yarnInstall = getYarnInstall(context);
-  const appBuildJob: CatladderJob | null =
-    buildConfig.buildCommand !== null
-      ? createBuildJob(context, {
-          image: buildConfig.jobImage ?? defaultImage,
-          variables: {
-            ...NODE_RUNNER_BUILD_VARIABLES,
-          },
-          cache: [
-            ...(ensureArray(buildConfig.jobCache) ?? []),
-            ...getNodeCache(context),
-            ...getNextCache(context),
-          ],
-          script: [
-            ...yarnInstall,
-            ...(ensureArray(buildConfig.buildCommand) ?? []),
-          ],
-          artifacts: {
-            paths: [
-              join(context.componentConfig.dir, "__build_info.json"),
-              join(context.componentConfig.dir, "dist"),
-              join(context.componentConfig.dir, ".next"),
-              ...(buildConfig.artifactsPaths?.map((path) =>
-                join(context.componentConfig.dir, path)
-              ) ?? []),
+
+  return createBuildJobs(context, {
+    appBuild:
+      buildConfig.buildCommand !== null
+        ? {
+            image: buildConfig.jobImage ?? defaultImage,
+            variables: {
+              ...NODE_RUNNER_BUILD_VARIABLES,
+            },
+            cache: [
+              ...(ensureArray(buildConfig.jobCache) ?? []),
+              ...getNodeCache(context),
+              ...getNextCache(context),
             ],
-            expire_in: "1 day",
-          },
-          jobTags: buildConfig.jobTags,
-        })
-      : null;
-  return [
-    ...(appBuildJob ? [appBuildJob] : []),
-    ...(requiresDockerBuild(context)
-      ? [
-          createDockerBuildJobDefault(context, {
             script: [
-              buildConfig.type === "node-static" ||
-              buildConfig.type === "storybook"
-                ? "ensureNginxDockerfile"
-                : "ensureNodeDockerfile",
+              ...yarnInstall,
+              ...(ensureArray(buildConfig.buildCommand) ?? []),
             ],
-            cache: [...getYarnCache(context, "pull")],
-            variables: {
-              // only required for non static
-              DOCKER_COPY_AND_INSTALL_APP:
-                getDockerAppCopyAndBuildScript(context),
-              DOCKER_COPY_WORKSPACE_FILES:
-                context.packageManagerInfo?.pathsToCopyInDocker
-                  .map((dir) => `COPY --chown=node:node ${dir} /app/${dir}`)
-                  ?.join("\n"),
+            artifacts: {
+              paths: [
+                join(context.componentConfig.dir, "__build_info.json"),
+                join(context.componentConfig.dir, "dist"),
+                join(context.componentConfig.dir, ".next"),
+                ...(buildConfig.artifactsPaths?.map((path) =>
+                  join(context.componentConfig.dir, path)
+                ) ?? []),
+              ],
+              expire_in: "1 day",
             },
+            jobTags: buildConfig.jobTags,
+          }
+        : undefined,
 
-            needs: appBuildJob ? [APP_BUILD_JOB_NAME] : [],
-          }),
-        ]
-      : []),
-    ...(sbomDeactivated(context) ? [] : [createSbomBuildJob(context)]),
-  ];
+    dockerBuild: {
+      script: getDockerBuildDefaultScript(
+        buildConfig.type === "node-static" || buildConfig.type === "storybook"
+          ? "ensureNginxDockerfile"
+          : "ensureNodeDockerfile"
+      ),
+      cache: [...getYarnCache(context, "pull")],
+      variables: {
+        // only required for non static
+        DOCKER_COPY_AND_INSTALL_APP: getDockerAppCopyAndBuildScript(context),
+        DOCKER_COPY_WORKSPACE_FILES:
+          context.packageManagerInfo?.pathsToCopyInDocker
+            .map((dir) => `COPY --chown=node:node ${dir} /app/${dir}`)
+            ?.join("\n"),
+      },
+    },
+  });
 };

package/src/build/node/meteor.ts

@@ -5,14 +5,11 @@ import type { Context } from "../../types/context";
 
 import type { CatladderJob } from "../../types/jobs";
 
-import { APP_BUILD_JOB_NAME } from "../base/constants";
-import { createBuildJob } from "../base/createBuildJob";
-import { createDockerBuildJobDefault } from "../docker";
+import { createBuildJobs } from "../base";
+import { getDockerBuildDefaultScript } from "../docker";
 import { isOfBuildType } from "../types";
 import { getNodeCache } from "./cache";
 import { getYarnInstall } from "./yarn";
-import { sbomDeactivated } from "../../deploy/sbom";
-import { createSbomBuildJob } from "../sbom";
 
 const getMeteorCache = (context: Context): GitlabJobCache[] => [
   {
@@ -37,42 +34,40 @@ export const createMeteorBuildJobs = (context: Context): CatladderJob[] => {
   }
 
   const yarnInstall = getYarnInstall(context);
-  const appBuildJob: CatladderJob | null =
-    buildConfig.buildCommand !== null
-      ? createBuildJob(context, {
-          cache: [...getNodeCache(context), ...getMeteorCache(context)],
-          image: getRunnerImage("jobs-meteor"),
-          variables: {
-            METEOR_DISABLE_OPTIMISTIC_CACHING: "1", // see https://forums.meteor.com/t/veeery-long-building-time-inside-docker-container/58673/17?u=macrozone
-          },
-          script: [
-            ...yarnInstall,
 
-            'echo "add healthcheck package"',
-            "meteor add panter:healthroute --allow-superuser",
-            "meteor add qualia:prod-shell --allow-superuser",
+  return createBuildJobs(context, {
+    appBuild:
+      buildConfig.buildCommand !== null
+        ? {
+            cache: [...getNodeCache(context), ...getMeteorCache(context)],
+            image: getRunnerImage("jobs-meteor"),
+            variables: {
+              METEOR_DISABLE_OPTIMISTIC_CACHING: "1", // see https://forums.meteor.com/t/veeery-long-building-time-inside-docker-container/58673/17?u=macrozone
+            },
+            script: [
+              ...yarnInstall,
 
-            'TOOL_NODE_FLAGS="--max_old_space_size=3584 --min_semi_space_size=8 --max_semi_space_size=256 --optimize_for_size" meteor build ./dist --architecture os.linux.x86_64 --allow-superuser --server-only --directory',
+              'echo "add healthcheck package"',
+              "meteor add panter:healthroute --allow-superuser",
+              "meteor add qualia:prod-shell --allow-superuser",
 
-            "cp ./__build_info.json ./dist/bundle/programs/server",
-          ],
-          artifacts: {
-            paths: [
-              context.componentConfig.dir + "/__build_info.json",
-              context.componentConfig.dir + "/dist",
+              'TOOL_NODE_FLAGS="--max_old_space_size=3584 --min_semi_space_size=8 --max_semi_space_size=256 --optimize_for_size" meteor build ./dist --architecture os.linux.x86_64 --allow-superuser --server-only --directory',
+
+              "cp ./__build_info.json ./dist/bundle/programs/server",
             ],
-          },
-        })
-      : null;
-  return [
-    ...(appBuildJob ? [appBuildJob] : []),
-    createDockerBuildJobDefault(context, {
-      script: ["ensureMeteorDockerfile"],
+            artifacts: {
+              paths: [
+                context.componentConfig.dir + "/__build_info.json",
+                context.componentConfig.dir + "/dist",
+              ],
+            },
+          }
+        : undefined,
+    dockerBuild: {
+      script: getDockerBuildDefaultScript("ensureMeteorDockerfile"),
       variables: {
         METEOR_INSTALL_SCRIPTS: buildConfig.installScripts ? "true" : "",
       },
-      needs: appBuildJob ? [APP_BUILD_JOB_NAME] : [],
-    }),
-    ...(sbomDeactivated(context) ? [] : [createSbomBuildJob(context)]),
-  ];
+    },
+  });
 };

package/src/build/rails/build.ts

@@ -1,9 +1,8 @@
 import type { Context } from "../..";
-import { isOfBuildType } from "../types";
 import type { CatladderJob } from "../../types/jobs";
-import { createDockerBuildJobBase, gitlabDockerLogin } from "../docker";
-import { sbomDeactivated } from "../../deploy/sbom";
-import { createSbomBuildJob } from "../sbom";
+import { createBuildJobs } from "../base";
+import { gitlabDockerLogin } from "../docker";
+import { isOfBuildType } from "../types";
 
 export const createRailsBuildJobs = (context: Context): CatladderJob[] => {
   const buildConfig = context.componentConfig.build;
@@ -17,9 +16,9 @@ export const createRailsBuildJobs = (context: Context): CatladderJob[] => {
   const packEnvArgs = Object.entries(cnbConf?.buildVars ?? {})
     .map(([k, v]) => `--env '${k}${v ? `=${v}` : ""}'`)
     .join(" ");
-
-  return [
-    createDockerBuildJobBase(context, {
+  return createBuildJobs(context, {
+    appBuild: undefined,
+    dockerBuild: {
       variables: {
         ...context.environment.jobOnlyVars.build.envVars,
         ...context.componentConfig.build.extraVars,
@@ -38,7 +37,6 @@ export const createRailsBuildJobs = (context: Context): CatladderJob[] => {
           cnbConf?.packExtraArgs?.join(" ") ?? ""
         }`,
       ],
-    }),
-    ...(sbomDeactivated(context) ? [] : [createSbomBuildJob(context)]),
-  ];
+    },
+  });
 };

package/src/build/types.ts

@@ -1,9 +1,5 @@
-import type {
-  Artifacts,
-  EnvVars,
-  GitlabJobImage,
-  GitlabJobService,
-} from "../types";
+import type { Artifacts, EnvVars, GitlabJobImage } from "../types";
+import type { Services } from "../types/gitlab-ci-yml";
 
 import type { CatladderJob } from "../types/jobs";
 
@@ -168,9 +164,9 @@ export type BuildConfigCustom = Omit<
   type: "custom";
   jobImage: GitlabJobImage;
   /**
-   * {@link GitlabJobService}s used in lint, test, audit, and build jobs.
+   * {@link Services}s used in lint, test, audit, and build jobs.
    */
-  jobServices?: GitlabJobService[];
+  jobServices?: Services;
 
   docker: BuildConfigCustomDocker;
 

package/src/deploy/base/deploy.ts

@@ -4,15 +4,23 @@ import {
 } from "../../build/docker";
 import { SBOM_BUILD_JOB_NAME } from "../../build/sbom";
 import type { Context } from "../../types/context";
+import type { CatladderJob } from "../../types/jobs";
 import { sbomDeactivated } from "../sbom";
 import { contextIsStoppable } from "../utils";
 import { STOP_JOB_NAME } from "./stop";
-import type { JobWithoutScript } from "./types";
+
 import { DEPLOY_RUNNER_VARIABLES } from "./variables";
 
 export const DEPLOY_JOB_NAME = "🚀 Deploy";
 
-export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
+export type DeployJobDefinition = Pick<
+  CatladderJob,
+  "script" | "variables" | "image" | "cache" | "artifacts" | "services"
+>;
+export const createDeployJob = (
+  context: Context,
+  jobDefinition: DeployJobDefinition
+): CatladderJob => {
   const hasDocker = requiresDockerBuild(context);
   const isStoppable = contextIsStoppable(context);
 
@@ -39,6 +47,11 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
 
   return {
     name: DEPLOY_JOB_NAME,
+    script: jobDefinition.script,
+    image: jobDefinition.image,
+    cache: jobDefinition.cache,
+    artifacts: jobDefinition.artifacts,
+    services: jobDefinition.services,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
 
     needs: [
@@ -83,6 +96,7 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
       ...(context.componentConfig.deploy
         ? context.componentConfig.deploy.extraVars ?? {}
         : {}),
+      ...jobDefinition.variables,
     },
     environment: {
       ...context.environment.gitlabEnvironment,

package/src/deploy/base/index.ts

@@ -1,3 +1,22 @@
-export * from "./deploy";
-export * from "./rollback";
-export * from "./stop";
+import type { Context } from "../../types/context";
+import type { CatladderJob } from "../../types/jobs";
+import { createDeployJob, type DeployJobDefinition } from "./deploy";
+import { createRollbackJob, type RollbackJobDefinition } from "./rollback";
+import { createStopJob, type StopJobDefinition } from "./stop";
+
+export const createDeployementJobs = (
+  context: Context,
+  definitions: {
+    deploy: DeployJobDefinition;
+    stop?: StopJobDefinition;
+    rollback?: RollbackJobDefinition;
+  }
+): CatladderJob[] => {
+  return [
+    createDeployJob(context, definitions.deploy),
+    ...(definitions.stop ? [createStopJob(context, definitions.stop)] : []),
+    ...(definitions.rollback
+      ? [createRollbackJob(context, definitions.rollback)]
+      : []),
+  ];
+};

package/src/deploy/base/rollback.ts

@@ -1,12 +1,20 @@
 import type { Context } from "../../types/context";
-import type { JobWithoutScript } from "./types";
+import type { CatladderJob } from "../../types/jobs";
 import { DEPLOY_RUNNER_VARIABLES } from "./variables";
 
 export const ROLLBACK_JOB_NAME = "↩️ Rollback ⚠️";
 
-export const getBaseRollbackJob = (context: Context): JobWithoutScript => {
+export type RollbackJobDefinition = Pick<
+  CatladderJob,
+  "script" | "variables" | "image"
+>;
+export const createRollbackJob = (
+  context: Context,
+  jobDefinition: RollbackJobDefinition
+): CatladderJob => {
   return {
     name: ROLLBACK_JOB_NAME,
+    image: jobDefinition.image,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
     needs: [], // can be executed even if the deploy job failed
     rules: [
@@ -19,11 +27,14 @@ export const getBaseRollbackJob = (context: Context): JobWithoutScript => {
       ...DEPLOY_RUNNER_VARIABLES,
       ...context.environment.jobOnlyVars.deploy.envVars,
       GIT_STRATEGY: "none",
+      ...jobDefinition.variables,
     },
     stage: "rollback",
     environment: {
       ...context.environment.gitlabEnvironment,
       action: "access",
     },
+
+    script: jobDefinition.script,
   };
 };

package/src/deploy/base/stop.ts

@@ -1,13 +1,20 @@
 import type { Context } from "../../types/context";
-import type { JobWithoutScript } from "./types";
+import type { CatladderJob } from "../../types/jobs";
+
 import { DEPLOY_RUNNER_VARIABLES } from "./variables";
 export const STOP_JOB_NAME = "🛑 Stop ⚠️";
 
-export const getBaseDeploymentStopJob = (
-  context: Context
-): JobWithoutScript => {
+export type StopJobDefinition = Pick<
+  CatladderJob,
+  "script" | "variables" | "image"
+>;
+export const createStopJob = (
+  context: Context,
+  jobDefinition: StopJobDefinition
+): CatladderJob => {
   return {
     name: STOP_JOB_NAME,
+    image: jobDefinition.image,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
     needs: [], // can be executed even if the deploy job failed
     rules: [
@@ -25,11 +32,13 @@ export const getBaseDeploymentStopJob = (
       ...DEPLOY_RUNNER_VARIABLES,
       ...context.environment.jobOnlyVars.deploy.envVars,
       GIT_STRATEGY: "none",
+      ...jobDefinition.variables,
     },
     stage: "stop",
     environment: {
       ...context.environment.gitlabEnvironment,
       action: "stop",
     },
+    script: jobDefinition.script,
   };
 };