@catladder/pipeline 1.111.5 → 1.113.0

package/src/build/node/meteor.ts

@@ -5,14 +5,11 @@ import type { Context } from "../../types/context";
 
 import type { CatladderJob } from "../../types/jobs";
 
-import { APP_BUILD_JOB_NAME } from "../base/constants";
-import { createBuildJob } from "../base/createBuildJob";
-import { createDockerBuildJobDefault } from "../docker";
+import { createBuildJobs } from "../base";
+import { getDockerBuildDefaultScript } from "../docker";
 import { isOfBuildType } from "../types";
 import { getNodeCache } from "./cache";
 import { getYarnInstall } from "./yarn";
-import { sbomDeactivated } from "../../deploy/sbom";
-import { createSbomBuildJob } from "../sbom";
 
 const getMeteorCache = (context: Context): GitlabJobCache[] => [
   {
@@ -37,42 +34,40 @@ export const createMeteorBuildJobs = (context: Context): CatladderJob[] => {
   }
 
   const yarnInstall = getYarnInstall(context);
-  const appBuildJob: CatladderJob | null =
-    buildConfig.buildCommand !== null
-      ? createBuildJob(context, {
-          cache: [...getNodeCache(context), ...getMeteorCache(context)],
-          image: getRunnerImage("jobs-meteor"),
-          variables: {
-            METEOR_DISABLE_OPTIMISTIC_CACHING: "1", // see https://forums.meteor.com/t/veeery-long-building-time-inside-docker-container/58673/17?u=macrozone
-          },
-          script: [
-            ...yarnInstall,
 
-            'echo "add healthcheck package"',
-            "meteor add panter:healthroute --allow-superuser",
-            "meteor add qualia:prod-shell --allow-superuser",
+  return createBuildJobs(context, {
+    appBuild:
+      buildConfig.buildCommand !== null
+        ? {
+            cache: [...getNodeCache(context), ...getMeteorCache(context)],
+            image: getRunnerImage("jobs-meteor"),
+            variables: {
+              METEOR_DISABLE_OPTIMISTIC_CACHING: "1", // see https://forums.meteor.com/t/veeery-long-building-time-inside-docker-container/58673/17?u=macrozone
+            },
+            script: [
+              ...yarnInstall,
 
-            'TOOL_NODE_FLAGS="--max_old_space_size=3584 --min_semi_space_size=8 --max_semi_space_size=256 --optimize_for_size" meteor build ./dist --architecture os.linux.x86_64 --allow-superuser --server-only --directory',
+              'echo "add healthcheck package"',
+              "meteor add panter:healthroute --allow-superuser",
+              "meteor add qualia:prod-shell --allow-superuser",
 
-            "cp ./__build_info.json ./dist/bundle/programs/server",
-          ],
-          artifacts: {
-            paths: [
-              context.componentConfig.dir + "/__build_info.json",
-              context.componentConfig.dir + "/dist",
+              'TOOL_NODE_FLAGS="--max_old_space_size=3584 --min_semi_space_size=8 --max_semi_space_size=256 --optimize_for_size" meteor build ./dist --architecture os.linux.x86_64 --allow-superuser --server-only --directory',
+
+              "cp ./__build_info.json ./dist/bundle/programs/server",
             ],
-          },
-        })
-      : null;
-  return [
-    ...(appBuildJob ? [appBuildJob] : []),
-    createDockerBuildJobDefault(context, {
-      script: ["ensureMeteorDockerfile"],
+            artifacts: {
+              paths: [
+                context.componentConfig.dir + "/__build_info.json",
+                context.componentConfig.dir + "/dist",
+              ],
+            },
+          }
+        : undefined,
+    dockerBuild: {
+      script: getDockerBuildDefaultScript("ensureMeteorDockerfile"),
       variables: {
         METEOR_INSTALL_SCRIPTS: buildConfig.installScripts ? "true" : "",
       },
-      needs: appBuildJob ? [APP_BUILD_JOB_NAME] : [],
-    }),
-    ...(sbomDeactivated(context) ? [] : [createSbomBuildJob(context)]),
-  ];
+    },
+  });
 };

package/src/build/rails/build.ts

@@ -1,9 +1,8 @@
 import type { Context } from "../..";
-import { isOfBuildType } from "../types";
 import type { CatladderJob } from "../../types/jobs";
-import { createDockerBuildJobBase, gitlabDockerLogin } from "../docker";
-import { sbomDeactivated } from "../../deploy/sbom";
-import { createSbomBuildJob } from "../sbom";
+import { createBuildJobs } from "../base";
+import { gitlabDockerLogin } from "../docker";
+import { isOfBuildType } from "../types";
 
 export const createRailsBuildJobs = (context: Context): CatladderJob[] => {
   const buildConfig = context.componentConfig.build;
@@ -17,9 +16,9 @@ export const createRailsBuildJobs = (context: Context): CatladderJob[] => {
   const packEnvArgs = Object.entries(cnbConf?.buildVars ?? {})
     .map(([k, v]) => `--env '${k}${v ? `=${v}` : ""}'`)
     .join(" ");
-
-  return [
-    createDockerBuildJobBase(context, {
+  return createBuildJobs(context, {
+    appBuild: undefined,
+    dockerBuild: {
       variables: {
         ...context.environment.jobOnlyVars.build.envVars,
         ...context.componentConfig.build.extraVars,
@@ -38,7 +37,6 @@ export const createRailsBuildJobs = (context: Context): CatladderJob[] => {
           cnbConf?.packExtraArgs?.join(" ") ?? ""
         }`,
       ],
-    }),
-    ...(sbomDeactivated(context) ? [] : [createSbomBuildJob(context)]),
-  ];
+    },
+  });
 };

package/src/build/types.ts

@@ -1,9 +1,5 @@
-import type {
-  Artifacts,
-  EnvVars,
-  GitlabJobImage,
-  GitlabJobService,
-} from "../types";
+import type { Artifacts, EnvVars, GitlabJobImage } from "../types";
+import type { Services } from "../types/gitlab-ci-yml";
 
 import type { CatladderJob } from "../types/jobs";
 
@@ -168,9 +164,9 @@ export type BuildConfigCustom = Omit<
   type: "custom";
   jobImage: GitlabJobImage;
   /**
-   * {@link GitlabJobService}s used in lint, test, audit, and build jobs.
+   * {@link Services}s used in lint, test, audit, and build jobs.
    */
-  jobServices?: GitlabJobService[];
+  jobServices?: Services;
 
   docker: BuildConfigCustomDocker;
 

package/src/context/getLabels.ts

@@ -10,6 +10,7 @@ const sanitize = (value?: string) => {
 export const getLabels = (context: Context) => {
   const labels = {
     "customer-name": sanitize(context.fullConfig.customerName),
+    "component-name": sanitize(context.componentName),
     "app-name": sanitize(context.fullConfig.appName),
     "env-type": sanitize(context.environment.envType),
     "env-name": sanitize(context.environment.shortName),

package/src/deploy/base/deploy.ts

@@ -4,15 +4,23 @@ import {
 } from "../../build/docker";
 import { SBOM_BUILD_JOB_NAME } from "../../build/sbom";
 import type { Context } from "../../types/context";
+import type { CatladderJob } from "../../types/jobs";
 import { sbomDeactivated } from "../sbom";
 import { contextIsStoppable } from "../utils";
 import { STOP_JOB_NAME } from "./stop";
-import type { JobWithoutScript } from "./types";
+
 import { DEPLOY_RUNNER_VARIABLES } from "./variables";
 
 export const DEPLOY_JOB_NAME = "🚀 Deploy";
 
-export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
+export type DeployJobDefinition = Pick<
+  CatladderJob,
+  "script" | "variables" | "image" | "cache" | "artifacts" | "services"
+>;
+export const createDeployJob = (
+  context: Context,
+  jobDefinition: DeployJobDefinition
+): CatladderJob => {
   const hasDocker = requiresDockerBuild(context);
   const isStoppable = contextIsStoppable(context);
 
@@ -39,6 +47,11 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
 
   return {
     name: DEPLOY_JOB_NAME,
+    script: jobDefinition.script,
+    image: jobDefinition.image,
+    cache: jobDefinition.cache,
+    artifacts: jobDefinition.artifacts,
+    services: jobDefinition.services,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
 
     needs: [
@@ -83,6 +96,7 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
       ...(context.componentConfig.deploy
         ? context.componentConfig.deploy.extraVars ?? {}
         : {}),
+      ...jobDefinition.variables,
     },
     environment: {
       ...context.environment.gitlabEnvironment,

package/src/deploy/base/index.ts

@@ -1,3 +1,22 @@
-export * from "./deploy";
-export * from "./rollback";
-export * from "./stop";
+import type { Context } from "../../types/context";
+import type { CatladderJob } from "../../types/jobs";
+import { createDeployJob, type DeployJobDefinition } from "./deploy";
+import { createRollbackJob, type RollbackJobDefinition } from "./rollback";
+import { createStopJob, type StopJobDefinition } from "./stop";
+
+export const createDeployementJobs = (
+  context: Context,
+  definitions: {
+    deploy: DeployJobDefinition;
+    stop?: StopJobDefinition;
+    rollback?: RollbackJobDefinition;
+  }
+): CatladderJob[] => {
+  return [
+    createDeployJob(context, definitions.deploy),
+    ...(definitions.stop ? [createStopJob(context, definitions.stop)] : []),
+    ...(definitions.rollback
+      ? [createRollbackJob(context, definitions.rollback)]
+      : []),
+  ];
+};

package/src/deploy/base/rollback.ts

@@ -1,12 +1,20 @@
 import type { Context } from "../../types/context";
-import type { JobWithoutScript } from "./types";
+import type { CatladderJob } from "../../types/jobs";
 import { DEPLOY_RUNNER_VARIABLES } from "./variables";
 
 export const ROLLBACK_JOB_NAME = "↩️ Rollback ⚠️";
 
-export const getBaseRollbackJob = (context: Context): JobWithoutScript => {
+export type RollbackJobDefinition = Pick<
+  CatladderJob,
+  "script" | "variables" | "image"
+>;
+export const createRollbackJob = (
+  context: Context,
+  jobDefinition: RollbackJobDefinition
+): CatladderJob => {
   return {
     name: ROLLBACK_JOB_NAME,
+    image: jobDefinition.image,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
     needs: [], // can be executed even if the deploy job failed
     rules: [
@@ -19,11 +27,14 @@ export const getBaseRollbackJob = (context: Context): JobWithoutScript => {
       ...DEPLOY_RUNNER_VARIABLES,
       ...context.environment.jobOnlyVars.deploy.envVars,
       GIT_STRATEGY: "none",
+      ...jobDefinition.variables,
     },
     stage: "rollback",
     environment: {
       ...context.environment.gitlabEnvironment,
       action: "access",
     },
+
+    script: jobDefinition.script,
   };
 };

package/src/deploy/base/stop.ts

@@ -1,13 +1,20 @@
 import type { Context } from "../../types/context";
-import type { JobWithoutScript } from "./types";
+import type { CatladderJob } from "../../types/jobs";
+
 import { DEPLOY_RUNNER_VARIABLES } from "./variables";
 export const STOP_JOB_NAME = "🛑 Stop ⚠️";
 
-export const getBaseDeploymentStopJob = (
-  context: Context
-): JobWithoutScript => {
+export type StopJobDefinition = Pick<
+  CatladderJob,
+  "script" | "variables" | "image"
+>;
+export const createStopJob = (
+  context: Context,
+  jobDefinition: StopJobDefinition
+): CatladderJob => {
   return {
     name: STOP_JOB_NAME,
+    image: jobDefinition.image,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
     needs: [], // can be executed even if the deploy job failed
     rules: [
@@ -25,11 +32,13 @@ export const getBaseDeploymentStopJob = (
       ...DEPLOY_RUNNER_VARIABLES,
       ...context.environment.jobOnlyVars.deploy.envVars,
       GIT_STRATEGY: "none",
+      ...jobDefinition.variables,
     },
     stage: "stop",
     environment: {
       ...context.environment.gitlabEnvironment,
       action: "stop",
     },
+    script: jobDefinition.script,
   };
 };

package/src/deploy/cloudRun/deployJob.ts

@@ -6,7 +6,12 @@ import { getLabels } from "../../context/getLabels";
 import { getRunnerImage } from "../../runner";
 import type { Context } from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
-import { getBaseDeploymentJob, getBaseDeploymentStopJob } from "../base";
+import { allowFailureInScripts } from "../../utils/gitlab";
+import { createDeployementJobs } from "../base";
+import {
+  getDependencyTrackDeleteScript,
+  getDependencyTrackUploadScript,
+} from "../sbom";
 import type {
   DeployConfigCloudRun,
   DeployConfigCloudRunJob,
@@ -14,18 +19,13 @@ import type {
   DeployConfigCloudRunService,
 } from "../types";
 import { isOfDeployType } from "../types";
-import { gcloudServiceAccountLoginCommands } from "./utils/gcloudServiceAccountLoginCommands";
+import { createArgsString } from "./utils/createArgsString";
 import {
   getDatabaseCreateScript,
   getDatabaseDeleteScript,
 } from "./utils/database";
-import { allowFailureInScripts } from "../../utils/gitlab";
+import { gcloudServiceAccountLoginCommands } from "./utils/gcloudServiceAccountLoginCommands";
 import { getCloudRunJobName } from "./utils/jobName";
-import { createArgsString } from "./utils/createArgsString";
-import {
-  getDependencyTrackDeleteScript,
-  getDependencyTrackUploadScript,
-} from "../sbom";
 
 const setExtraVarsScripts = (deployConfig: DeployConfigCloudRun) => [
   `export GCLOUD_PROJECT_NUMBER=$(gcloud projects describe ${deployConfig.projectId} --format="value(projectNumber)")`,
@@ -43,7 +43,6 @@ export const createGoogleCloudRunDeployJobs = (
     // should not happen
     throw new Error("deploy config is wrong");
   }
-  const baseDeploymentJob = getBaseDeploymentJob(context);
 
   const fullAppName = `${context.fullConfig.customerName}-${context.fullConfig.appName}`;
   const dockerUrl = `${deployConfig.region}-docker.pkg.dev/${deployConfig.projectId}/catladder-deploy/${fullAppName}`;
@@ -257,8 +256,8 @@ export const createGoogleCloudRunDeployJobs = (
       : []),
     ...getDependencyTrackDeleteScript(context),
   ];
-  return [
-    merge({}, baseDeploymentJob, getDockerJobBaseProps(context), {
+  return createDeployementJobs(context, {
+    deploy: merge(getDockerJobBaseProps(context), {
       artifacts: { paths: ["____envvars.yaml"] },
       variables: {
         CLOUDSDK_CORE_DISABLE_PROMPTS: "1",
@@ -271,13 +270,12 @@ export const createGoogleCloudRunDeployJobs = (
       image: getRunnerImage("gcloud"),
       script: deployScripts,
     }),
-
-    merge({}, getBaseDeploymentStopJob(context), {
+    stop: {
       image: getRunnerImage("gcloud"),
       variables: {
         CLOUDSDK_CORE_DISABLE_PROMPTS: "1",
       },
       script: allowFailureInScripts(stopScripts),
-    }),
-  ];
+    },
+  });
 };

package/src/deploy/custom/deployJob.ts

@@ -1,15 +1,13 @@
-import { merge } from "lodash";
-
 import { getYarnInstall } from "../../build/node/yarn";
 import { getRunnerImage } from "../../runner";
 import type { Context } from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
-import { getBaseDeploymentJob, getBaseDeploymentStopJob } from "../base";
-import { isOfDeployType } from "../types";
+import { createDeployementJobs } from "../base";
 import {
   getDependencyTrackDeleteScript,
   getDependencyTrackUploadScript,
 } from "../sbom";
+import { isOfDeployType } from "../types";
 
 export const createCustomDeployJobs = (context: Context): CatladderJob[] => {
   const deployConfig = context.componentConfig.deploy;
@@ -20,11 +18,10 @@ export const createCustomDeployJobs = (context: Context): CatladderJob[] => {
     // should not happen
     throw new Error("deploy config is not custom");
   }
-  const baseDeploymentJob = getBaseDeploymentJob(context);
   // FIXME: custom deploy currently assumes yarn-based project
   const yarnInstall = getYarnInstall(context, { noCustomPostInstall: true });
-  return [
-    merge({}, baseDeploymentJob, {
+  return createDeployementJobs(context, {
+    deploy: {
       image: deployConfig.jobImage ?? getRunnerImage("jobs-default"),
       cache: deployConfig.jobCache ?? [],
       script: [
@@ -33,17 +30,17 @@ export const createCustomDeployJobs = (context: Context): CatladderJob[] => {
         ...deployConfig.script,
         ...getDependencyTrackUploadScript(context),
       ],
-    }),
-    ...(deployConfig.stopScript
-      ? [
-          merge({}, getBaseDeploymentStopJob(context), {
-            image: deployConfig.jobImage ?? getRunnerImage("jobs-default"),
-            script: [
-              ...deployConfig.stopScript,
-              ...getDependencyTrackDeleteScript(context),
-            ],
-          }),
-        ]
-      : []),
-  ];
+      variables: {},
+    },
+    stop: deployConfig.stopScript
+      ? {
+          image: deployConfig.jobImage ?? getRunnerImage("jobs-default"),
+          script: [
+            ...deployConfig.stopScript,
+            ...getDependencyTrackDeleteScript(context),
+          ],
+          variables: {},
+        }
+      : undefined,
+  });
 };

package/src/deploy/dockerTag/deployJob.ts

@@ -1,9 +1,8 @@
-import { merge } from "lodash";
 import { getDockerJobBaseProps, gitlabDockerLogin } from "../../build/docker";
 
 import type { Context } from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
-import { getBaseDeploymentJob } from "../base";
+import { createDeployementJobs } from "../base";
 import { isOfDeployType } from "../types";
 
 export const createDockerTagDeployJobs = (context: Context): CatladderJob[] => {
@@ -15,10 +14,10 @@ export const createDockerTagDeployJobs = (context: Context): CatladderJob[] => {
     // should not happen
     throw new Error("deploy config is not dockerTag");
   }
-  const baseDeploymentJob = getBaseDeploymentJob(context);
   const tag = deployConfig.tag;
-  return [
-    merge({}, baseDeploymentJob, getDockerJobBaseProps(context), {
+  return createDeployementJobs(context, {
+    deploy: {
+      ...getDockerJobBaseProps(context),
       script: [
         gitlabDockerLogin,
         `docker pull $DOCKER_IMAGE:$DOCKER_IMAGE_TAG`,
@@ -26,6 +25,6 @@ export const createDockerTagDeployJobs = (context: Context): CatladderJob[] => {
         `docker push $DOCKER_IMAGE:${tag}`,
         `echo "pushed as $DOCKER_IMAGE:${tag}"`,
       ],
-    }),
-  ];
+    },
+  });
 };

package/src/deploy/kubernetes/deployJob.ts

@@ -1,20 +1,15 @@
 import { dump } from "js-yaml";
-import { merge } from "lodash";
-import { getSecretVarNameForContext } from "../..";
 import { getRunnerImage } from "../../runner";
 import type { Context } from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
-import {
-  getBaseDeploymentJob,
-  getBaseDeploymentStopJob,
-  getBaseRollbackJob,
-} from "../base";
-import { isOfDeployType } from "../types";
-import { createKubeValues } from "./kubeValues";
+import { createDeployementJobs } from "../base";
 import {
   getDependencyTrackDeleteScript,
   getDependencyTrackUploadScript,
 } from "../sbom";
+import { isOfDeployType } from "../types";
+import { createKubeValues } from "./kubeValues";
+import { getSecretVarNameForContext } from "../../context/getEnvironmentVariables";
 
 export const createKubernetesDeployJobs = (
   context: Context
@@ -29,8 +24,7 @@ export const createKubernetesDeployJobs = (
   }
 
   const kubeValues = createKubeValues(context);
-
-  const shared = {
+  const shared: Pick<CatladderJob, "image" | "variables"> = {
     image: getRunnerImage("kubernetes"),
     variables: {
       ...context.environment.envVars,
@@ -53,9 +47,6 @@ export const createKubernetesDeployJobs = (
     },
   };
 
-  const baseDeploymentJob = getBaseDeploymentJob(context);
-  const baseStopJob = getBaseDeploymentStopJob(context);
-  const baseRollbackJob = getBaseRollbackJob(context);
   const clusterName = `kube-${context.environment.fullName}`;
   const connectContext = [
     `kubectl config set-cluster "${clusterName}" --server="$${getSecretVarNameForContext(
@@ -72,8 +63,10 @@ export const createKubernetesDeployJobs = (
     `kubectl config set-context "${clusterName}" --cluster="${clusterName}" --user="${clusterName}" --namespace="${context.environment.envVars.KUBE_NAMESPACE}"`,
     `kubectl config use-context "${clusterName}"`,
   ];
-  return [
-    merge({}, baseDeploymentJob, shared, {
+
+  return createDeployementJobs(context, {
+    deploy: {
+      ...shared,
       script: [
         ...connectContext,
         "kubernetesCreateSecret",
@@ -81,17 +74,18 @@ export const createKubernetesDeployJobs = (
         ...getDependencyTrackUploadScript(context),
         "echo deployment successful 😻",
       ],
-    }),
-    merge({}, baseStopJob, shared, {
+    },
+    stop: {
+      ...shared,
       script: [
         ...connectContext,
         "kubernetesDelete",
         ...getDependencyTrackDeleteScript(context),
       ],
-    }),
-
-    merge({}, baseRollbackJob, shared, {
+    },
+    rollback: {
+      ...shared,
       script: [...connectContext, "kubernetesRollback"],
-    }),
-  ];
+    },
+  });
 };

package/src/deploy/kubernetes/mongodb.ts

@@ -1,6 +1,7 @@
 import { range } from "lodash";
-import type { Context } from "../..";
-import { getSecretVarNameForContext, isOfDeployType } from "../..";
+import { getSecretVarNameForContext } from "../../context/getEnvironmentVariables";
+import type { Context } from "../../types/context";
+import { isOfDeployType } from "../types";
 
 const getCredentialString = (context: Context) =>
   `root:$${getSecretVarNameForContext(context, "MONGODB_ROOT_PASSWORD")}@`;

package/src/pipeline/gitlab/createGitlabJobs.ts

@@ -3,6 +3,7 @@ import { BASE_RETRY } from "../../defaults";
 import type { GitlabJobDef } from "../../types";
 import type { CatladderJob, CatladderJobNeed } from "../../types/jobs";
 import type { AllCatladderJobs } from "../createAllJobs";
+import { notNil } from "../../utils";
 
 type AllGitlabJobs = Record<string, GitlabJobDef>;
 
@@ -52,6 +53,7 @@ export const makeGitlabJob = (
     name,
     needs,
     jobTags,
+    script,
     ...job
   }: CatladderJob<string>,
   allJobs: AllCatladderJobs
@@ -106,6 +108,7 @@ export const makeGitlabJob = (
 
   const gitlabJob: GitlabJobDef = removeUndefined({
     ...job,
+    script: script?.filter(notNil),
     tags: jobTags,
     stage,
     environment: job.environment?.on_stop