@catladder/pipeline 1.12.0 → 1.15.0

package/src/build/index.ts

@@ -1,5 +1,6 @@
 import { Context } from "../types/context";
-import { GitlabJobs } from "../types/gitlab-types";
+
+import { CatladderJob } from "../types/jobs";
 import { createNodeJobs, createStorybookJobs, createMeteorJobs } from "./node";
 
 import { BuildConfig } from "./types";
@@ -8,7 +9,7 @@ export * from "./node";
 
 export type BuildTypes = {
   [type in BuildConfig["type"]]: {
-    jobs: (context: Context) => GitlabJobs;
+    jobs: (context: Context) => CatladderJob[];
     defaults: () => Partial<Extract<BuildConfig, { type: type }>>;
   };
 };

package/src/build/node/buildJob.ts

@@ -1,16 +1,16 @@
 import { Context } from "../../types/context";
-import { GitlabJob, GitlabJobs } from "../../types/gitlab-types";
 import { ensureArray } from "../../utils";
 import { APP_BUILD_JOB_NAME } from "../base/constants";
 import { createBuildJob } from "../base/createBuildJob";
-import { createDockerBuildJob } from "../docker";
+import { createDockerBuildJobDefault } from "../docker";
 import { join } from "path";
 import { isOfBuildType } from "../types";
 import { getNextCache, getNodeCache, getYarnCache } from "./cache";
 import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
 import { getYarnInstall, getYarnInstallCommand } from "./yarn";
+import { CatladderJob } from "../../types/jobs";
 
-export const createNodeBuildJobs = (context: Context): GitlabJobs => {
+export const createNodeBuildJobs = (context: Context): CatladderJob[] => {
   const buildConfig = context.componentConfig.build;
 
   if (!isOfBuildType(buildConfig, "node", "node-static", "storybook")) {
@@ -18,7 +18,7 @@ export const createNodeBuildJobs = (context: Context): GitlabJobs => {
   }
 
   const yarnInstall = getYarnInstall(context);
-  const appBuildJob: GitlabJob | null =
+  const appBuildJob: CatladderJob | null =
     buildConfig.buildCommand !== null
       ? createBuildJob(context, {
           variables: {
@@ -43,7 +43,7 @@ export const createNodeBuildJobs = (context: Context): GitlabJobs => {
       : null;
   return [
     ...(appBuildJob ? [appBuildJob] : []),
-    createDockerBuildJob(context, {
+    createDockerBuildJobDefault(context, {
       script: [
         buildConfig.type === "node-static" || buildConfig.type === "storybook"
           ? "ensureNginxDockerfile"

package/src/build/node/index.ts

@@ -1,17 +1,17 @@
 import type { Context } from "../../types/context";
-import type { GitlabJobs } from "../../types/gitlab-types";
+import { CatladderJob } from "../../types/jobs";
 import { createNodeBuildJobs } from "./buildJob";
 import { createMeteorBuildJobs } from "./meteor";
 import { createNodeTestJobs } from "./testJob";
 
-export const createNodeJobs = (context: Context): GitlabJobs => {
+export const createNodeJobs = (context: Context): CatladderJob[] => {
   return [...createNodeTestJobs(context), ...createNodeBuildJobs(context)];
 };
 
-export const createStorybookJobs = (context: Context): GitlabJobs => {
+export const createStorybookJobs = (context: Context): CatladderJob[] => {
   return [...createNodeBuildJobs(context)];
 };
 
-export const createMeteorJobs = (context: Context): GitlabJobs => {
+export const createMeteorJobs = (context: Context): CatladderJob[] => {
   return [...createNodeTestJobs(context), ...createMeteorBuildJobs(context)];
 };

package/src/build/node/meteor.ts

@@ -1,15 +1,13 @@
 import { join } from "path";
 import { getRunnerImage } from "../../runner";
+import { GitlabJobCache } from "../../types";
 import type { Context } from "../../types/context";
-import type {
-  GitlabJob,
-  GitlabJobCache,
-  GitlabJobs,
-} from "../../types/gitlab-types";
+
+import { CatladderJob } from "../../types/jobs";
 
 import { APP_BUILD_JOB_NAME } from "../base/constants";
 import { createBuildJob } from "../base/createBuildJob";
-import { createDockerBuildJob } from "../docker";
+import { createDockerBuildJobDefault } from "../docker";
 import { isOfBuildType } from "../types";
 import { getNodeCache } from "./cache";
 import { getYarnInstall } from "./yarn";
@@ -29,7 +27,7 @@ const getMeteorCache = (context: Context): GitlabJobCache[] => [
     ],
   },
 ];
-export const createMeteorBuildJobs = (context: Context): GitlabJobs => {
+export const createMeteorBuildJobs = (context: Context): CatladderJob[] => {
   const buildConfig = context.componentConfig.build;
 
   if (!isOfBuildType(buildConfig, "meteor")) {
@@ -37,7 +35,7 @@ export const createMeteorBuildJobs = (context: Context): GitlabJobs => {
   }
 
   const yarnInstall = getYarnInstall(context);
-  const appBuildJob: GitlabJob | null =
+  const appBuildJob: CatladderJob | null =
     buildConfig.buildCommand !== null
       ? createBuildJob(context, {
           cache: [...getNodeCache(context), ...getMeteorCache(context)],
@@ -63,7 +61,7 @@ export const createMeteorBuildJobs = (context: Context): GitlabJobs => {
       : null;
   return [
     ...(appBuildJob ? [appBuildJob] : []),
-    createDockerBuildJob(context, {
+    createDockerBuildJobDefault(context, {
       script: ["ensureMeteorDockerfile"],
       needs: appBuildJob ? [APP_BUILD_JOB_NAME] : [],
     }),

package/src/build/node/testJob.ts

@@ -1,12 +1,11 @@
-import { BASE_RETRY } from "../../defaults";
 import { Context } from "../../types/context";
-import { GitlabJob, GitlabJobDef, GitlabJobs } from "../../types/gitlab-types";
+import { CatladderJob } from "../../types/jobs";
 import { ensureArray, notNil } from "../../utils";
 import { getNodeCache } from "./cache";
 import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
 import { getYarnInstall } from "./yarn";
 
-export const createNodeTestJobs = (context: Context): GitlabJobs => {
+export const createNodeTestJobs = (context: Context): CatladderJob[] => {
   // don't run tests after release
   if (context.commitInfo?.trigger === "taggedRelease") {
     return [];
@@ -14,66 +13,58 @@ export const createNodeTestJobs = (context: Context): GitlabJobs => {
 
   const buildConfig = context.componentConfig.build;
 
-  const base: Omit<GitlabJobDef, "script"> = {
+  const base: Omit<CatladderJob, "script" | "name"> = {
     variables: {
       APP_PATH: context.componentConfig.dir,
       ...NODE_RUNNER_BUILD_VARIABLES,
       ...(buildConfig.extraVars ?? {}),
     },
-
     stage: "test",
-    interruptible: true,
     needs: [],
-    retry: BASE_RETRY,
+    envMode: "none",
   };
   const yarnInstall = getYarnInstall(context);
-  const auditJob: GitlabJob | null =
+  const auditJob: CatladderJob | null =
     buildConfig.audit !== false
       ? {
           name: "🛡 audit",
-          envMode: "none",
-          job: {
-            ...base,
-            cache: undefined, // audit does not need yarn install and no cache
-            script: [
-              `cd ${context.componentConfig.dir}`,
-              ...(ensureArray(buildConfig.audit?.command) ?? ["yarn audit"]),
-            ],
-            allow_failure: true,
-          },
+
+          ...base,
+          cache: undefined, // audit does not need yarn install and no cache
+          script: [
+            `cd ${context.componentConfig.dir}`,
+            ...(ensureArray(buildConfig.audit?.command) ?? ["yarn audit"]),
+          ],
+          allow_failure: true,
         }
       : null;
 
-  const lintJob: GitlabJob | null =
+  const lintJob: CatladderJob | null =
     buildConfig.lint !== false
       ? {
           name: "👮 lint",
-          envMode: "none",
-          job: {
-            ...base,
-            cache: getNodeCache(context),
-            script: [
-              `cd ${context.componentConfig.dir}`,
-              ...yarnInstall,
-              ...(ensureArray(buildConfig.lint?.command) ?? ["yarn lint"]),
-            ],
-          },
+
+          ...base,
+          cache: getNodeCache(context),
+          script: [
+            `cd ${context.componentConfig.dir}`,
+            ...yarnInstall,
+            ...(ensureArray(buildConfig.lint?.command) ?? ["yarn lint"]),
+          ],
         }
       : null;
-  const testJob: GitlabJob | null =
+  const testJob: CatladderJob | null =
     buildConfig.test !== false
       ? {
           name: "🧪 test",
-          envMode: "none",
-          job: {
-            ...base,
-            cache: getNodeCache(context),
-            script: [
-              `cd ${context.componentConfig.dir}`,
-              ...yarnInstall,
-              ...(ensureArray(buildConfig.test?.command) ?? ["yarn test"]),
-            ],
-          },
+
+          ...base,
+          cache: getNodeCache(context),
+          script: [
+            `cd ${context.componentConfig.dir}`,
+            ...yarnInstall,
+            ...(ensureArray(buildConfig.test?.command) ?? ["yarn test"]),
+          ],
         }
       : null;
   return [auditJob, lintJob, testJob].filter(notNil);

package/src/build/tests/storybook.test.ts

@@ -26,7 +26,8 @@ describe("storybook build", () => {
     };
 
     it("creates a pipeline for storybook that does not contain test stages", async () => {
-      const { image, stages, workflow, ...jobs } = await createChildPipeline(
+      const { jobs } = await createChildPipeline(
+        "gitlab",
         "mainBranch",
         config
       );
@@ -40,7 +41,8 @@ describe("storybook build", () => {
     });
 
     it("runs build and renames folder", async () => {
-      const { image, stages, workflow, ...jobs } = await createChildPipeline(
+      const { jobs } = await createChildPipeline(
+        "gitlab",
         "mainBranch",
         config
       );

package/src/catladder-gitlab.ts

@@ -1,4 +1,5 @@
 import { writeFileSync } from "fs";
+import { dump } from "js-yaml";
 import { readConfigSync } from "./config";
 import { PIPELINE_IMAGE_TAG } from "./constants";
 import { createChildPipeline } from "./pipeline";
@@ -32,11 +33,14 @@ if (trigger) {
   if (!config) {
     throw new Error("no catladder config found");
   }
-  createChildPipeline(trigger, config).then((mainPipeline) => {
-    writeFileSync(`__pipeline.yml`, JSON.stringify(mainPipeline, null, 2), {
-      encoding: "utf-8",
-    });
-  });
+  createChildPipeline("gitlab", trigger, config).then(
+    ({ jobs, ...mainPipeline }) => {
+      // need to spread out the jobs
+      writeFileSync(`__pipeline.yml`, dump({ ...jobs, ...mainPipeline }), {
+        encoding: "utf-8",
+      });
+    }
+  );
 } else {
   throw new Error(
     "no matching trigger: " +

package/src/deploy/base.ts

@@ -1,4 +1,5 @@
-import { GitlabJob, Context, GitlabJobDef } from "../types";
+import { Context } from "../types";
+import { CatladderJob } from "../types/jobs";
 
 export const DEPLOY_JOB_NAME = "🚀 Deploy";
 export const STOP_JOB_NAME = "🛑 Stop ⚠️";
@@ -9,9 +10,7 @@ const DEPLOY_RUNNER_VARIABLES = {
   KUBERNETES_MEMORY_REQUEST: "200Mi",
   KUBERNETES_MEMORY_LIMIT: "500Mi",
 };
-type JobWithoutScript = Omit<GitlabJob, "job"> & {
-  job: Omit<GitlabJobDef, "script">;
-};
+type JobWithoutScript = Omit<CatladderJob, "script">;
 export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
   const environment = {
     name: context.environment.fullName,
@@ -45,26 +44,24 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
         artifacts: false,
       },
     ], // workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/220758
-    job: {
-      rules: [
-        autoDeploy
-          ? {
-              when: "on_success",
-            }
-          : {
-              when: "manual",
-            },
-      ],
-      stage: "deploy",
-      dependencies: [],
-      variables: {
-        ...DEPLOY_RUNNER_VARIABLES,
-      },
-      environment: {
-        ...environment,
-        on_stop: STOP_JOB_NAME,
-        auto_stop_in: autoStop,
-      },
+
+    rules: [
+      autoDeploy
+        ? {
+            when: "on_success",
+          }
+        : {
+            when: "manual",
+          },
+    ],
+    stage: "deploy",
+    variables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+    },
+    environment: {
+      ...environment,
+      on_stop: STOP_JOB_NAME,
+      auto_stop_in: autoStop,
     },
   };
 };
@@ -79,29 +76,27 @@ export const getBaseDeploymentStopJob = (
   return {
     name: STOP_JOB_NAME,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
-    job: {
-      needs: [DEPLOY_JOB_NAME],
-      rules: [
-        {
-          if: "$CI_COMMIT_BRANCH =~ /^[0-9]+\\.([0-9]+|x)\\.x$/", // automatic on hotfix branches
-          when: "on_success",
-          allow_failure: true,
-        },
-        {
-          when: "manual",
-          allow_failure: true,
-        },
-      ],
-      variables: {
-        ...DEPLOY_RUNNER_VARIABLES,
-        GIT_STRATEGY: "none",
+
+    needs: [DEPLOY_JOB_NAME],
+    rules: [
+      {
+        if: "$CI_COMMIT_BRANCH =~ /^[0-9]+\\.([0-9]+|x)\\.x$/", // automatic on hotfix branches
+        when: "on_success",
+        allow_failure: true,
       },
-      stage: "stop",
-      dependencies: [],
-      environment: {
-        ...environment,
-        action: "stop",
+      {
+        when: "manual",
+        allow_failure: true,
       },
+    ],
+    variables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+      GIT_STRATEGY: "none",
+    },
+    stage: "stop",
+    environment: {
+      ...environment,
+      action: "stop",
     },
   };
 };

package/src/deploy/index.ts

@@ -1,5 +1,5 @@
-import { GitlabJobs } from "../types/gitlab-types";
 import { Context } from "../types/context";
+import { CatladderJob } from "../types/jobs";
 import { createKubernetesDeployJobs } from "./kubernetes";
 import { DeployConfig } from "./types";
 export * from "./kubernetes";
@@ -7,7 +7,7 @@ export * from "./types";
 export * from "./utils";
 export type DeployTypes = {
   [type in DeployConfig["type"]]: {
-    jobs: (context: Context) => GitlabJobs;
+    jobs: (context: Context) => CatladderJob[];
     defaults: () => Partial<Extract<DeployConfig, { type: type }>>;
   };
 };

package/src/deploy/kubernetes/deployJob.ts

@@ -1,18 +1,19 @@
-import { GitlabJobs } from "../../types/gitlab-types";
-import { Context } from "../../types/context";
-import { isOfDeployType } from "../types";
-import { getRunnerImage } from "../../runner";
-import { getBaseDeploymentJob, getBaseDeploymentStopJob } from "../base";
-import { merge } from "lodash";
 import { dump } from "js-yaml";
-import { createMongodbBaseConfig } from "./mongodb";
-import { createCloudsqlBaseConfig } from "./cloudsql";
+import { merge } from "lodash";
+import { DeployConfigKubernetesValues } from "..";
 import { getSecretVarNameForContext } from "../..";
+import { getRunnerImage } from "../../runner";
+import { Context } from "../../types/context";
+import { CatladderJob } from "../../types/jobs";
 import { mergeWithMergingArrays } from "../../utils";
-import { DeployConfigKubernetesValues } from "..";
-import { PartialDeep } from "type-fest";
+import { getBaseDeploymentJob, getBaseDeploymentStopJob } from "../base";
+import { isOfDeployType } from "../types";
+import { createCloudsqlBaseConfig } from "./cloudsql";
+import { createMongodbBaseConfig } from "./mongodb";
 
-export const createKubernetesDeployJobs = (context: Context): GitlabJobs => {
+export const createKubernetesDeployJobs = (
+  context: Context
+): CatladderJob[] => {
   const deployConfig = context.componentConfig.deploy;
   if (deployConfig === false) {
     return [];
@@ -91,28 +92,26 @@ export const createKubernetesDeployJobs = (context: Context): GitlabJobs => {
     namespace: context.environment.envVars.KUBE_NAMESPACE,
   };
   const shared = {
-    job: {
-      image: getRunnerImage("kubernetes"),
-      variables: {
-        ...context.environment.envVars,
-        HELM_EXPERIMENTAL_OCI: "1",
-        IMAGE_PULL_SECRET: `gitlab-registry-${context.componentName}`,
-        KUBE_VALUES: dump(kubeValues, {
-          lineWidth: -1,
-          quotingType: "'",
-          forceQuotes: true,
-        }),
-        HELM_GITLAB_CHART_NAME: "the-panter-chart",
-        HELM_ARGS: [
-          ...(deployConfig.debug ? ["--debug"] : []),
-          ...(deployConfig.additionalHelmArgs ?? []),
-        ].join(" "),
-        COMPONENT_NAME: context.componentName,
-        BUILD_ID: context.commitInfo?.buildId,
-        // TODO: unify with docker build stage
-        IMAGE_NAME: context.environment.shortName + "/" + context.componentName,
-        IMAGE_TAG: "$CI_COMMIT_SHA",
-      },
+    image: getRunnerImage("kubernetes"),
+    variables: {
+      ...context.environment.envVars,
+      HELM_EXPERIMENTAL_OCI: "1",
+      IMAGE_PULL_SECRET: `gitlab-registry-${context.componentName}`,
+      KUBE_VALUES: dump(kubeValues, {
+        lineWidth: -1,
+        quotingType: "'",
+        forceQuotes: true,
+      }),
+      HELM_GITLAB_CHART_NAME: "the-panter-chart",
+      HELM_ARGS: [
+        ...(deployConfig.debug ? ["--debug"] : []),
+        ...(deployConfig.additionalHelmArgs ?? []),
+      ].join(" "),
+      COMPONENT_NAME: context.componentName,
+      BUILD_ID: context.commitInfo?.buildId,
+      // TODO: unify with docker build stage
+      IMAGE_NAME: context.environment.shortName + "/" + context.componentName,
+      IMAGE_TAG: "$CI_COMMIT_SHA",
     },
   };
 
@@ -136,24 +135,20 @@ export const createKubernetesDeployJobs = (context: Context): GitlabJobs => {
   ];
   return [
     merge({}, baseDeploymentJob, shared, {
-      job: {
-        script: [
-          ...connectContext,
-          "kubernetesCreateSecret",
-          "kubernetesDeploy",
-          "echo deployment successful 😻",
-        ],
-        environment: {
-          kubernetes: kubernetesEnvironment,
-        },
+      script: [
+        ...connectContext,
+        "kubernetesCreateSecret",
+        "kubernetesDeploy",
+        "echo deployment successful 😻",
+      ],
+      environment: {
+        kubernetes: kubernetesEnvironment,
       },
     }),
     merge({}, baseStopJob, shared, {
-      job: {
-        script: [...connectContext, "kubernetesDelete"],
-        environment: {
-          kubernetes: kubernetesEnvironment,
-        },
+      script: [...connectContext, "kubernetesDelete"],
+      environment: {
+        kubernetes: kubernetesEnvironment,
       },
     }),
   ];

package/src/pipeline/createChildPipeline.test.ts

@@ -26,7 +26,8 @@ describe("createChildPipeline", () => {
     };
 
     it("creates a pipeline for a single app on the main branch", async () => {
-      const { image, stages, workflow, ...jobs } = await createChildPipeline(
+      const { image, jobs } = await createChildPipeline(
+        "gitlab",
         "mainBranch",
         config
       );

package/src/pipeline/createChildPipeline.ts

@@ -1,24 +1,25 @@
 import { getAllEnvsByTrigger, getAllEnvsInAllComponents } from "../config";
-import { createJobs } from "./createJobs";
 import { RULES_ALWAYS } from "../rules";
 import { getRunnerImage } from "../runner";
+import { GitlabPipeline, Pipeline, PipelineJob, PipelineType } from "../types";
 import { Config, PipelineTrigger } from "../types/config";
-import { GitlabJobDef } from "../types/gitlab-types";
+import { createJobs } from "./createJobs";
+
 const baseStages = ["setup", "test", "build", "deploy", "verify", "stop"];
-export const createChildPipeline = async (
+export const createChildPipeline = async <T extends PipelineType>(
+  type: T,
   trigger: PipelineTrigger,
   config: Config
-) => {
+): Promise<Pipeline<T>> => {
   const components = Object.keys(config.components);
 
   // 2. write the triggering pipeline
-
-  const jobs = await components.reduce<Promise<Record<string, GitlabJobDef>>>(
+  const jobs = await components.reduce<Promise<Record<string, PipelineJob<T>>>>(
     async (acc, componentName) => {
       const envs = getAllEnvsByTrigger(config, componentName, trigger);
       return {
         ...(await acc),
-        ...(await createJobs(envs, config, componentName, trigger)),
+        ...(await createJobs(type, envs, config, componentName, trigger)),
       };
     },
     Promise.resolve({})
@@ -37,18 +38,19 @@ export const createChildPipeline = async (
     ],
     []
   );
-
-  const childPipeline = {
-    image: getRunnerImage("jobs-default"), // default image
-    variables: {
-      FF_USE_FASTZIP: "true",
-    },
-    workflow: {
-      rules: RULES_ALWAYS,
-    },
-    stages,
-    ...jobs,
-  };
-
-  return childPipeline as typeof childPipeline & Record<string, GitlabJobDef>;
+  if (type === "gitlab") {
+    const pipeline: GitlabPipeline = {
+      image: getRunnerImage("jobs-default"), // default image
+      variables: {
+        FF_USE_FASTZIP: "true",
+      },
+      workflow: {
+        rules: RULES_ALWAYS,
+      },
+      stages,
+      jobs,
+    };
+    return pipeline as Pipeline<T>;
+  }
+  throw new Error(`${type} is not supported`);
 };