@catladder/pipeline 1.12.0 → 1.13.0

package/src/deploy/index.ts

@@ -1,5 +1,5 @@
-import { GitlabJobs } from "../types/gitlab-types";
 import { Context } from "../types/context";
+import { CatladderJob } from "../types/jobs";
 import { createKubernetesDeployJobs } from "./kubernetes";
 import { DeployConfig } from "./types";
 export * from "./kubernetes";
@@ -7,7 +7,7 @@ export * from "./types";
 export * from "./utils";
 export type DeployTypes = {
   [type in DeployConfig["type"]]: {
-    jobs: (context: Context) => GitlabJobs;
+    jobs: (context: Context) => CatladderJob[];
     defaults: () => Partial<Extract<DeployConfig, { type: type }>>;
   };
 };

package/src/deploy/kubernetes/deployJob.ts

@@ -1,18 +1,19 @@
-import { GitlabJobs } from "../../types/gitlab-types";
-import { Context } from "../../types/context";
-import { isOfDeployType } from "../types";
-import { getRunnerImage } from "../../runner";
-import { getBaseDeploymentJob, getBaseDeploymentStopJob } from "../base";
-import { merge } from "lodash";
 import { dump } from "js-yaml";
-import { createMongodbBaseConfig } from "./mongodb";
-import { createCloudsqlBaseConfig } from "./cloudsql";
+import { merge } from "lodash";
+import { DeployConfigKubernetesValues } from "..";
 import { getSecretVarNameForContext } from "../..";
+import { getRunnerImage } from "../../runner";
+import { Context } from "../../types/context";
+import { CatladderJob } from "../../types/jobs";
 import { mergeWithMergingArrays } from "../../utils";
-import { DeployConfigKubernetesValues } from "..";
-import { PartialDeep } from "type-fest";
+import { getBaseDeploymentJob, getBaseDeploymentStopJob } from "../base";
+import { isOfDeployType } from "../types";
+import { createCloudsqlBaseConfig } from "./cloudsql";
+import { createMongodbBaseConfig } from "./mongodb";
 
-export const createKubernetesDeployJobs = (context: Context): GitlabJobs => {
+export const createKubernetesDeployJobs = (
+  context: Context
+): CatladderJob[] => {
   const deployConfig = context.componentConfig.deploy;
   if (deployConfig === false) {
     return [];
@@ -91,28 +92,26 @@ export const createKubernetesDeployJobs = (context: Context): GitlabJobs => {
     namespace: context.environment.envVars.KUBE_NAMESPACE,
   };
   const shared = {
-    job: {
-      image: getRunnerImage("kubernetes"),
-      variables: {
-        ...context.environment.envVars,
-        HELM_EXPERIMENTAL_OCI: "1",
-        IMAGE_PULL_SECRET: `gitlab-registry-${context.componentName}`,
-        KUBE_VALUES: dump(kubeValues, {
-          lineWidth: -1,
-          quotingType: "'",
-          forceQuotes: true,
-        }),
-        HELM_GITLAB_CHART_NAME: "the-panter-chart",
-        HELM_ARGS: [
-          ...(deployConfig.debug ? ["--debug"] : []),
-          ...(deployConfig.additionalHelmArgs ?? []),
-        ].join(" "),
-        COMPONENT_NAME: context.componentName,
-        BUILD_ID: context.commitInfo?.buildId,
-        // TODO: unify with docker build stage
-        IMAGE_NAME: context.environment.shortName + "/" + context.componentName,
-        IMAGE_TAG: "$CI_COMMIT_SHA",
-      },
+    image: getRunnerImage("kubernetes"),
+    variables: {
+      ...context.environment.envVars,
+      HELM_EXPERIMENTAL_OCI: "1",
+      IMAGE_PULL_SECRET: `gitlab-registry-${context.componentName}`,
+      KUBE_VALUES: dump(kubeValues, {
+        lineWidth: -1,
+        quotingType: "'",
+        forceQuotes: true,
+      }),
+      HELM_GITLAB_CHART_NAME: "the-panter-chart",
+      HELM_ARGS: [
+        ...(deployConfig.debug ? ["--debug"] : []),
+        ...(deployConfig.additionalHelmArgs ?? []),
+      ].join(" "),
+      COMPONENT_NAME: context.componentName,
+      BUILD_ID: context.commitInfo?.buildId,
+      // TODO: unify with docker build stage
+      IMAGE_NAME: context.environment.shortName + "/" + context.componentName,
+      IMAGE_TAG: "$CI_COMMIT_SHA",
     },
   };
 
@@ -136,24 +135,20 @@ export const createKubernetesDeployJobs = (context: Context): GitlabJobs => {
   ];
   return [
     merge({}, baseDeploymentJob, shared, {
-      job: {
-        script: [
-          ...connectContext,
-          "kubernetesCreateSecret",
-          "kubernetesDeploy",
-          "echo deployment successful 😻",
-        ],
-        environment: {
-          kubernetes: kubernetesEnvironment,
-        },
+      script: [
+        ...connectContext,
+        "kubernetesCreateSecret",
+        "kubernetesDeploy",
+        "echo deployment successful 😻",
+      ],
+      environment: {
+        kubernetes: kubernetesEnvironment,
       },
     }),
     merge({}, baseStopJob, shared, {
-      job: {
-        script: [...connectContext, "kubernetesDelete"],
-        environment: {
-          kubernetes: kubernetesEnvironment,
-        },
+      script: [...connectContext, "kubernetesDelete"],
+      environment: {
+        kubernetes: kubernetesEnvironment,
       },
     }),
   ];

package/src/pipeline/createChildPipeline.test.ts

@@ -26,7 +26,8 @@ describe("createChildPipeline", () => {
     };
 
     it("creates a pipeline for a single app on the main branch", async () => {
-      const { image, stages, workflow, ...jobs } = await createChildPipeline(
+      const { image, jobs } = await createChildPipeline(
+        "gitlab",
         "mainBranch",
         config
       );

package/src/pipeline/createChildPipeline.ts

@@ -1,24 +1,25 @@
 import { getAllEnvsByTrigger, getAllEnvsInAllComponents } from "../config";
-import { createJobs } from "./createJobs";
 import { RULES_ALWAYS } from "../rules";
 import { getRunnerImage } from "../runner";
+import { GitlabPipeline, Pipeline, PipelineJob, PipelineType } from "../types";
 import { Config, PipelineTrigger } from "../types/config";
-import { GitlabJobDef } from "../types/gitlab-types";
+import { createJobs } from "./createJobs";
+
 const baseStages = ["setup", "test", "build", "deploy", "verify", "stop"];
-export const createChildPipeline = async (
+export const createChildPipeline = async <T extends PipelineType>(
+  type: T,
   trigger: PipelineTrigger,
   config: Config
-) => {
+): Promise<Pipeline<T>> => {
   const components = Object.keys(config.components);
 
   // 2. write the triggering pipeline
-
-  const jobs = await components.reduce<Promise<Record<string, GitlabJobDef>>>(
+  const jobs = await components.reduce<Promise<Record<string, PipelineJob<T>>>>(
     async (acc, componentName) => {
       const envs = getAllEnvsByTrigger(config, componentName, trigger);
       return {
         ...(await acc),
-        ...(await createJobs(envs, config, componentName, trigger)),
+        ...(await createJobs(type, envs, config, componentName, trigger)),
       };
     },
     Promise.resolve({})
@@ -37,18 +38,19 @@ export const createChildPipeline = async (
     ],
     []
   );
-
-  const childPipeline = {
-    image: getRunnerImage("jobs-default"), // default image
-    variables: {
-      FF_USE_FASTZIP: "true",
-    },
-    workflow: {
-      rules: RULES_ALWAYS,
-    },
-    stages,
-    ...jobs,
-  };
-
-  return childPipeline as typeof childPipeline & Record<string, GitlabJobDef>;
+  if (type === "gitlab") {
+    const pipeline: GitlabPipeline = {
+      image: getRunnerImage("jobs-default"), // default image
+      variables: {
+        FF_USE_FASTZIP: "true",
+      },
+      workflow: {
+        rules: RULES_ALWAYS,
+      },
+      stages,
+      jobs,
+    };
+    return pipeline as Pipeline<T>;
+  }
+  throw new Error(`${type} is not supported`);
 };

package/src/pipeline/createJobs.ts

@@ -3,13 +3,15 @@ import { isObject } from "lodash";
 import { BUILD_TYPES } from "../build";
 import { createContext } from "../context";
 import { DEPLOY_TYPES } from "../deploy";
+import { PipelineJob, PipelineType } from "../types";
 import { Config, PipelineTrigger } from "../types/config";
-import { Context, CommitInfo } from "../types/context";
-import { GitlabJob, GitlabJobDef, GitlabJobs } from "../types/gitlab-types";
+import { CommitInfo, Context } from "../types/context";
+import { CatladderJob } from "../types/jobs";
 import { notNil } from "../utils";
+import { makeGitlabJob } from "./gitlab/makeGitlabJob";
 import { getPackageManagerInfo } from "./packageManager";
 
-const createRawJobs = (context: Context): GitlabJobs => {
+const createRawJobs = (context: Context): CatladderJob[] => {
   if (context.componentConfig.deploy === false) {
     return [];
   }
@@ -35,7 +37,7 @@ const getFullReferencedJobName = (
   referencedJobName: string,
   componentName: string,
   env: string,
-  allRawJobs: GitlabJobs
+  allRawJobs: CatladderJob[]
 ) => {
   const referencedJob = allRawJobs.find((j) => j.name === referencedJobName);
   if (!referencedJob) {
@@ -47,18 +49,17 @@ const getFullReferencedJobName = (
 // replaces references to other jobs with the full name
 // the full name contains the componentname and the env name (if any)
 const replaceReferences = (
-  job: GitlabJob,
+  job: CatladderJob,
   componentName: string,
   env: string,
-  allRawJobs: GitlabJobs
-): GitlabJobDef => {
-  const def = job.job;
+  allRawJobs: CatladderJob[]
+): CatladderJob<string> => {
   const stage =
-    job.envMode === "stagePerEnv" ? `${def.stage} ${env}` : def.stage;
+    job.envMode === "stagePerEnv" ? `${job.stage} ${env}` : job.stage;
   return {
-    ...def,
+    ...job,
     stage,
-    needs: def.needs?.map((n) =>
+    needs: job.needs?.map((n) =>
       isObject(n)
         ? {
             job: getFullReferencedJobName(
@@ -71,25 +72,22 @@ const replaceReferences = (
           }
         : getFullReferencedJobName(n, componentName, env, allRawJobs)
     ),
-    environment: def.environment?.on_stop
+    environment: job.environment?.on_stop
       ? {
-          ...def.environment,
+          ...job.environment,
           on_stop: getFullReferencedJobName(
-            def.environment.on_stop,
+            job.environment.on_stop,
             componentName,
             env,
             allRawJobs
           ),
         }
-      : def.environment,
-    dependencies: def.dependencies?.map((n) =>
-      getFullReferencedJobName(n, componentName, env, allRawJobs)
-    ),
+      : job.environment,
   };
 };
 
 // this can be removed once https://gitlab.com/gitlab-org/gitlab/-/issues/220758 is resolved
-const addStageNeeds = (jobs: GitlabJobs): GitlabJobs => {
+const addStageNeeds = (jobs: CatladderJob[]): CatladderJob[] => {
   // when a job defines needsStages, we add these as needs
   return jobs.map((job) => {
     if (!job.needsStages || job.needsStages.length === 0) {
@@ -98,9 +96,7 @@ const addStageNeeds = (jobs: GitlabJobs): GitlabJobs => {
 
     const neededJobs = jobs
       .map((j) => {
-        const neededStage = job.needsStages?.find(
-          (s) => j.job.stage === s.stage
-        );
+        const neededStage = job.needsStages?.find((s) => j.stage === s.stage);
         if (neededStage) {
           return {
             job: j.name,
@@ -111,19 +107,19 @@ const addStageNeeds = (jobs: GitlabJobs): GitlabJobs => {
       .filter(notNil);
     return {
       ...job,
-      job: {
-        ...job.job,
-        needs: [...(job.job.needs ?? []), ...neededJobs],
-      },
+
+      needs: [...(job.needs ?? []), ...neededJobs],
     };
   });
 };
-export const createJobs = async (
+
+export const createJobs = async <T extends PipelineType>(
+  type: T,
   envs: string[],
   config: Config,
   componentName: string,
   trigger: PipelineTrigger
-): Promise<Record<string, GitlabJobDef>> => {
+): Promise<Record<string, PipelineJob<T>>> => {
   const commitInfo: CommitInfo = {
     refName: process.env.CI_COMMIT_REF_NAME ?? "unknown",
     refSlug: process.env.CI_COMMIT_REF_SLUG ?? "unknown",
@@ -146,18 +142,31 @@ export const createJobs = async (
     );
     const jobs = addStageNeeds(createRawJobs(context));
 
-    return {
+    const result = {
       ...acc,
-      ...jobs.reduce((acc, job) => {
-        return {
-          ...acc,
-          [getFullJobName(
-            job.name,
-            componentName,
-            job.envMode !== "none" ? env : undefined
-          )]: replaceReferences(job, componentName, env, jobs),
-        };
+      ...jobs.reduce<Record<string, PipelineJob<T>>>((acc, job) => {
+        const jobWithResolvedReferences = replaceReferences(
+          job,
+          componentName,
+          env,
+          jobs
+        );
+        const jobName = getFullJobName(
+          job.name,
+          componentName,
+          job.envMode !== "none" ? env : undefined
+        );
+        if (type === "gitlab") {
+          return {
+            ...acc,
+            [jobName]: makeGitlabJob(
+              jobWithResolvedReferences
+            ) as PipelineJob<T>,
+          };
+        }
+        throw new Error("not supported");
       }, {}),
     };
+    return result;
   }, {});
 };

package/src/pipeline/gitlab/makeGitlabJob.ts

@@ -0,0 +1,16 @@
+import { BASE_RETRY } from "../../defaults";
+import { GitlabJobDef } from "../../types";
+import { CatladderJob } from "../../types/jobs";
+
+export const makeGitlabJob = ({
+  envMode,
+  needsStages,
+  name,
+  ...rest
+}: CatladderJob<string>): GitlabJobDef => {
+  return {
+    ...rest,
+    retry: BASE_RETRY,
+    interruptible: true,
+  };
+};

package/src/types/gitlab-types.ts

@@ -56,39 +56,19 @@ export type GitlabJobDef = {
   };
 };
 
-export const GITLAB_BASE_STAGES = [
-  "setup",
-  "test",
-  "build",
-  "deploy",
-  "verify",
-  "stop",
-] as const;
-export type GitlabBaseStage = typeof GITLAB_BASE_STAGES[number];
 export type GitlabVariables = Record<string, string | undefined>;
 
-export type GitlabJob = {
-  /**
-   * the name of the job (without any env or app prefix and suffix)
-   */
-  name: string;
-  /**
-   * envMode sets the behavior of the job regarding multiple envs:
-   * - none: the job does not run per env, but once for all envs
-   * - jobPerEnv: the job runs once per env
-   * - stagePerEnv: the job runs once per env and is organized in its own stage. This mproves usability in gitlab, but works the same as `jobPerEnv`
-   */
-  envMode: "jobPerEnv" | "stagePerEnv" | "none";
-  job: GitlabJobDef;
-  needsStages?: {
-    stage: GitlabBaseStage;
-    artifacts?: boolean;
-  }[];
-};
-export type GitlabJobs = GitlabJob[];
-
+/**
+ * this is not precicily the type of a gitlab-pipeline
+ * the jobs nee to be merge into the object.
+ * Problem is, that this type cannot be represented properly with typescript, see https://github.com/microsoft/TypeScript/issues/17867
+ */
 export type GitlabPipeline = {
+  image: string;
   variables: GitlabVariables;
+  workflow?: {
+    rules: GitlabRule[];
+  };
   stages: string[];
-  jobs: GitlabJobs;
+  jobs: Record<string, GitlabJobDef>;
 };

package/src/types/index.ts

@@ -1,3 +1,4 @@
 export * from "./config";
 export * from "./gitlab-types";
 export * from "./context";
+export * from "./pipeline";

package/src/types/jobs.ts

@@ -0,0 +1,88 @@
+import {
+  Artifacts,
+  GitlabEnvironment,
+  GitlabJobCache,
+  GitlabRule,
+  GitlabVariables,
+  Service,
+} from "./gitlab-types";
+
+export const BASE_STAGES = [
+  "setup",
+  "test",
+  "build",
+  "deploy",
+  "verify",
+  "stop",
+] as const;
+export type BaseStage = typeof BASE_STAGES[number];
+
+export type CatladderJob<S = BaseStage> = {
+  /**
+   * the name of the job (without any env or app prefix and suffix)
+   */
+  name: string;
+  /**
+   * envMode sets the behavior of the job regarding multiple envs:
+   * - none: the job does not run per env, but once for all envs
+   * - jobPerEnv: the job runs once per env
+   * - stagePerEnv: the job runs once per env and is organized in its own stage. This mproves usability in gitlab, but works the same as `jobPerEnv`
+   */
+  envMode: "jobPerEnv" | "stagePerEnv" | "none";
+
+  /**
+   * the stage of the job
+   */
+  stage: S;
+  /**
+   * does this require another stage?
+   */
+
+  /**
+   * script to run
+   */
+  script: (string | undefined)[];
+
+  needsStages?: {
+    stage: S;
+    artifacts?: boolean;
+  }[];
+
+  /**
+   * does this require another job?
+   */
+  needs?: Array<string | { job: string; artifacts: boolean }>;
+  /**
+   * cache config, we use here the same shape as gitlab itself
+   */
+  cache?: GitlabJobCache | GitlabJobCache[];
+
+  /**
+   * job artifacts, we also use gitlab shape here
+   */
+  artifacts?: Artifacts;
+
+  /**
+   * additional services, mainly used for docker
+   */
+  services?: Service[];
+
+  /**
+   * image to use
+   */
+  image?: string;
+
+  /**
+   * variables to pass
+   */
+  variables: GitlabVariables;
+
+  /**
+   * whether failures are allowed
+   */
+  allow_failure?: boolean;
+
+  environment?: GitlabEnvironment;
+
+  rules?: GitlabRule[];
+};

package/src/types/pipeline.ts

@@ -0,0 +1,11 @@
+import { GitlabJobDef, GitlabPipeline } from "./gitlab-types";
+
+export type PipelineType = "gitlab";
+
+export type PipelineJob<T extends PipelineType> = T extends "gitlab"
+  ? GitlabJobDef
+  : never;
+
+export type Pipeline<T extends PipelineType> = T extends "gitlab"
+  ? GitlabPipeline
+  : never;