@catladder/pipeline 0.0.0-refactore-catenv-command-validates-argument-78d05852 → 0.0.0-reference-database-url-1befbbee

package/src/pipeline/__tests__/getPipelineStages.test.ts

@@ -0,0 +1,63 @@
+import { type DeployConfigKubernetesCluster } from "../..";
+import type { Config } from "../../types";
+import { getPipelineStages } from "../getPipelineStages";
+
+describe("getPipelineStages()", () => {
+  const cluster: DeployConfigKubernetesCluster = {
+    type: "gcloud",
+    name: "mega-cluster",
+    projectId: "super-google-project",
+    region: "ch-blabla",
+  };
+  const SIMPLE_CONFIG1: Config = {
+    appName: "my-app",
+    customerName: "pan",
+    components: {
+      app1: {
+        dir: "dir1",
+        build: {
+          type: "node",
+        },
+        deploy: { type: "kubernetes", cluster },
+      },
+      app2: {
+        dir: "dir2",
+        build: {
+          type: "node",
+        },
+        deploy: { type: "kubernetes", cluster },
+        env: {
+          dev2: {
+            type: "dev",
+          },
+          review2: {
+            type: "review",
+          },
+          stage2: {
+            type: "stage",
+          },
+          prod2: {
+            type: "prod",
+          },
+        },
+      },
+      app3: {
+        dir: "dir2",
+        build: {
+          type: "node",
+        },
+        deploy: { type: "kubernetes", cluster },
+        env: {
+          dev: false,
+          review: false,
+          stage: false,
+          prod: false,
+        },
+      },
+    },
+  };
+
+  it("should return all envs for SIMPLE_CONFIG1", () => {
+    expect(getPipelineStages(SIMPLE_CONFIG1)).toMatchSnapshot();
+  });
+});

package/src/pipeline/createAllJobs.ts

@@ -1,38 +1,72 @@
-import { getAllEnvsByTrigger } from "../config";
-import type { Config, PipelineTrigger } from "../types";
+import type { CreateAllComponentsContextProps } from "../context/createAllComponentsContext";
+import { createAllComponentsContext } from "../context/createAllComponentsContext";
+import { createWorkspaceContext } from "../context/createWorkspaceContext";
+import type { ComponentContext, WorkspaceContext } from "../types";
+import { componentContextHasWorkspaceBuild } from "../types";
 import type { CatladderJob } from "../types/jobs";
-import { createJobsForComponent } from "./createJobsForComponent";
+import { createJobsForComponentContext } from "./createJobsForComponent";
+import { createJobsForWorkspace } from "./createJobsForWorkspace";
 
 export type AllCatladderJobs = {
-  [componentName: string]: {
-    [env: string]: Array<CatladderJob>;
-  };
+  workspaces: Array<{
+    context: WorkspaceContext;
+    jobs: Array<CatladderJob>;
+  }>;
+  components: Array<{
+    context: ComponentContext;
+    jobs: Array<CatladderJob>;
+  }>;
 };
-export const createAllJobs = async (
-  config: Config,
-  trigger: PipelineTrigger
-): Promise<AllCatladderJobs> => {
-  return Object.fromEntries(
-    await Promise.all(
-      Object.keys(config.components).map(async (componentName) => {
-        const envs = getAllEnvsByTrigger(config, componentName, trigger);
-        return [
-          componentName,
-          Object.fromEntries(
-            await Promise.all(
-              envs.map(async (env) => [
-                env,
-                await createJobsForComponent(
-                  config,
-                  componentName,
-                  env,
-                  trigger
-                ),
-              ])
-            )
-          ),
-        ];
-      })
-    )
-  );
+
+export const createAllJobs = async ({
+  config,
+  trigger,
+  pipelineType,
+}: CreateAllComponentsContextProps): Promise<AllCatladderJobs> => {
+  const allComponentsContext = await createAllComponentsContext({
+    config,
+    trigger,
+    pipelineType,
+  });
+
+  return {
+    workspaces: await Promise.all(
+      Object.keys(config.builds ?? {}).map(async (workspaceName) => {
+        const componentsInAllEnvs = allComponentsContext.filter(
+          (context) =>
+            componentContextHasWorkspaceBuild(context) &&
+            context.build.workspaceName === workspaceName,
+        );
+
+        const allEnvs = [...new Set(componentsInAllEnvs.map(({ env }) => env))];
+
+        return await Promise.all(
+          allEnvs.map(async (env) => {
+            const workspaceContext = await createWorkspaceContext({
+              components: componentsInAllEnvs.filter(
+                ({ env: componentEnv }) => componentEnv === env,
+              ),
+              workspaceName,
+              config,
+              trigger,
+              pipelineType,
+              env,
+            });
+
+            return {
+              context: workspaceContext,
+              jobs: createJobsForWorkspace(workspaceContext),
+            };
+          }),
+        );
+      }),
+    ).then((f) => f.flat()),
+
+    components: allComponentsContext.map((context) => {
+      return {
+        context,
+        jobs: createJobsForComponentContext(context),
+      };
+    }),
+  };
 };

package/src/pipeline/createJobsForComponent.ts

@@ -1,16 +1,14 @@
-import { isFunction } from "lodash";
 import { BUILD_TYPES } from "../build";
-import { createContext } from "../context";
 import { DEPLOY_TYPES } from "../deploy";
-import type { Config, PipelineTrigger } from "../types/config";
-import type { CommitInfo, Context } from "../types/context";
+import type {
+  ComponentContext,
+  ComponentContextWithBuild,
+} from "../types/context";
 import type { CatladderJob } from "../types/jobs";
-import { getBaseCommitInfo } from "./commitInfo/getCommitInfo";
-import { getPackageManagerInfo } from "./packageManager";
 
 const injectDefaultVarsInCustomJobs = (
-  context: Context,
-  jobs: CatladderJob[]
+  context: ComponentContext,
+  jobs: CatladderJob[],
 ) =>
   jobs.map(({ variables, ...job }) => ({
     variables: {
@@ -19,18 +17,22 @@ const injectDefaultVarsInCustomJobs = (
     },
     ...job,
   }));
-const getCustomJobs = (context: Context) => {
-  if (!context.componentConfig.customJobs) {
+const getCustomJobs = (context: ComponentContext) => {
+  if (!context.customJobs) {
     return [];
   }
-  const rawJobs = isFunction(context.componentConfig.customJobs)
-    ? context.componentConfig.customJobs(context)
-    : context.componentConfig.customJobs;
+  const rawJobs = context.customJobs;
   return injectDefaultVarsInCustomJobs(context, rawJobs);
 };
-const createRawJobs = (context: Context): CatladderJob[] => {
+export const createJobsForComponentContext = (
+  context: ComponentContext,
+): CatladderJob[] => {
   const buildJobs =
-    BUILD_TYPES[context.componentConfig.build.type].jobs(context);
+    context.build.type !== "disabled"
+      ? BUILD_TYPES[context.build.buildType].jobs(
+          context as ComponentContextWithBuild,
+        )
+      : [];
   const deployJobs =
     context.componentConfig.deploy !== false
       ? DEPLOY_TYPES[context.componentConfig.deploy.type].jobs(context)
@@ -39,25 +41,3 @@ const createRawJobs = (context: Context): CatladderJob[] => {
   const customJobs = getCustomJobs(context);
   return [...buildJobs, ...deployJobs, ...customJobs];
 };
-export const createJobsForComponent = async (
-  config: Config,
-  componentName: string,
-  env: string,
-  trigger: PipelineTrigger
-): Promise<Array<CatladderJob>> => {
-  const commitInfo: CommitInfo = {
-    ...(await getBaseCommitInfo()),
-    trigger,
-  };
-
-  const packageManagerInfo = await getPackageManagerInfo(config, componentName);
-
-  const context = await createContext(
-    config,
-    componentName,
-    env,
-    commitInfo,
-    packageManagerInfo
-  );
-  return createRawJobs(context);
-};

package/src/pipeline/createJobsForWorkspace.ts

@@ -0,0 +1,12 @@
+import { WORKSPACE_BUILD_TYPES } from "../build";
+import type { WorkspaceContext } from "../types/context";
+import type { CatladderJob } from "../types/jobs";
+
+export const createJobsForWorkspace = (
+  context: WorkspaceContext,
+): CatladderJob[] => {
+  const buildJobs =
+    WORKSPACE_BUILD_TYPES[context.build.buildType].jobs(context);
+
+  return buildJobs;
+};

package/src/pipeline/createMainPipeline.ts

@@ -0,0 +1,118 @@
+import {
+  RULE_IS_MAIN_BRANCH_AND_NOT_RELEASE_COMMIT,
+  RULE_IS_MERGE_REQUEST,
+  RULE_IS_TAGGED_RELEASE,
+} from "../rules";
+import type {
+  ComponentContext,
+  GitlabRule,
+  Pipeline,
+  PipelineTrigger,
+  PipelineType,
+  WorkspaceContext,
+} from "../types";
+import { ALL_PIPELINE_TRIGGERS, type Config } from "../types/config";
+import { createAllJobs } from "./createAllJobs";
+import { getPipelineStages } from "./getPipelineStages";
+import type { GitlabJobWithContext } from "./gitlab/createGitlabJobs";
+import { createGitlabJobs } from "./gitlab/createGitlabJobs";
+import { createGitlabPipelineWithDefaults } from "./gitlab/createGitlabPipeline";
+import { getGitlabReleaseJobs } from "./gitlab/gitlabReleaseJobs";
+
+export const createMainPipeline = async <T extends PipelineType>(
+  pipelineType: T,
+  config: Config,
+): Promise<Pipeline<T>> => {
+  const stages = getPipelineStages(config);
+
+  if (pipelineType === "gitlab") {
+    // for all triggers create jobs and add base rules
+
+    const allJobsPerTrigger = await Promise.all(
+      ALL_PIPELINE_TRIGGERS.map(
+        async (trigger) =>
+          await createGitlabJobs(
+            await createAllJobs({ config, trigger, pipelineType }),
+            getGitlabRulesForTrigger(trigger),
+          ),
+      ),
+    ).then((j) => j.flat());
+
+    const allWorkspaceJobs = allJobsPerTrigger
+      .filter((j) => j.context?.type === "workspace") // sort by componentName in the same order as they appear in the config
+      // this is purely for better readability in git diffs when you add new components
+      .sort((a, b) => {
+        const workspaceNames = Object.keys(config.builds ?? {});
+        const aIndex = workspaceNames.findIndex(
+          (c) => c === (a.context as WorkspaceContext).name,
+        );
+        const bIndex = workspaceNames.findIndex(
+          (c) => c === (b.context as WorkspaceContext).name,
+        );
+        return aIndex - bIndex;
+      });
+
+    const allComponentJobs = allJobsPerTrigger
+      .filter((j) => j.context?.type === "component")
+      // sort by componentName in the same order as they appear in the config
+      // this is purely for better readability in git diffs when you add new components
+      .sort((a, b) => {
+        const componentNames = Object.keys(config.components);
+        const aIndex = componentNames.findIndex(
+          (c) => c === (a.context as ComponentContext).name,
+        );
+        const bIndex = componentNames.findIndex(
+          (c) => c === (b.context as ComponentContext).name,
+        );
+        return aIndex - bIndex;
+      });
+    const allJobs = [...allWorkspaceJobs, ...allComponentJobs].reduce(
+      (acc, { gitlabJob, name, context }) => {
+        // merge jobs, if a job is already there, merge the rules
+        // this is currently needed because of envMode: "none", which creates the same job for all triggers, so it can appear multiple times
+        if (acc[name]) {
+          acc[name].gitlabJob.rules = [
+            ...(acc[name].gitlabJob.rules ?? []),
+            ...(gitlabJob.rules ?? []),
+          ];
+        } else {
+          acc[name] = { context, gitlabJob };
+        }
+
+        return acc;
+      },
+      {} as { [key: string]: GitlabJobWithContext },
+    );
+
+    return createGitlabPipelineWithDefaults({
+      stages: [...stages, "release"],
+      jobs: {
+        ...allJobs,
+        ...Object.fromEntries(
+          Object.entries(getGitlabReleaseJobs()).map(([name, gitlabJob]) => [
+            name,
+            {
+              gitlabJob,
+              context: null,
+            },
+          ]),
+        ),
+      },
+      variables: config.runnerVariables,
+    }) as Pipeline<T>;
+  }
+  throw new Error(`${pipelineType} is not supported`);
+};
+function getGitlabRulesForTrigger(trigger: PipelineTrigger): GitlabRule[] {
+  // mainBranch: on push to main branch except it's a release commit
+  // mr: on merge request
+  // taggedRelease: on tag
+  switch (trigger) {
+    case "mainBranch":
+      return [RULE_IS_MAIN_BRANCH_AND_NOT_RELEASE_COMMIT];
+    case "mr":
+      return [RULE_IS_MERGE_REQUEST];
+    case "taggedRelease":
+      return [RULE_IS_TAGGED_RELEASE];
+  }
+}

package/src/pipeline/generatePipelineFiles.ts

@@ -0,0 +1,100 @@
+import { mkdir, rm } from "fs/promises";
+import { dirname } from "path";
+import type { Config, GitlabJobDef, PipelineType } from "../types";
+import { writeYamlfile } from "../utils/writeFiles";
+import { createMainPipeline } from "./createMainPipeline";
+import { sortGitLabJobDefProps } from "./gitlab/sortGitLabJobDefProps";
+
+const CATLADDER_GENERATED_FOLDER = ".catladder-generated";
+
+const GITLAB_GENERATED_FOLDER = CATLADDER_GENERATED_FOLDER + "/gitlab";
+
+type YamlFile = {
+  path: string;
+  content: Record<string, unknown>;
+};
+export async function generatePipelineFiles<T extends PipelineType>(
+  config: Config,
+  pipelineType: T,
+) {
+  if (pipelineType !== "gitlab") {
+    throw new Error("Pipeline type not supported");
+  }
+  const includes = await getGitlabPipelineIncludes(config);
+
+  const mainFile: YamlFile = {
+    path: ".gitlab-ci.yml",
+    content: {
+      include: includes.map((i) => i.path),
+    },
+  };
+
+  const files = [mainFile, ...includes];
+
+  // first clean up the folder
+  await rm(GITLAB_GENERATED_FOLDER, { force: true, recursive: true });
+  // write files
+  await Promise.all(
+    files.map(async ({ path, content }) => {
+      await mkdir(dirname(path), { recursive: true });
+      await writeYamlfile(path, content);
+    }),
+  );
+}
+async function getGitlabPipelineIncludes(config: Config) {
+  const { jobs, image, stages, variables, workflow, ...pipelineRest } =
+    await createMainPipeline("gitlab", config);
+  // we will create 1 include per component or workspace
+  // this is for better readability in git diffs and to avoid problems with yaml files beeing too large
+  // group by context
+  const groups = Object.entries(jobs).reduce(
+    (acc, [jobName, { gitlabJob, context }]) => {
+      const group = !context
+        ? "global-jobs"
+        : context?.type + "/" + context.name;
+      if (!acc[group]) {
+        acc[group] = {};
+      }
+
+      acc[group][jobName] = sortGitLabJobDefProps(gitlabJob); // also sort properties for more consistent diffing
+      return acc;
+    },
+    {} as Record<string, Record<string, GitlabJobDef>>,
+  );
+
+  const componentIncludes = Object.entries(groups).map(([group, jobs]) => {
+    return {
+      path: GITLAB_GENERATED_FOLDER + "/" + group + ".yaml",
+      content: jobs,
+    };
+  });
+
+  const mainInclude: YamlFile = {
+    path: GITLAB_GENERATED_FOLDER + "/main.yaml",
+    content: {
+      image,
+      stages,
+      variables,
+      workflow,
+      ...pipelineRest,
+    },
+  };
+
+  const includes = [mainInclude, ...componentIncludes];
+  return includes;
+}
+
+/**
+ *
+ * for testing purposes
+ */
+export async function getGitlabCompletePipeline(config: Config) {
+  const includes = await getGitlabPipelineIncludes(config);
+
+  return includes.reduce((acc, { content }) => {
+    return {
+      ...acc,
+      ...content, // merge all includes into one object
+    };
+  }, {});
+}

package/src/pipeline/getPipelineStages.ts

@@ -0,0 +1,23 @@
+import { getAllEnvsInAllComponents } from "../config";
+import type { Config } from "../types/config";
+import { BASE_STAGES } from "../types/jobs";
+
+/**
+ * while technically not required, we group different envs in its own stage
+ * each job from `createJobs` that is defined as `envMode: "stagePerEnv"` will have `deploy dev`, etc. instead of just `deploy`
+ * this is just so that it looks nicer in gitlab and makes running mutliple manual tasks more easy to use
+ */
+export function getPipelineStages(config: Config) {
+  const allEnvs = getAllEnvsInAllComponents(config).filter(
+    (e) => e !== "local",
+  );
+  const stages = BASE_STAGES.reduce<string[]>(
+    (acc, baseStage) => [
+      ...acc,
+      baseStage,
+      ...allEnvs.map((e) => `${baseStage} ${e}`),
+    ],
+    [],
+  );
+  return stages;
+}