@catladder/pipeline 0.0.0-more-collapse-deeb54d7 → 0.0.0-multiple-gitlab-includes-1a40efc0

package/src/pipeline/createJobsForComponent.ts

@@ -1,16 +1,11 @@
-import { isFunction } from "lodash";
 import { BUILD_TYPES } from "../build";
-import { createContext } from "../context";
 import { DEPLOY_TYPES } from "../deploy";
-import type { Config, PipelineTrigger } from "../types/config";
-import type { CommitInfo, Context } from "../types/context";
+import type { ComponentContext } from "../types/context";
 import type { CatladderJob } from "../types/jobs";
-import { getBaseCommitInfo } from "./commitInfo/getCommitInfo";
-import { getPackageManagerInfo } from "./packageManager";
 
 const injectDefaultVarsInCustomJobs = (
-  context: Context,
-  jobs: CatladderJob[]
+  context: ComponentContext,
+  jobs: CatladderJob[],
 ) =>
   jobs.map(({ variables, ...job }) => ({
     variables: {
@@ -19,18 +14,17 @@ const injectDefaultVarsInCustomJobs = (
     },
     ...job,
   }));
-const getCustomJobs = (context: Context) => {
-  if (!context.componentConfig.customJobs) {
+const getCustomJobs = (context: ComponentContext) => {
+  if (!context.customJobs) {
     return [];
   }
-  const rawJobs = isFunction(context.componentConfig.customJobs)
-    ? context.componentConfig.customJobs(context)
-    : context.componentConfig.customJobs;
+  const rawJobs = context.customJobs;
   return injectDefaultVarsInCustomJobs(context, rawJobs);
 };
-const createRawJobs = (context: Context): CatladderJob[] => {
-  const buildJobs =
-    BUILD_TYPES[context.componentConfig.build.type].jobs(context);
+export const createJobsForComponentContext = (
+  context: ComponentContext,
+): CatladderJob[] => {
+  const buildJobs = BUILD_TYPES[context.build.buildType].jobs(context);
   const deployJobs =
     context.componentConfig.deploy !== false
       ? DEPLOY_TYPES[context.componentConfig.deploy.type].jobs(context)
@@ -39,25 +33,3 @@ const createRawJobs = (context: Context): CatladderJob[] => {
   const customJobs = getCustomJobs(context);
   return [...buildJobs, ...deployJobs, ...customJobs];
 };
-export const createJobsForComponent = async (
-  config: Config,
-  componentName: string,
-  env: string,
-  trigger: PipelineTrigger
-): Promise<Array<CatladderJob>> => {
-  const commitInfo: CommitInfo = {
-    ...(await getBaseCommitInfo()),
-    trigger,
-  };
-
-  const packageManagerInfo = await getPackageManagerInfo(config, componentName);
-
-  const context = await createContext(
-    config,
-    componentName,
-    env,
-    commitInfo,
-    packageManagerInfo
-  );
-  return createRawJobs(context);
-};

package/src/pipeline/createJobsForWorkspace.ts

@@ -0,0 +1,12 @@
+import { WORKSPACE_BUILD_TYPES } from "../build";
+import type { WorkspaceContext } from "../types/context";
+import type { CatladderJob } from "../types/jobs";
+
+export const createJobsForWorkspace = (
+  context: WorkspaceContext,
+): CatladderJob[] => {
+  const buildJobs =
+    WORKSPACE_BUILD_TYPES[context.build.buildType].jobs(context);
+
+  return buildJobs;
+};

package/src/pipeline/createMainPipeline.ts

@@ -0,0 +1,119 @@
+import {
+  RULE_IS_MAIN_BRANCH,
+  RULE_IS_MERGE_REQUEST,
+  RULE_IS_TAGGED_RELEASE,
+  RULE_NEVER_ON_RELEASE_COMMIT,
+} from "../rules";
+import type {
+  ComponentContext,
+  GitlabRule,
+  Pipeline,
+  PipelineTrigger,
+  PipelineType,
+  WorkspaceContext,
+} from "../types";
+import { ALL_PIPELINE_TRIGGERS, type Config } from "../types/config";
+import { createAllJobs } from "./createAllJobs";
+import { getPipelineStages } from "./getPipelineStages";
+import type { GitlabJobWithContext } from "./gitlab/createGitlabJobs";
+import { createGitlabJobs } from "./gitlab/createGitlabJobs";
+import { createGitlabPipelineWithDefaults } from "./gitlab/createGitlabPipeline";
+import { getGitlabReleaseJobs } from "./gitlab/gitlabReleaseJobs";
+
+export const createMainPipeline = async <T extends PipelineType>(
+  pipelineType: T,
+  config: Config,
+): Promise<Pipeline<T>> => {
+  const stages = getPipelineStages(config);
+
+  if (pipelineType === "gitlab") {
+    // for all triggers create jobs and add base rules
+
+    const allJobsPerTrigger = await Promise.all(
+      ALL_PIPELINE_TRIGGERS.map(
+        async (trigger) =>
+          await createGitlabJobs(
+            await createAllJobs({ config, trigger, pipelineType }),
+            getGitlabRulesForTrigger(trigger),
+          ),
+      ),
+    ).then((j) => j.flat());
+
+    const allWorkspaceJobs = allJobsPerTrigger
+      .filter((j) => j.context?.type === "workspace") // sort by componentName in the same order as they appear in the config
+      // this is purely for better readability in git diffs when you add new components
+      .sort((a, b) => {
+        const workspaceNames = Object.keys(config.builds ?? {});
+        const aIndex = workspaceNames.findIndex(
+          (c) => c === (a.context as WorkspaceContext).name,
+        );
+        const bIndex = workspaceNames.findIndex(
+          (c) => c === (b.context as WorkspaceContext).name,
+        );
+        return aIndex - bIndex;
+      });
+
+    const allComponentJobs = allJobsPerTrigger
+      .filter((j) => j.context?.type === "component")
+      // sort by componentName in the same order as they appear in the config
+      // this is purely for better readability in git diffs when you add new components
+      .sort((a, b) => {
+        const componentNames = Object.keys(config.components);
+        const aIndex = componentNames.findIndex(
+          (c) => c === (a.context as ComponentContext).name,
+        );
+        const bIndex = componentNames.findIndex(
+          (c) => c === (b.context as ComponentContext).name,
+        );
+        return aIndex - bIndex;
+      });
+    const allJobs = [...allWorkspaceJobs, ...allComponentJobs].reduce(
+      (acc, { gitlabJob, name, context }) => {
+        // merge jobs, if a job is already there, merge the rules
+        // this is currently needed because of envMode: "none", which creates the same job for all triggers, so it can appear multiple times
+        if (acc[name]) {
+          acc[name].gitlabJob.rules = [
+            ...(acc[name].gitlabJob.rules ?? []),
+            ...(gitlabJob.rules ?? []),
+          ];
+        } else {
+          acc[name] = { context, gitlabJob };
+        }
+
+        return acc;
+      },
+      {} as { [key: string]: GitlabJobWithContext },
+    );
+
+    return createGitlabPipelineWithDefaults({
+      stages: [...stages, "release"],
+      jobs: {
+        ...allJobs,
+        ...Object.fromEntries(
+          Object.entries(getGitlabReleaseJobs()).map(([name, gitlabJob]) => [
+            name,
+            {
+              gitlabJob,
+              context: null,
+            },
+          ]),
+        ),
+      },
+      variables: config.runnerVariables,
+    }) as Pipeline<T>;
+  }
+  throw new Error(`${pipelineType} is not supported`);
+};
+function getGitlabRulesForTrigger(trigger: PipelineTrigger): GitlabRule[] {
+  // mainBranch: on push to main branch
+  // mr: on merge request
+  // taggedRelease: on tag
+  switch (trigger) {
+    case "mainBranch":
+      return [RULE_NEVER_ON_RELEASE_COMMIT, RULE_IS_MAIN_BRANCH];
+    case "mr":
+      return [RULE_IS_MERGE_REQUEST];
+    case "taggedRelease":
+      return [RULE_IS_TAGGED_RELEASE];
+  }
+}

package/src/pipeline/generatePipelineFiles.ts

@@ -0,0 +1,100 @@
+import { mkdir, rm } from "fs/promises";
+import { dirname } from "path";
+import type { Config, GitlabJobDef, PipelineType } from "../types";
+import { writeYamlfile } from "../utils/writeFiles";
+import { createMainPipeline } from "./createMainPipeline";
+import { sortGitLabJobDefProps } from "./gitlab/sortGitLabJobDefProps";
+
+const CATLADDER_GENERATED_FOLDER = ".catladder-generated";
+
+const GITLAB_GENERATED_FOLDER = CATLADDER_GENERATED_FOLDER + "/gitlab";
+
+type YamlFile = {
+  path: string;
+  content: Record<string, unknown>;
+};
+export async function generatePipelineFiles<T extends PipelineType>(
+  config: Config,
+  pipelineType: T,
+) {
+  if (pipelineType !== "gitlab") {
+    throw new Error("Pipeline type not supported");
+  }
+  const includes = await getGitlabPipelineIncludes(config);
+
+  const mainFile: YamlFile = {
+    path: ".gitlab-ci.yml",
+    content: {
+      include: includes.map((i) => i.path),
+    },
+  };
+
+  const files = [mainFile, ...includes];
+
+  // first clean up the folder
+  await rm(GITLAB_GENERATED_FOLDER, { recursive: true });
+  // write files
+  await Promise.all(
+    files.map(async ({ path, content }) => {
+      await mkdir(dirname(path), { recursive: true });
+      await writeYamlfile(path, content);
+    }),
+  );
+}
+async function getGitlabPipelineIncludes(config: Config) {
+  const { jobs, image, stages, variables, workflow, ...pipelineRest } =
+    await createMainPipeline("gitlab", config);
+  // we will create 1 include per component or workspace
+  // this is for better readability in git diffs and to avoid problems with yaml files beeing too large
+  // group by context
+  const groups = Object.entries(jobs).reduce(
+    (acc, [jobName, { gitlabJob, context }]) => {
+      const group = !context
+        ? "global-jobs"
+        : context?.type + "/" + context.name;
+      if (!acc[group]) {
+        acc[group] = {};
+      }
+
+      acc[group][jobName] = sortGitLabJobDefProps(gitlabJob); // also sort properties for more consistent diffing
+      return acc;
+    },
+    {} as Record<string, Record<string, GitlabJobDef>>,
+  );
+
+  const componentIncludes = Object.entries(groups).map(([group, jobs]) => {
+    return {
+      path: GITLAB_GENERATED_FOLDER + "/" + group + ".yaml",
+      content: jobs,
+    };
+  });
+
+  const mainInclude: YamlFile = {
+    path: GITLAB_GENERATED_FOLDER + "/main.yaml",
+    content: {
+      image,
+      stages,
+      variables,
+      workflow,
+      ...pipelineRest,
+    },
+  };
+
+  const includes = [mainInclude, ...componentIncludes];
+  return includes;
+}
+
+/**
+ *
+ * for testing purposes
+ */
+export async function getGitlabCompletePipeline(config: Config) {
+  const includes = await getGitlabPipelineIncludes(config);
+
+  return includes.reduce((acc, { content }) => {
+    return {
+      ...acc,
+      ...content, // merge all includes into one object
+    };
+  }, {});
+}

package/src/pipeline/getPipelineStages.ts

@@ -0,0 +1,23 @@
+import { getAllEnvsInAllComponents } from "../config";
+import type { Config } from "../types/config";
+import { BASE_STAGES } from "../types/jobs";
+
+/**
+ * while technically not required, we group different envs in its own stage
+ * each job from `createJobs` that is defined as `envMode: "stagePerEnv"` will have `deploy dev`, etc. instead of just `deploy`
+ * this is just so that it looks nicer in gitlab and makes running mutliple manual tasks more easy to use
+ */
+export function getPipelineStages(config: Config) {
+  const allEnvs = getAllEnvsInAllComponents(config).filter(
+    (e) => e !== "local",
+  );
+  const stages = BASE_STAGES.reduce<string[]>(
+    (acc, baseStage) => [
+      ...acc,
+      baseStage,
+      ...allEnvs.map((e) => `${baseStage} ${e}`),
+    ],
+    [],
+  );
+  return stages;
+}