@catladder/pipeline 1.144.1 → 1.146.0

package/src/pipeline/createAllJobs.ts

@@ -1,17 +1,29 @@
 import { getAllEnvsByTrigger } from "../config";
-import type { Config, PipelineTrigger } from "../types";
-import type { CatladderJob } from "../types/jobs";
+import type {
+  CatladderJobWithContext,
+  Config,
+  PipelineTrigger,
+  PipelineType,
+} from "../types";
 import { createJobsForComponent } from "./createJobsForComponent";
 
 export type AllCatladderJobs = {
   [componentName: string]: {
-    [env: string]: Array<CatladderJob>;
+    [env: string]: Array<CatladderJobWithContext>;
   };
 };
-export const createAllJobs = async (
-  config: Config,
-  trigger: PipelineTrigger
-): Promise<AllCatladderJobs> => {
+
+export type AllJobsContext = {
+  config: Config;
+  trigger: PipelineTrigger;
+  pipelineType: PipelineType;
+};
+
+export const createAllJobs = async ({
+  config,
+  trigger,
+  pipelineType,
+}: AllJobsContext): Promise<AllCatladderJobs> => {
   return Object.fromEntries(
     await Promise.all(
       Object.keys(config.components).map(async (componentName) => {
@@ -22,17 +34,19 @@ export const createAllJobs = async (
             await Promise.all(
               envs.map(async (env) => [
                 env,
-                await createJobsForComponent(
+
+                await createJobsForComponent({
                   config,
                   componentName,
                   env,
-                  trigger
-                ),
-              ])
-            )
+                  trigger,
+                  pipelineType,
+                }),
+              ]),
+            ),
           ),
         ];
-      })
-    )
+      }),
+    ),
   );
 };

package/src/pipeline/createChildPipeline.ts

@@ -1,24 +1,35 @@
+import { RULES_ALWAYS } from "../rules";
+
 import type { Pipeline, PipelineType } from "../types";
 import type { Config, PipelineTrigger } from "../types/config";
 import { createAllJobs } from "./createAllJobs";
 import { getPipelineStages } from "./getPipelineStages";
 import { createGitlabJobs } from "./gitlab/createGitlabJobs";
-import { createGitlabPipelineFromStagesAndJobs } from "./gitlab/createGitlabPipeline";
+import { createGitlabPipelineWithDefaults } from "./gitlab/createGitlabPipeline";
 
 export const createChildPipeline = async <T extends PipelineType>(
-  type: T,
+  pipelineType: T,
   trigger: PipelineTrigger,
-  config: Config
+  config: Config,
 ): Promise<Pipeline<T>> => {
-  const jobs = await createAllJobs(config, trigger);
+  const jobs = await createAllJobs({
+    config,
+    trigger,
+    pipelineType,
+  });
   const stages = getPipelineStages(config);
 
-  if (type === "gitlab") {
+  if (pipelineType === "gitlab") {
     const gitlabJobs = await createGitlabJobs(jobs);
-    return createGitlabPipelineFromStagesAndJobs(
+    return createGitlabPipelineWithDefaults({
+      workflow: {
+        rules: RULES_ALWAYS,
+      },
       stages,
-      gitlabJobs
-    ) as Pipeline<T>;
+      jobs: Object.fromEntries(
+        gitlabJobs.map(({ gitlabJob, name }) => [name, gitlabJob]),
+      ),
+    }) as Pipeline<T>;
   }
-  throw new Error(`${type} is not supported`);
+  throw new Error(`${pipelineType} is not supported`);
 };

package/src/pipeline/createJobsForComponent.ts

@@ -1,16 +1,15 @@
 import { isFunction } from "lodash";
 import { BUILD_TYPES } from "../build";
+import type { CreateContextContext } from "../context";
 import { createContext } from "../context";
 import { DEPLOY_TYPES } from "../deploy";
-import type { Config, PipelineTrigger } from "../types/config";
-import type { CommitInfo, Context } from "../types/context";
+import type { CatladderJobWithContext, Context } from "../types/context";
 import type { CatladderJob } from "../types/jobs";
-import { getBaseCommitInfo } from "./commitInfo/getCommitInfo";
 import { getPackageManagerInfo } from "./packageManager";
 
 const injectDefaultVarsInCustomJobs = (
   context: Context,
-  jobs: CatladderJob[]
+  jobs: CatladderJob[],
 ) =>
   jobs.map(({ variables, ...job }) => ({
     variables: {
@@ -39,25 +38,18 @@ const createRawJobs = (context: Context): CatladderJob[] => {
   const customJobs = getCustomJobs(context);
   return [...buildJobs, ...deployJobs, ...customJobs];
 };
-export const createJobsForComponent = async (
-  config: Config,
-  componentName: string,
-  env: string,
-  trigger: PipelineTrigger
-): Promise<Array<CatladderJob>> => {
-  const commitInfo: CommitInfo = {
-    ...(await getBaseCommitInfo()),
-    trigger,
-  };
-
-  const packageManagerInfo = await getPackageManagerInfo(config, componentName);
 
-  const context = await createContext(
-    config,
-    componentName,
-    env,
-    commitInfo,
-    packageManagerInfo
+export const createJobsForComponent = async (
+  contextContext: Omit<CreateContextContext, "packageManagerInfo">,
+): Promise<Array<CatladderJobWithContext>> => {
+  const packageManagerInfo = await getPackageManagerInfo(
+    contextContext.config,
+    contextContext.componentName,
   );
-  return createRawJobs(context);
+
+  const context = await createContext({
+    ...contextContext,
+    packageManagerInfo,
+  });
+  return createRawJobs(context).map((job) => ({ ...job, context }));
 };

package/src/pipeline/createMainPipeline.ts

@@ -0,0 +1,91 @@
+import {
+  RULE_IS_MAIN_BRANCH,
+  RULE_IS_MERGE_REQUEST,
+  RULE_IS_TAGGED_RELEASE,
+  RULE_NEVER_ON_RELEASE_COMMIT,
+} from "../rules";
+import type {
+  GitlabJobDef,
+  GitlabRule,
+  Pipeline,
+  PipelineTrigger,
+  PipelineType,
+} from "../types";
+import { ALL_PIPELINE_TRIGGERS, type Config } from "../types/config";
+import { createAllJobs } from "./createAllJobs";
+import { getPipelineStages } from "./getPipelineStages";
+import { createGitlabJobs } from "./gitlab/createGitlabJobs";
+import { createGitlabPipelineWithDefaults } from "./gitlab/createGitlabPipeline";
+import { getGitlabReleaseJobs } from "./gitlab/gitlabReleaseJobs";
+
+export const createMainPipeline = async <T extends PipelineType>(
+  pipelineType: T,
+  config: Config,
+): Promise<Pipeline<T>> => {
+  const stages = getPipelineStages(config);
+
+  if (pipelineType === "gitlab") {
+    // for all triggers create jobs and add base rules
+
+    const allJobsPerTrigger = await Promise.all(
+      ALL_PIPELINE_TRIGGERS.map(
+        async (trigger) =>
+          await createGitlabJobs(
+            await createAllJobs({ config, trigger, pipelineType }),
+            getGitlabRulesForTrigger(trigger),
+          ),
+      ),
+    );
+
+    const allJobs = allJobsPerTrigger
+      .flat()
+      // sort by componentName in the same order as they appear in the config
+      // this is purely for better readability in git diffs when you add new components
+      .sort((a, b) => {
+        const componentNames = Object.keys(config.components);
+        const aIndex = componentNames.findIndex((c) => c === a.componentName);
+        const bIndex = componentNames.findIndex((c) => c === b.componentName);
+        return aIndex - bIndex;
+      })
+
+      .reduce(
+        (acc, { gitlabJob, name }) => {
+          // merge jobs, if a job is already there, merge the rules
+          // this is currently needed because of envMode: "none", which creates the same job for all triggers, so it can appear multiple times
+          if (acc[name]) {
+            acc[name].rules = [
+              ...(acc[name].rules ?? []),
+              ...(gitlabJob.rules ?? []),
+            ];
+          } else {
+            acc[name] = gitlabJob;
+          }
+
+          return acc;
+        },
+        {} as { [key: string]: GitlabJobDef },
+      );
+
+    return createGitlabPipelineWithDefaults({
+      stages: [...stages, "release"],
+      jobs: {
+        ...allJobs,
+        ...getGitlabReleaseJobs(),
+      },
+    }) as Pipeline<T>;
+  }
+  throw new Error(`${pipelineType} is not supported`);
+};
+function getGitlabRulesForTrigger(trigger: PipelineTrigger): GitlabRule[] {
+  // mainBranch: on push to main branch
+  // mr: on merge request
+  // taggedRelease: on tag
+  switch (trigger) {
+    case "mainBranch":
+      return [RULE_NEVER_ON_RELEASE_COMMIT, RULE_IS_MAIN_BRANCH];
+    case "mr":
+      return [RULE_IS_MERGE_REQUEST];
+    case "taggedRelease":
+      return [RULE_IS_TAGGED_RELEASE];
+  }
+}

package/src/pipeline/generatePipelineFiles.ts

@@ -0,0 +1,29 @@
+import type { Config, PipelineMode, PipelineType } from "../types";
+import { writeYamlfile } from "../utils/writeFiles";
+import { createChildPipeline } from "./createChildPipeline";
+import { createMainPipeline } from "./createMainPipeline";
+import { getPipelineTriggerForGitlabChildPipeline } from "./gitlab/getPipelineTriggerForGitlabChildPipeline";
+export async function generatePipelineFiles<T extends PipelineType>(
+  config: Config,
+  pipelineType: T,
+  mode: PipelineMode<T>,
+) {
+  if (mode === "childpipeline") {
+    const trigger = getPipelineTriggerForGitlabChildPipeline();
+
+    const { jobs, ...mainPipeline } = await createChildPipeline(
+      pipelineType,
+      trigger,
+      config,
+    );
+    // need to spread out the jobs, forgot why
+    await writeYamlfile(`__pipeline.yml`, { ...jobs, ...mainPipeline });
+  } else {
+    const { jobs, ...mainPipeline } = await createMainPipeline(
+      pipelineType,
+      config,
+    );
+    // need to spread out the jobs, forgot why
+    await writeYamlfile(`.gitlab-ci.yml`, { ...jobs, ...mainPipeline });
+  }
+}

package/src/pipeline/getPipelineStages.ts

@@ -16,7 +16,7 @@ export function getPipelineStages(config: Config) {
       baseStage,
       ...allEnvs.map((e) => `${baseStage} ${e}`),
     ],
-    []
+    [],
   );
   return stages;
 }

package/src/pipeline/gitlab/createGitlabJobs.ts

@@ -1,20 +1,33 @@
-import { isObject } from "lodash";
+import { isObject, merge } from "lodash";
+import { getInjectVarsScript } from "../../bash/getInjectVarsScript";
 import { BASE_RETRY } from "../../defaults";
-import type { GitlabJobDef } from "../../types";
+import type {
+  CatladderJobWithContext,
+  Context,
+  GitlabJobDef,
+  GitlabRule,
+} from "../../types";
 import type { CatladderJob, CatladderJobNeed } from "../../types/jobs";
-import type { AllCatladderJobs } from "../createAllJobs";
 import { notNil } from "../../utils";
+import { collapseableSection } from "../../utils/gitlab";
+import type { AllCatladderJobs } from "../createAllJobs";
 
-export type AllGitlabJobs = Record<string, GitlabJobDef>;
+export type AllGitlabJobs = {
+  name: string;
+  gitlabJob: GitlabJobDef;
+  componentName: string;
+  env: string;
+}[];
 
-const removeUndefined = <T extends Record<string, unknown>>(obj: T): T =>
+export const GITLAB_ENVIRONMENT_URL_VARIABLE = "CL_GITLAB_ENVIRONMENT_URL";
+const removeUndefined = (obj: GitlabJobDef): GitlabJobDef =>
   Object.fromEntries(
-    Object.entries(obj).filter(([, value]) => value !== undefined)
-  ) as T;
+    Object.entries(obj).filter(([, value]) => value !== undefined),
+  ) as GitlabJobDef;
 const getFullJobName = (
   name: string,
   componentName: string,
-  env?: string | null
+  env?: string | null,
 ) => {
   if (env) {
     return `${componentName} ${name} | ${env} `;
@@ -26,14 +39,14 @@ const getFullReferencedJobName = (
   referencedJobName: string,
   componentName: string,
   env: string,
-  allJobs: AllCatladderJobs
+  allJobs: AllCatladderJobs,
 ) => {
   const referencedJob = allJobs[componentName]?.[env]?.find(
-    (j) => j.name === referencedJobName
+    (j) => j.name === referencedJobName,
   );
   if (!referencedJob) {
     throw new Error(
-      `unknown job referenced: '${referencedJobName}' from '${env}:${componentName}'`
+      `unknown job referenced: '${referencedJobName}' from '${env}:${componentName}'`,
     );
   }
   const envToSet = referencedJob.envMode !== "none" ? env : null;
@@ -47,6 +60,7 @@ export const makeGitlabJob = (
   componentName: string,
   env: string,
   {
+    environment,
     envMode,
     needsStages,
     needsOtherComponent,
@@ -54,9 +68,14 @@ export const makeGitlabJob = (
     needs,
     jobTags,
     script,
+    context,
+    variables,
+    runnerVariables,
+    when,
     ...job
-  }: CatladderJob<string>,
-  allJobs: AllCatladderJobs
+  }: CatladderJobWithContext<string>,
+  allJobs: AllCatladderJobs,
+  baseRules?: GitlabRule[],
 ): [fullName: string, job: GitlabJobDef] => {
   const stage = envMode === "stagePerEnv" ? `${job.stage} ${env}` : job.stage;
 
@@ -88,11 +107,11 @@ export const makeGitlabJob = (
               n.job,
               n.componentName ?? componentName,
               env,
-              allJobs
+              allJobs,
             ),
             artifacts: n.artifacts,
           }
-        : getFullReferencedJobName(n, componentName, env, allJobs)
+        : getFullReferencedJobName(n, componentName, env, allJobs),
     ) // sort in a predictable manner for snapshot tests
     .sort((a, b) => getJobName(a).localeCompare(getJobName(b)));
 
@@ -103,60 +122,164 @@ export const makeGitlabJob = (
   const fullJobName = getFullJobName(
     name,
     componentName,
-    envMode !== "none" ? env : undefined
+    envMode !== "none" ? env : undefined,
   );
 
-  const gitlabJob: GitlabJobDef = removeUndefined({
-    ...job,
-    script: script?.filter(notNil),
-    tags: jobTags,
-    stage,
-    environment: job.environment?.on_stop
-      ? {
-          ...job.environment,
-          on_stop: getFullReferencedJobName(
-            job.environment.on_stop,
-            componentName,
-            env,
-            allJobs
-          ),
-        }
-      : job.environment,
-    // sort in a predictable manner for snapshot tests
-    needs: deduplicatedGitlabNeeds,
-    retry: BASE_RETRY,
-    interruptible: true,
-  });
+  // backwards compatibility, some may still use KUBERNETES_CPU_REQUEST, KUBERNETES_MEMORY_REQUEST, etc. in variables.
+  // those should now be set in the runnerVariables as they don't work in the variables key of the catladder job, becuase those get injected
+  const PIPELINE_RUNNER_VARIABLES = [
+    "KUBERNETES_CPU_REQUEST",
+    "KUBERNETES_MEMORY_REQUEST",
+    "KUBERNETES_CPU_LIMIT",
+    "KUBERNETES_MEMORY_LIMIT",
+  ];
+  // remove those from variables and add them to runnerVariables
+
+  const varsInjectScripts = collapseableSection(
+    "injectvars",
+    "Injecting variables",
+  )([
+    ...getInjectVarsScript(
+      // remove legacy variables
+      Object.fromEntries(
+        Object.entries(variables ?? {}).filter(
+          ([key]) => !PIPELINE_RUNNER_VARIABLES.includes(key),
+        ),
+      ),
+    ),
+  ]);
+
+  const legacyRunnerVariables = Object.fromEntries(
+    Object.entries(variables ?? {}).filter(([key]) =>
+      PIPELINE_RUNNER_VARIABLES.includes(key),
+    ),
+  );
+
+  if (Object.keys(legacyRunnerVariables).length > 0) {
+    console.warn(
+      `Legacy variables detected in ${fullJobName}: ${Object.keys(
+        legacyRunnerVariables,
+      ).join(", ")}. Please move them to the runnerVariables key.`,
+    );
+  }
+  const rules = [
+    ...(job.rules ?? []),
+    ...(baseRules
+      ? baseRules.map((rule) => ({
+          when: when,
+          ...rule,
+        }))
+      : when
+        ? [{ when }]
+        : []),
+  ];
+  const modified = addGitlabEnvironment(
+    context,
+    environment,
+    {
+      ...job,
+      rules: rules.length > 0 ? rules : undefined,
+      variables: {
+        ...legacyRunnerVariables,
+        ...runnerVariables,
+      },
+      script: [...varsInjectScripts, ...(script?.filter(notNil) ?? [])],
+      tags: jobTags,
+      stage,
+
+      // sort in a predictable manner for snapshot tests
+      needs: deduplicatedGitlabNeeds,
+      retry: BASE_RETRY,
+      interruptible: true,
+    },
+    componentName,
+    env,
+    allJobs,
+  );
+
+  const gitlabJob: GitlabJobDef = removeUndefined(modified);
 
   return [fullJobName, gitlabJob];
 };
 
+const addGitlabEnvironment = (
+  context: Context,
+  environment: CatladderJob["environment"],
+  job: GitlabJobDef,
+  componentName: string,
+  env: string, // TODO: we could actually pull this from contxt
+  allJobs: AllCatladderJobs,
+): GitlabJobDef => {
+  if (!environment) {
+    return job;
+  }
+  const { url, envType } = context.environment;
+  const { on_stop, ...restEnvironment } = environment;
+  // those can be dynamic, so we therefore have to do this: https://docs.gitlab.com/ee/ci/environments/#set-a-dynamic-environment-url
+
+  const dotEnvFile = "gitlab_environment.env";
+
+  const scriptToAdd = [
+    `echo "${GITLAB_ENVIRONMENT_URL_VARIABLE}=${url}" >> ${dotEnvFile}`,
+  ];
+
+  // this is NOT a bashVariable since it NEEDS to be used as a string in gitlab
+  const gitlabEnvironmentName =
+    envType === "review"
+      ? `${env}/$CI_COMMIT_REF_NAME/${componentName}` // FIXME: should be replaced with mr name as well
+      : `${env}/${componentName}`;
+
+  return {
+    ...job,
+    environment: {
+      name: gitlabEnvironmentName,
+      url: `$${GITLAB_ENVIRONMENT_URL_VARIABLE}`,
+      ...(on_stop
+        ? {
+            on_stop: getFullReferencedJobName(
+              on_stop,
+              componentName,
+              env,
+              allJobs,
+            ),
+          }
+        : {}),
+      ...restEnvironment,
+    },
+    artifacts: merge(job.artifacts ?? {}, {
+      reports: {
+        dotenv: `${dotEnvFile}`,
+      },
+    }),
+
+    script: [...(job.script ?? []), ...scriptToAdd],
+  };
+};
+
 export const createGitlabJobs = async (
-  allJobs: AllCatladderJobs
+  allJobs: AllCatladderJobs,
+  baseRules?: GitlabRule[],
 ): Promise<AllGitlabJobs> => {
-  return Object.keys(allJobs).reduce((accForComponents, componentName) => {
+  return Object.keys(allJobs).flatMap((componentName) => {
     const componentJobs = allJobs[componentName];
-    return {
-      ...accForComponents,
-      ...Object.keys(componentJobs).reduce((accForEnvs, env) => {
-        const jobs = componentJobs[env];
+    return Object.keys(componentJobs).flatMap((env) => {
+      const jobs = componentJobs[env];
 
+      return jobs.flatMap((job) => {
+        const [fullJobName, gitlabJob] = makeGitlabJob(
+          componentName,
+          env,
+          job,
+          allJobs,
+          baseRules,
+        );
         return {
-          ...accForEnvs,
-          ...jobs.reduce((accForJobs, job) => {
-            const [fullJobName, gitlabJob] = makeGitlabJob(
-              componentName,
-              env,
-              job,
-              allJobs
-            );
-            return {
-              ...accForJobs,
-              [fullJobName]: gitlabJob,
-            };
-          }, {} as AllGitlabJobs),
+          name: fullJobName,
+          gitlabJob,
+          componentName,
+          env,
         };
-      }, {} as AllGitlabJobs),
-    };
-  }, {} as AllGitlabJobs);
+      });
+    });
+  });
 };

package/src/pipeline/gitlab/createGitlabPipeline.ts

@@ -1,22 +1,24 @@
-import { RULES_ALWAYS } from "../../rules";
 import { getRunnerImage } from "../../runner";
 import type { Pipeline } from "../../types";
-import type { AllGitlabJobs } from "./createGitlabJobs";
 
-export function createGitlabPipelineFromStagesAndJobs(
-  stages: string[],
-  gitlabJobs: AllGitlabJobs
-): Pipeline<"gitlab"> {
+type PickRequired<T, K extends keyof T> = Required<Pick<T, K>> & Omit<T, K>;
+
+export const createGitlabPipelineWithDefaults = ({
+  image,
+  variables,
+  ...config
+}: PickRequired<
+  Partial<Pipeline<"gitlab">>,
+  "stages" | "jobs"
+>): Pipeline<"gitlab"> => {
   return {
-    image: getRunnerImage("jobs-default"), // default image
+    image: image ?? getRunnerImage("jobs-default"), // default image
     variables: {
       FF_USE_FASTZIP: "true",
-      GIT_DEPTH: 1, // no need the full depth
-    },
-    workflow: {
-      rules: RULES_ALWAYS,
+      GIT_DEPTH: "1", // no need the full depth
+      ...(variables ?? {}),
     },
-    stages,
-    jobs: gitlabJobs,
+
+    ...config,
   };
-}
+};

package/src/pipeline/gitlab/getPipelineTriggerForGitlabChildPipeline.ts

@@ -0,0 +1,46 @@
+import type { PipelineTrigger } from "../..";
+import { PIPELINE_IMAGE_TAG } from "../../constants";
+
+export function getPipelineTriggerForGitlabChildPipeline() {
+  const {
+    CI_MERGE_REQUEST_ID,
+    CI_COMMIT_TAG,
+    CI_COMMIT_BRANCH,
+    CI_DEFAULT_BRANCH,
+  } = process.env;
+
+  const isDefaultBranch =
+    Boolean(CI_DEFAULT_BRANCH) && CI_COMMIT_BRANCH === CI_DEFAULT_BRANCH;
+  const isHotfixBranch = CI_COMMIT_BRANCH
+    ? /^[0-9]+\.([0-9]+|x)\.x$/.test(CI_COMMIT_BRANCH)
+    : false;
+  const isMergeRequest = Boolean(CI_MERGE_REQUEST_ID);
+  const isTaggedRelease = Boolean(CI_COMMIT_TAG);
+
+  console.info(`catladder version ${PIPELINE_IMAGE_TAG}`);
+
+  const trigger: PipelineTrigger | null =
+    isMergeRequest || isHotfixBranch
+      ? "mr"
+      : isDefaultBranch
+        ? "mainBranch"
+        : isTaggedRelease
+          ? "taggedRelease"
+          : null;
+
+  if (!trigger) {
+    throw new Error(
+      "no matching trigger: " +
+        JSON.stringify(
+          {
+            isMergeRequest,
+            isDefaultBranch,
+            isTaggedRelease,
+          },
+          null,
+          2,
+        ),
+    );
+  }
+  return trigger;
+}