@catladder/pipeline 0.0.0-more-collapse-deeb54d7 → 0.0.0-multiple-gitlab-includes-1a40efc0

package/src/pipeline/gitlab/createGitlabJobs.ts

@@ -1,162 +1,442 @@
-import { isObject } from "lodash";
+import { isObject, merge } from "lodash";
+import { getInjectVarsScript } from "../../bash/getInjectVarsScript";
 import { BASE_RETRY } from "../../defaults";
-import type { GitlabJobDef } from "../../types";
+import type {
+  ComponentContext,
+  Context,
+  GitlabJobDef,
+  GitlabRule,
+  WorkspaceContext,
+} from "../../types";
 import type { CatladderJob, CatladderJobNeed } from "../../types/jobs";
-import type { AllCatladderJobs } from "../createAllJobs";
 import { notNil } from "../../utils";
+import { collapseableSection } from "../../utils/gitlab";
+import { removeUndefined } from "../../utils/removeUndefined";
+import type { AllCatladderJobs } from "../createAllJobs";
 
-type AllGitlabJobs = Record<string, GitlabJobDef>;
+export type GitlabJobWithContext = {
+  gitlabJob: GitlabJobDef;
+  context: Context | null;
+};
+export type AllGitlabJobs = (GitlabJobWithContext & { name: string })[];
 
-const removeUndefined = <T extends Record<string, unknown>>(obj: T): T =>
-  Object.fromEntries(
-    Object.entries(obj).filter(([, value]) => value !== undefined)
-  ) as T;
-const getFullJobName = (
-  name: string,
-  componentName: string,
-  env?: string | null
-) => {
+export const GITLAB_ENVIRONMENT_URL_VARIABLE = "CL_GITLAB_ENVIRONMENT_URL";
+const getFullJobName = ({
+  type,
+  name,
+  baseName,
+  allJobs,
+  env,
+}: {
+  type: "component" | "workspace";
+  name: string;
+  baseName: string;
+  allJobs: AllCatladderJobs;
+  env?: string | null;
+}) => {
+  const shouldAddIcon = allJobs.workspaces.length > 0;
+  const icon = type === "component" ? "🔹" : "🔸";
+  const prefix = shouldAddIcon ? icon + " " : "";
   if (env) {
-    return `${componentName} ${name} | ${env} `;
+    return `${prefix}${baseName} ${name} | ${env} `;
   }
-  return `${componentName} ${name}`;
+  return `${prefix}${baseName} ${name}`;
 };
 
-const getFullReferencedJobName = (
+const getFullReferencedJobNameFromComponent = (
   referencedJobName: string,
   componentName: string,
   env: string,
-  allJobs: AllCatladderJobs
+  allJobs: AllCatladderJobs,
 ) => {
-  const referencedJob = allJobs[componentName]?.[env]?.find(
-    (j) => j.name === referencedJobName
-  );
+  const referencedJob = allJobs.components
+    .find((j) => j.context.name === componentName && j.context.env === env)
+    ?.jobs?.find((j) => j.name === referencedJobName);
+
   if (!referencedJob) {
     throw new Error(
-      `unknown job referenced: '${referencedJobName}' from '${env}:${componentName}'`
+      `unknown job referenced: '${referencedJobName}' from '${env}:${componentName}'`,
     );
   }
   const envToSet = referencedJob.envMode !== "none" ? env : null;
-  return getFullJobName(referencedJobName, componentName, envToSet);
+  return getFullJobName({
+    type: "component",
+    name: referencedJobName,
+    baseName: componentName,
+    env: envToSet,
+    allJobs,
+  });
+};
+
+const getFullReferencedJobNameFromWorkspace = (
+  referencedJobName: string,
+  workspaceName: string,
+  env: string,
+  allJobs: AllCatladderJobs,
+) => {
+  const referencedJob = allJobs.workspaces
+    .find((w) => w.context.name === workspaceName)
+    ?.jobs?.find((j) => j.name === referencedJobName);
+  if (!referencedJob) {
+    throw new Error(
+      `unknown job referenced: '${referencedJobName}' from workspace ${env}:${workspaceName}'`,
+    );
+  }
+  const envToSet = referencedJob.envMode !== "none" ? env : null;
+
+  return getFullJobName({
+    type: "workspace",
+    name: referencedJobName,
+    baseName: workspaceName,
+    env: envToSet,
+    allJobs,
+  });
 };
 
 const getJobName = (need: CatladderJobNeed) =>
   isObject(need) ? need.job : need;
 
 export const makeGitlabJob = (
-  componentName: string,
-  env: string,
-  {
+  context: Context,
+  job: CatladderJob<string>,
+  allJobs: AllCatladderJobs,
+  baseRules?: GitlabRule[],
+): [fullName: string, job: GitlabJobDef] => {
+  const {
+    environment,
     envMode,
     needsStages,
-    needsOtherComponent,
     name,
     needs,
     jobTags,
     script,
-    ...job
-  }: CatladderJob<string>,
-  allJobs: AllCatladderJobs
-): [fullName: string, job: GitlabJobDef] => {
-  const stage = envMode === "stagePerEnv" ? `${job.stage} ${env}` : job.stage;
 
-  const needsFromStages: CatladderJob["needs"] = needsStages?.flatMap((n) => {
-    const referencedComponentName = componentName;
+    variables,
+    runnerVariables,
+    when,
+    ...rest
+  } = job;
+  const stage =
+    envMode === "stagePerEnv" ? `${job.stage} ${context.env}` : job.stage;
+
+  const deduplicatedGitlabNeeds: GitlabJobDef["needs"] = getGitlabNeeds(
+    context,
+    job,
+    allJobs,
+  );
+
+  const fullJobName = getFullJobName({
+    type: context.type,
+    name,
+    baseName: context.name,
+    env: envMode !== "none" ? context.env : undefined,
+    allJobs,
+  });
+
+  // backwards compatibility, some may still use KUBERNETES_CPU_REQUEST, KUBERNETES_MEMORY_REQUEST, etc. in variables.
+  // those should now be set in the runnerVariables as they don't work in the variables key of the catladder job, becuase those get injected
+  const PIPELINE_RUNNER_VARIABLES = [
+    "KUBERNETES_CPU_REQUEST",
+    "KUBERNETES_MEMORY_REQUEST",
+    "KUBERNETES_CPU_LIMIT",
+    "KUBERNETES_MEMORY_LIMIT",
+  ];
+  // remove those from variables and add them to runnerVariables
+
+  const varsInjectScripts = collapseableSection(
+    "injectvars",
+    "Injecting variables",
+  )([
+    ...getInjectVarsScript(
+      // remove legacy variables
+      Object.fromEntries(
+        Object.entries(variables ?? {}).filter(
+          ([key]) => !PIPELINE_RUNNER_VARIABLES.includes(key),
+        ),
+      ),
+    ),
+  ]);
+
+  const legacyRunnerVariables = Object.fromEntries(
+    Object.entries(variables ?? {}).filter(([key]) =>
+      PIPELINE_RUNNER_VARIABLES.includes(key),
+    ),
+  );
+
+  if (Object.keys(legacyRunnerVariables).length > 0) {
+    console.warn(
+      `Legacy variables detected in ${fullJobName}: ${Object.keys(
+        legacyRunnerVariables,
+      ).join(", ")}. Please move them to the runnerVariables key.`,
+    );
+  }
+  const rules = [
+    ...(job.rules ?? []),
+    ...(baseRules
+      ? baseRules.map((rule) => ({
+          when: when,
+          ...rule,
+        }))
+      : when
+        ? [{ when }]
+        : []),
+  ];
+
+  const gitlabJob: GitlabJobDef = {
+    ...rest,
+    rules: rules.length > 0 ? rules : undefined,
+    variables: {
+      ...legacyRunnerVariables,
+      ...runnerVariables,
+    },
+    script: [...varsInjectScripts, ...(script?.filter(notNil) ?? [])],
+    tags: jobTags,
+    stage,
+
+    // sort in a predictable manner for snapshot tests
+    needs: deduplicatedGitlabNeeds,
+    retry: BASE_RETRY,
+    interruptible: true,
+  };
+  const modified = addGitlabEnvironment(
+    context,
+    environment,
+    gitlabJob,
+    allJobs,
+  );
+
+  return [fullJobName, removeUndefined(modified)];
+};
+
+const addGitlabEnvironment = (
+  context: Context,
+  catladderJobEnvironment: CatladderJob["environment"],
+  job: GitlabJobDef,
+  allJobs: AllCatladderJobs,
+): GitlabJobDef => {
+  if (!catladderJobEnvironment) {
+    return job;
+  }
+  if (context.type !== "component") {
+    // don't add enviornment for workspace jobs atm.
+    return job;
+  }
+  const { env, name, environment } = context;
+  const { url, envType } = environment;
+  const { on_stop, ...restEnvironment } = catladderJobEnvironment;
+  // those can be dynamic, so we therefore have to do this: https://docs.gitlab.com/ee/ci/environments/#set-a-dynamic-environment-url
+
+  const dotEnvFile = "gitlab_environment.env";
+
+  const scriptToAdd = [
+    `echo "${GITLAB_ENVIRONMENT_URL_VARIABLE}=${url}" >> ${dotEnvFile}`,
+  ];
+
+  // this is NOT a bashVariable since it NEEDS to be used as a string in gitlab
+  const gitlabEnvironmentName =
+    envType === "review"
+      ? `${env}/$CI_COMMIT_REF_NAME/${name}` // FIXME: should be replaced with mr name as well
+      : `${env}/${name}`;
+
+  return {
+    ...job,
+    environment: {
+      name: gitlabEnvironmentName,
+      url: `$${GITLAB_ENVIRONMENT_URL_VARIABLE}`,
+      ...(on_stop
+        ? {
+            on_stop: getFullReferencedJobNameFromComponent(
+              on_stop,
+              name,
+              env,
+              allJobs,
+            ),
+          }
+        : {}),
+      ...restEnvironment,
+    },
+    artifacts: merge(job.artifacts ?? {}, {
+      reports: {
+        dotenv: `${dotEnvFile}`,
+      },
+    }),
+
+    script: [...(job.script ?? []), ...scriptToAdd],
+  };
+};
+
+export const createGitlabJobs = async (
+  allJobs: AllCatladderJobs,
+  baseRules?: GitlabRule[],
+): Promise<AllGitlabJobs> => {
+  // TODO: add workspace jobs
+  return [...allJobs.workspaces, ...allJobs.components].flatMap(
+    ({ context, jobs }) => {
+      return jobs.map((job) => {
+        const [fullJobName, gitlabJob] = makeGitlabJob(
+          context,
+          job,
+          allJobs,
+          baseRules,
+        );
+        return {
+          name: fullJobName,
+          gitlabJob,
+          context,
+        };
+      });
+    },
+  );
+};
+
+function getGitlabNeeds(
+  context: Context,
+  job: CatladderJob<string>,
+  allJobs: AllCatladderJobs,
+): GitlabJobDef["needs"] {
+  const needs =
+    context.type === "workspace"
+      ? getGitlabNeedsForWorkspaceJob(context, job, allJobs)
+      : getGitlabNeedsForComponentJob(context, job, allJobs);
+
+  return deduplicateNeeds(needs);
+}
+function deduplicateNeeds(needs: GitlabJobDef["needs"]): GitlabJobDef["needs"] {
+  return needs
+    ? [...new Map(needs.map((n) => [isObject(n) ? n.job : n, n])).values()]
+    : undefined;
+}
+
+function getGitlabNeedsForComponentJob(
+  context: ComponentContext,
+  { needsStages, needs }: CatladderJob<string>,
+  allJobs: AllCatladderJobs,
+): GitlabJobDef["needs"] {
+  const needsFromStages = needsStages?.flatMap<CatladderJobNeed>((n) => {
+    const componentName = context.name;
+    if (!n.workspaceName) {
+      const allJobNamesFromThatStage =
+        allJobs.components
+          .filter(
+            (j) =>
+              j.context.name === componentName && j.context.env === context.env,
+          )
+          .flatMap((j) => j.jobs)
+          ?.filter((j) => j.stage === n.stage)
+          ?.map((j) => j.name) ?? [];
+
+      return allJobNamesFromThatStage.map((job) => ({
+        job,
+        artifacts: n.artifacts ?? false,
+        componentName,
+      }));
+    } else {
+      const allJobNamesFromThatStage =
+        allJobs.workspaces
+          .find(
+            (w) =>
+              w.context.name === n.workspaceName &&
+              w.context.env === context.env,
+          )
+          ?.jobs?.flatMap((j) => j)
+          ?.filter((j) => j.stage === n.stage)
+          ?.map((j) => j.name) ?? [];
+
+      return allJobNamesFromThatStage.map((job) => ({
+        job,
+        artifacts: n.artifacts ?? false,
+        workspaceName: n.workspaceName,
+      }));
+    }
+  });
+  const cleanedNeeds: CatladderJob["needs"] = [
+    ...(needsFromStages ?? []),
+    ...(needs ?? []),
+  ];
+
+  return cleanedNeeds
+    ?.map((n) =>
+      isObject(n)
+        ? "workspaceName" in n
+          ? {
+              job: getFullReferencedJobNameFromWorkspace(
+                n.job,
+                n.workspaceName,
+                context.env,
+                allJobs,
+              ),
+              artifacts: n.artifacts,
+            }
+          : {
+              job: getFullReferencedJobNameFromComponent(
+                n.job,
+                n.componentName ?? context.name,
+                context.env,
+                allJobs,
+              ),
+              artifacts: n.artifacts,
+            }
+        : getFullReferencedJobNameFromComponent(
+            n,
+            context.name,
+            context.env,
+            allJobs,
+          ),
+    ) // sort in a predictable manner for snapshot tests
+    .sort((a, b) => getJobName(a).localeCompare(getJobName(b)));
+}
+/**
+ *
+ *unclear whether we actually need this. So far jobs in a workspace don't have needs to other jobs from the same workspace
+ */
+function getGitlabNeedsForWorkspaceJob(
+  context: WorkspaceContext,
+  { needsStages, needs }: CatladderJob<string>,
+  allJobs: AllCatladderJobs,
+): GitlabJobDef["needs"] {
+  const needsFromStages = needsStages?.flatMap<CatladderJobNeed>((n) => {
+    const workspaceName = n.workspaceName ?? context.name;
     const allJobNamesFromThatStage =
-      allJobs[referencedComponentName]?.[env]
+      allJobs.workspaces
+        .filter(
+          (j) =>
+            j.context.name === workspaceName && j.context.env === context.env,
+        )
+        .flatMap((j) => j.jobs)
         ?.filter((j) => j.stage === n.stage)
         ?.map((j) => j.name) ?? [];
 
     return allJobNamesFromThatStage.map((job) => ({
       job,
       artifacts: n.artifacts ?? false,
-      componentName: referencedComponentName,
+      workspaceName: workspaceName,
     }));
   });
   const cleanedNeeds: CatladderJob["needs"] = [
     ...(needsFromStages ?? []),
     ...(needs ?? []),
-    // pull in legacy needs from other component, which is now identical to needs
-    ...(needsOtherComponent ?? []),
   ];
 
-  const gitlabNeeds: GitlabJobDef["needs"] = cleanedNeeds
+  return cleanedNeeds
     ?.map((n) =>
       isObject(n)
         ? {
-            job: getFullReferencedJobName(
+            job: getFullReferencedJobNameFromWorkspace(
               n.job,
-              n.componentName ?? componentName,
-              env,
-              allJobs
+              "workspaceName" in n && n.workspaceName
+                ? n.workspaceName
+                : context.name,
+              context.env,
+              allJobs,
             ),
             artifacts: n.artifacts,
           }
-        : getFullReferencedJobName(n, componentName, env, allJobs)
+        : getFullReferencedJobNameFromWorkspace(
+            n,
+            context.name,
+            context.env,
+            allJobs,
+          ),
     ) // sort in a predictable manner for snapshot tests
     .sort((a, b) => getJobName(a).localeCompare(getJobName(b)));
-
-  const deduplicatedGitlabNeeds: GitlabJobDef["needs"] = [
-    ...new Map(gitlabNeeds.map((n) => [isObject(n) ? n.job : n, n])).values(),
-  ];
-
-  const fullJobName = getFullJobName(
-    name,
-    componentName,
-    envMode !== "none" ? env : undefined
-  );
-
-  const gitlabJob: GitlabJobDef = removeUndefined({
-    ...job,
-    script: script?.filter(notNil),
-    tags: jobTags,
-    stage,
-    environment: job.environment?.on_stop
-      ? {
-          ...job.environment,
-          on_stop: getFullReferencedJobName(
-            job.environment.on_stop,
-            componentName,
-            env,
-            allJobs
-          ),
-        }
-      : job.environment,
-    // sort in a predictable manner for snapshot tests
-    needs: deduplicatedGitlabNeeds,
-    retry: BASE_RETRY,
-    interruptible: true,
-  });
-
-  return [fullJobName, gitlabJob];
-};
-
-export const createGitlabJobs = async (
-  allJobs: AllCatladderJobs
-): Promise<AllGitlabJobs> => {
-  return Object.keys(allJobs).reduce((accForComponents, componentName) => {
-    const componentJobs = allJobs[componentName];
-    return {
-      ...accForComponents,
-      ...Object.keys(componentJobs).reduce((accForEnvs, env) => {
-        const jobs = componentJobs[env];
-
-        return {
-          ...accForEnvs,
-          ...jobs.reduce((accForJobs, job) => {
-            const [fullJobName, gitlabJob] = makeGitlabJob(
-              componentName,
-              env,
-              job,
-              allJobs
-            );
-            return {
-              ...accForJobs,
-              [fullJobName]: gitlabJob,
-            };
-          }, {} as AllGitlabJobs),
-        };
-      }, {} as AllGitlabJobs),
-    };
-  }, {} as AllGitlabJobs);
-};
+}

package/src/pipeline/gitlab/createGitlabPipeline.ts

@@ -0,0 +1,35 @@
+import { getRunnerImage } from "../../runner";
+import type { Pipeline } from "../../types";
+import { globalScriptFunctions } from "../../globalScriptFunctions";
+type PickRequired<T, K extends keyof T> = Required<Pick<T, K>> & Omit<T, K>;
+
+export const createGitlabPipelineWithDefaults = ({
+  image,
+  variables,
+  before_script,
+  ...config
+}: PickRequired<
+  Partial<Pipeline<"gitlab">>,
+  "stages" | "jobs"
+>): Pipeline<"gitlab"> => {
+  return {
+    image: image ?? getRunnerImage("jobs-default"), // default image
+    variables: {
+      FF_USE_FASTZIP: "true", // enable fastzip - a faster zip implementation that also supports level configuration.
+      ARTIFACT_COMPRESSION_LEVEL: "fast", // we value speed over compression
+      CACHE_COMPRESSION_LEVEL: "fast", // same as above, but for caches
+      TRANSFER_METER_FREQUENCY: "5s", // how often we should update the transfer meter for cache upload/download
+
+      GIT_DEPTH: "1", // no need the full depth
+      ...(variables ?? {}),
+    },
+    before_script: [
+      ...[...globalScriptFunctions.values()].map((script) =>
+        script.toBashFunction(),
+      ),
+      ...(before_script ?? []),
+    ],
+
+    ...config,
+  };
+};

package/src/pipeline/gitlab/gitlabReleaseJobs.ts

@@ -0,0 +1,23 @@
+import { RULES_MANUAL_RELEASE, RULES_RELEASE } from "../../rules";
+import { getRunnerImage } from "../../runner";
+
+const EXPIRED_TOKEN_HELP =
+  "echo '👉 The project access token might be invald - run `project-renew-token` in catladder CLI to fix.'";
+
+export const getGitlabReleaseJobs = () => {
+  return {
+    ["create release"]: {
+      stage: "release",
+      image: getRunnerImage("semantic-release"),
+      script: ["semanticRelease", EXPIRED_TOKEN_HELP],
+      rules: RULES_RELEASE,
+    },
+    ["⚠️ force create release"]: {
+      stage: "release",
+      image: getRunnerImage("semantic-release"),
+      script: ["semanticRelease", EXPIRED_TOKEN_HELP],
+      needs: [],
+      rules: RULES_MANUAL_RELEASE,
+    },
+  };
+};

package/src/pipeline/gitlab/sortGitLabJobDefProps.ts

@@ -0,0 +1,59 @@
+import type { GitlabJobDef } from "../../types";
+
+type GitlabJobDefKey = keyof GitlabJobDef;
+
+/**
+ * The desired order of GitLab job definition keys.
+ */
+const jobDefOrder: GitlabJobDefKey[] = [
+  "stage",
+  "tags",
+  "image",
+  "services",
+  "variables",
+  "before_script",
+  "script",
+  "after_script",
+  "cache",
+  "coverage",
+  "environment",
+  "release",
+  "artifacts",
+  "rules",
+  "only",
+  "except",
+  "needs",
+  "dependencies",
+  "trigger",
+  "retry",
+  "interruptible",
+  "allow_failure",
+  "parallel",
+  "hooks",
+  "resource_group",
+];
+
+const sortGitLabJobDefKeys = (jobDef: GitlabJobDef): GitlabJobDefKey[] =>
+  (Object.keys(jobDef) as GitlabJobDefKey[]).sort((a, b) =>
+    jobDefOrder.indexOf(a) === -1 && jobDefOrder.indexOf(b) === -1
+      ? 0
+      : jobDefOrder.indexOf(a) === -1
+        ? 1
+        : jobDefOrder.indexOf(b) === -1
+          ? -1
+          : jobDefOrder.indexOf(a) - jobDefOrder.indexOf(b),
+  );
+
+/**
+ * Sorts the properties of a GitLab job definition based on the desired order.
+ *
+ * This is only useful when generating a GitLab pipeline YAML file.
+ */
+export const sortGitLabJobDefProps = (jobDef: GitlabJobDef): GitlabJobDef =>
+  sortGitLabJobDefKeys(jobDef).reduce(
+    (acc, key) =>
+      jobDef[key] !== undefined
+        ? Object.assign(acc, { [key]: jobDef[key] })
+        : acc,
+    {} as GitlabJobDef,
+  );

package/src/pipeline/index.ts

@@ -1,2 +1,3 @@
-export * from "./createChildPipeline";
 export * from "./createJobsForComponent";
+export * from "./generatePipelineFiles";
+export * from "./gitlab/gitlabReleaseJobs";