@catladder/pipeline 0.0.0-node16-ddfae617 → 0.0.0-perf-cli-reduce-bundle-size-with-tsconfig-optimization-3556aaa4

package/src/pipeline/gitlab/createGitlabJobs.ts

@@ -0,0 +1,445 @@
+import { isObject, merge } from "lodash";
+import { getInjectVarsScript } from "../../bash/getInjectVarsScript";
+import { BASE_RETRY } from "../../defaults";
+import type {
+  ComponentContext,
+  Context,
+  GitlabJobDef,
+  GitlabRule,
+  WorkspaceContext,
+} from "../../types";
+import type { CatladderJob, CatladderJobNeed } from "../../types/jobs";
+import { notNil } from "../../utils";
+import { collapseableSection } from "../../utils/gitlab";
+import { removeUndefined } from "../../utils/removeUndefined";
+import type { AllCatladderJobs } from "../createAllJobs";
+
+export type AllGitlabJobs = {
+  name: string;
+  gitlabJob: GitlabJobDef;
+  context: Context;
+}[];
+
+export const GITLAB_ENVIRONMENT_URL_VARIABLE = "CL_GITLAB_ENVIRONMENT_URL";
+const getFullJobName = ({
+  type,
+  name,
+  baseName,
+  allJobs,
+  env,
+}: {
+  type: "component" | "workspace";
+  name: string;
+  baseName: string;
+  allJobs: AllCatladderJobs;
+  env?: string | null;
+}) => {
+  const shouldAddIcon = allJobs.workspaces.length > 0;
+  const icon = type === "component" ? "🔹" : "🔸";
+  const prefix = shouldAddIcon ? icon + " " : "";
+  if (env) {
+    return `${prefix}${baseName} ${name} | ${env} `;
+  }
+  return `${prefix}${baseName} ${name}`;
+};
+
+const getFullReferencedJobNameFromComponent = (
+  referencedJobName: string,
+  componentName: string,
+  env: string,
+  allJobs: AllCatladderJobs,
+) => {
+  const referencedJob = allJobs.components
+    .find((j) => j.context.name === componentName && j.context.env === env)
+    ?.jobs?.find((j) => j.name === referencedJobName);
+
+  if (!referencedJob) {
+    throw new Error(
+      `unknown job referenced: '${referencedJobName}' from '${env}:${componentName}'`,
+    );
+  }
+  const envToSet = referencedJob.envMode !== "none" ? env : null;
+  return getFullJobName({
+    type: "component",
+    name: referencedJobName,
+    baseName: componentName,
+    env: envToSet,
+    allJobs,
+  });
+};
+
+const getFullReferencedJobNameFromWorkspace = (
+  referencedJobName: string,
+  workspaceName: string,
+  env: string,
+  allJobs: AllCatladderJobs,
+) => {
+  const referencedJob = allJobs.workspaces
+    .find((w) => w.context.name === workspaceName)
+    ?.jobs?.find((j) => j.name === referencedJobName);
+  if (!referencedJob) {
+    throw new Error(
+      `unknown job referenced: '${referencedJobName}' from workspace ${env}:${workspaceName}'`,
+    );
+  }
+  const envToSet = referencedJob.envMode !== "none" ? env : null;
+
+  return getFullJobName({
+    type: "workspace",
+    name: referencedJobName,
+    baseName: workspaceName,
+    env: envToSet,
+    allJobs,
+  });
+};
+
+const getJobName = (need: CatladderJobNeed) =>
+  isObject(need) ? need.job : need;
+
+export const makeGitlabJob = (
+  context: Context,
+  job: CatladderJob<string>,
+  allJobs: AllCatladderJobs,
+  baseRules?: GitlabRule[],
+): [fullName: string, job: GitlabJobDef] => {
+  const {
+    environment,
+    envMode,
+    needsStages,
+    needsOtherComponent,
+    name,
+    needs,
+    jobTags,
+    script,
+
+    variables,
+    runnerVariables,
+    when,
+    ...rest
+  } = job;
+  const stage =
+    envMode === "stagePerEnv" ? `${job.stage} ${context.env}` : job.stage;
+
+  const deduplicatedGitlabNeeds: GitlabJobDef["needs"] = getGitlabNeeds(
+    context,
+    job,
+    allJobs,
+  );
+
+  const fullJobName = getFullJobName({
+    type: context.type,
+    name,
+    baseName: context.name,
+    env: envMode !== "none" ? context.env : undefined,
+    allJobs,
+  });
+
+  // backwards compatibility, some may still use KUBERNETES_CPU_REQUEST, KUBERNETES_MEMORY_REQUEST, etc. in variables.
+  // those should now be set in the runnerVariables as they don't work in the variables key of the catladder job, becuase those get injected
+  const PIPELINE_RUNNER_VARIABLES = [
+    "KUBERNETES_CPU_REQUEST",
+    "KUBERNETES_MEMORY_REQUEST",
+    "KUBERNETES_CPU_LIMIT",
+    "KUBERNETES_MEMORY_LIMIT",
+  ];
+  // remove those from variables and add them to runnerVariables
+
+  const varsInjectScripts = collapseableSection(
+    "injectvars",
+    "Injecting variables",
+  )([
+    ...getInjectVarsScript(
+      // remove legacy variables
+      Object.fromEntries(
+        Object.entries(variables ?? {}).filter(
+          ([key]) => !PIPELINE_RUNNER_VARIABLES.includes(key),
+        ),
+      ),
+    ),
+  ]);
+
+  const legacyRunnerVariables = Object.fromEntries(
+    Object.entries(variables ?? {}).filter(([key]) =>
+      PIPELINE_RUNNER_VARIABLES.includes(key),
+    ),
+  );
+
+  if (Object.keys(legacyRunnerVariables).length > 0) {
+    console.warn(
+      `Legacy variables detected in ${fullJobName}: ${Object.keys(
+        legacyRunnerVariables,
+      ).join(", ")}. Please move them to the runnerVariables key.`,
+    );
+  }
+  const rules = [
+    ...(job.rules ?? []),
+    ...(baseRules
+      ? baseRules.map((rule) => ({
+          when: when,
+          ...rule,
+        }))
+      : when
+        ? [{ when }]
+        : []),
+  ];
+
+  const gitlabJob: GitlabJobDef = {
+    ...rest,
+    rules: rules.length > 0 ? rules : undefined,
+    variables: {
+      ...legacyRunnerVariables,
+      ...runnerVariables,
+    },
+    script: [...varsInjectScripts, ...(script?.filter(notNil) ?? [])],
+    tags: jobTags,
+    stage,
+
+    // sort in a predictable manner for snapshot tests
+    needs: deduplicatedGitlabNeeds,
+    retry: BASE_RETRY,
+    interruptible: true,
+  };
+  const modified = addGitlabEnvironment(
+    context,
+    environment,
+    gitlabJob,
+    allJobs,
+  );
+
+  return [fullJobName, removeUndefined(modified)];
+};
+
+const addGitlabEnvironment = (
+  context: Context,
+  catladderJobEnvironment: CatladderJob["environment"],
+  job: GitlabJobDef,
+  allJobs: AllCatladderJobs,
+): GitlabJobDef => {
+  if (!catladderJobEnvironment) {
+    return job;
+  }
+  if (context.type !== "component") {
+    // don't add enviornment for workspace jobs atm.
+    return job;
+  }
+  const { env, name, environment } = context;
+  const { url, envType } = environment;
+  const { on_stop, ...restEnvironment } = catladderJobEnvironment;
+  // those can be dynamic, so we therefore have to do this: https://docs.gitlab.com/ee/ci/environments/#set-a-dynamic-environment-url
+
+  const dotEnvFile = "gitlab_environment.env";
+
+  const scriptToAdd = [
+    `echo "${GITLAB_ENVIRONMENT_URL_VARIABLE}=${url}" >> ${dotEnvFile}`,
+  ];
+
+  // this is NOT a bashVariable since it NEEDS to be used as a string in gitlab
+  const gitlabEnvironmentName =
+    envType === "review"
+      ? `${env}/$CI_COMMIT_REF_NAME/${name}` // FIXME: should be replaced with mr name as well
+      : `${env}/${name}`;
+
+  return {
+    ...job,
+    environment: {
+      name: gitlabEnvironmentName,
+      url: `$${GITLAB_ENVIRONMENT_URL_VARIABLE}`,
+      ...(on_stop
+        ? {
+            on_stop: getFullReferencedJobNameFromComponent(
+              on_stop,
+              name,
+              env,
+              allJobs,
+            ),
+          }
+        : {}),
+      ...restEnvironment,
+    },
+    artifacts: merge(job.artifacts ?? {}, {
+      reports: {
+        dotenv: `${dotEnvFile}`,
+      },
+    }),
+
+    script: [...(job.script ?? []), ...scriptToAdd],
+  };
+};
+
+export const createGitlabJobs = async (
+  allJobs: AllCatladderJobs,
+  baseRules?: GitlabRule[],
+): Promise<AllGitlabJobs> => {
+  // TODO: add workspace jobs
+  return [...allJobs.workspaces, ...allJobs.components].flatMap(
+    ({ context, jobs }) => {
+      return jobs.map((job) => {
+        const [fullJobName, gitlabJob] = makeGitlabJob(
+          context,
+          job,
+          allJobs,
+          baseRules,
+        );
+        return {
+          name: fullJobName,
+          gitlabJob,
+          context,
+        };
+      });
+    },
+  );
+};
+
+function getGitlabNeeds(
+  context: Context,
+  job: CatladderJob<string>,
+  allJobs: AllCatladderJobs,
+): GitlabJobDef["needs"] {
+  const needs =
+    context.type === "workspace"
+      ? getGitlabNeedsForWorkspaceJob(context, job, allJobs)
+      : getGitlabNeedsForComponentJob(context, job, allJobs);
+
+  return deduplicateNeeds(needs);
+}
+function deduplicateNeeds(needs: GitlabJobDef["needs"]): GitlabJobDef["needs"] {
+  return needs
+    ? [...new Map(needs.map((n) => [isObject(n) ? n.job : n, n])).values()]
+    : undefined;
+}
+
+function getGitlabNeedsForComponentJob(
+  context: ComponentContext,
+  { needsStages, needs, needsOtherComponent }: CatladderJob<string>,
+  allJobs: AllCatladderJobs,
+): GitlabJobDef["needs"] {
+  const needsFromStages = needsStages?.flatMap<CatladderJobNeed>((n) => {
+    const componentName = context.name;
+    if (!n.workspaceName) {
+      const allJobNamesFromThatStage =
+        allJobs.components
+          .filter(
+            (j) =>
+              j.context.name === componentName && j.context.env === context.env,
+          )
+          .flatMap((j) => j.jobs)
+          ?.filter((j) => j.stage === n.stage)
+          ?.map((j) => j.name) ?? [];
+
+      return allJobNamesFromThatStage.map((job) => ({
+        job,
+        artifacts: n.artifacts ?? false,
+        componentName,
+      }));
+    } else {
+      const allJobNamesFromThatStage =
+        allJobs.workspaces
+          .find(
+            (w) =>
+              w.context.name === n.workspaceName &&
+              w.context.env === context.env,
+          )
+          ?.jobs?.flatMap((j) => j)
+          ?.filter((j) => j.stage === n.stage)
+          ?.map((j) => j.name) ?? [];
+
+      return allJobNamesFromThatStage.map((job) => ({
+        job,
+        artifacts: n.artifacts ?? false,
+        workspaceName: n.workspaceName,
+      }));
+    }
+  });
+  const cleanedNeeds: CatladderJob["needs"] = [
+    ...(needsFromStages ?? []),
+    ...(needs ?? []),
+    // pull in legacy needs from other component, which is now identical to needs
+    ...(needsOtherComponent ?? []),
+  ];
+
+  return cleanedNeeds
+    ?.map((n) =>
+      isObject(n)
+        ? "workspaceName" in n
+          ? {
+              job: getFullReferencedJobNameFromWorkspace(
+                n.job,
+                n.workspaceName,
+                context.env,
+                allJobs,
+              ),
+              artifacts: n.artifacts,
+            }
+          : {
+              job: getFullReferencedJobNameFromComponent(
+                n.job,
+                n.componentName ?? context.name,
+                context.env,
+                allJobs,
+              ),
+              artifacts: n.artifacts,
+            }
+        : getFullReferencedJobNameFromComponent(
+            n,
+            context.name,
+            context.env,
+            allJobs,
+          ),
+    ) // sort in a predictable manner for snapshot tests
+    .sort((a, b) => getJobName(a).localeCompare(getJobName(b)));
+}
+/**
+ *
+ *unclear whether we actually need this. So far jobs in a workspace don't have needs to other jobs from the same workspace
+ */
+function getGitlabNeedsForWorkspaceJob(
+  context: WorkspaceContext,
+  { needsStages, needs }: CatladderJob<string>,
+  allJobs: AllCatladderJobs,
+): GitlabJobDef["needs"] {
+  const needsFromStages = needsStages?.flatMap<CatladderJobNeed>((n) => {
+    const workspaceName = n.workspaceName ?? context.name;
+    const allJobNamesFromThatStage =
+      allJobs.workspaces
+        .filter(
+          (j) =>
+            j.context.name === workspaceName && j.context.env === context.env,
+        )
+        .flatMap((j) => j.jobs)
+        ?.filter((j) => j.stage === n.stage)
+        ?.map((j) => j.name) ?? [];
+
+    return allJobNamesFromThatStage.map((job) => ({
+      job,
+      artifacts: n.artifacts ?? false,
+      workspaceName: workspaceName,
+    }));
+  });
+  const cleanedNeeds: CatladderJob["needs"] = [
+    ...(needsFromStages ?? []),
+    ...(needs ?? []),
+  ];
+
+  return cleanedNeeds
+    ?.map((n) =>
+      isObject(n)
+        ? {
+            job: getFullReferencedJobNameFromWorkspace(
+              n.job,
+              "workspaceName" in n && n.workspaceName
+                ? n.workspaceName
+                : context.name,
+              context.env,
+              allJobs,
+            ),
+            artifacts: n.artifacts,
+          }
+        : getFullReferencedJobNameFromWorkspace(
+            n,
+            context.name,
+            context.env,
+            allJobs,
+          ),
+    ) // sort in a predictable manner for snapshot tests
+    .sort((a, b) => getJobName(a).localeCompare(getJobName(b)));
+}

package/src/pipeline/gitlab/createGitlabPipeline.ts

@@ -0,0 +1,28 @@
+import { getRunnerImage } from "../../runner";
+import type { Pipeline } from "../../types";
+
+type PickRequired<T, K extends keyof T> = Required<Pick<T, K>> & Omit<T, K>;
+
+export const createGitlabPipelineWithDefaults = ({
+  image,
+  variables,
+  ...config
+}: PickRequired<
+  Partial<Pipeline<"gitlab">>,
+  "stages" | "jobs"
+>): Pipeline<"gitlab"> => {
+  return {
+    image: image ?? getRunnerImage("jobs-default"), // default image
+    variables: {
+      FF_USE_FASTZIP: "true", // enable fastzip - a faster zip implementation that also supports level configuration.
+      ARTIFACT_COMPRESSION_LEVEL: "fast", // we value speed over compression
+      CACHE_COMPRESSION_LEVEL: "fast", // same as above, but for caches
+      TRANSFER_METER_FREQUENCY: "5s", // how often we should update the transfer meter for cache upload/download
+
+      GIT_DEPTH: "1", // no need the full depth
+      ...(variables ?? {}),
+    },
+
+    ...config,
+  };
+};

package/src/pipeline/gitlab/getPipelineTriggerForGitlabChildPipeline.ts

@@ -0,0 +1,46 @@
+import type { PipelineTrigger } from "../..";
+import { PIPELINE_IMAGE_TAG } from "../../constants";
+
+export function getPipelineTriggerForGitlabChildPipeline() {
+  const {
+    CI_MERGE_REQUEST_ID,
+    CI_COMMIT_TAG,
+    CI_COMMIT_BRANCH,
+    CI_DEFAULT_BRANCH,
+  } = process.env;
+
+  const isDefaultBranch =
+    Boolean(CI_DEFAULT_BRANCH) && CI_COMMIT_BRANCH === CI_DEFAULT_BRANCH;
+  const isHotfixBranch = CI_COMMIT_BRANCH
+    ? /^[0-9]+\.([0-9]+|x)\.x$/.test(CI_COMMIT_BRANCH)
+    : false;
+  const isMergeRequest = Boolean(CI_MERGE_REQUEST_ID);
+  const isTaggedRelease = Boolean(CI_COMMIT_TAG);
+
+  console.info(`catladder version ${PIPELINE_IMAGE_TAG}`);
+
+  const trigger: PipelineTrigger | null =
+    isMergeRequest || isHotfixBranch
+      ? "mr"
+      : isDefaultBranch
+        ? "mainBranch"
+        : isTaggedRelease
+          ? "taggedRelease"
+          : null;
+
+  if (!trigger) {
+    throw new Error(
+      "no matching trigger: " +
+        JSON.stringify(
+          {
+            isMergeRequest,
+            isDefaultBranch,
+            isTaggedRelease,
+          },
+          null,
+          2,
+        ),
+    );
+  }
+  return trigger;
+}

package/src/pipeline/gitlab/gitlabReleaseJobs.ts

@@ -0,0 +1,25 @@
+import { RULES_MANUAL_RELEASE, RULES_RELEASE } from "../../rules";
+import { getRunnerImage } from "../../runner";
+
+const EXPIRED_TOKEN_HELP =
+  "echo '👉 The project access token might be invald - run `project-renew-token` in catladder CLI to fix.'";
+
+export const getGitlabReleaseJobs = () => {
+  return {
+    ["create release"]: {
+      stage: "release",
+      image: getRunnerImage("semantic-release"),
+      script: ["semanticRelease"],
+      after_script: [EXPIRED_TOKEN_HELP],
+      rules: RULES_RELEASE,
+    },
+    ["⚠️ force create release"]: {
+      stage: "release",
+      image: getRunnerImage("semantic-release"),
+      script: ["semanticRelease"],
+      after_script: [EXPIRED_TOKEN_HELP],
+      needs: [],
+      rules: RULES_MANUAL_RELEASE,
+    },
+  };
+};

package/src/pipeline/gitlab/sortGitLabJobDefProps.ts

@@ -0,0 +1,59 @@
+import type { GitlabJobDef } from "../../types";
+
+type GitlabJobDefKey = keyof GitlabJobDef;
+
+/**
+ * The desired order of GitLab job definition keys.
+ */
+const jobDefOrder: GitlabJobDefKey[] = [
+  "stage",
+  "tags",
+  "image",
+  "services",
+  "variables",
+  "before_script",
+  "script",
+  "after_script",
+  "cache",
+  "coverage",
+  "environment",
+  "release",
+  "artifacts",
+  "rules",
+  "only",
+  "except",
+  "needs",
+  "dependencies",
+  "trigger",
+  "retry",
+  "interruptible",
+  "allow_failure",
+  "parallel",
+  "hooks",
+  "resource_group",
+];
+
+const sortGitLabJobDefKeys = (jobDef: GitlabJobDef): GitlabJobDefKey[] =>
+  (Object.keys(jobDef) as GitlabJobDefKey[]).sort((a, b) =>
+    jobDefOrder.indexOf(a) === -1 && jobDefOrder.indexOf(b) === -1
+      ? 0
+      : jobDefOrder.indexOf(a) === -1
+        ? 1
+        : jobDefOrder.indexOf(b) === -1
+          ? -1
+          : jobDefOrder.indexOf(a) - jobDefOrder.indexOf(b),
+  );
+
+/**
+ * Sorts the properties of a GitLab job definition based on the desired order.
+ *
+ * This is only useful when generating a GitLab pipeline YAML file.
+ */
+export const sortGitLabJobDefProps = (jobDef: GitlabJobDef): GitlabJobDef =>
+  sortGitLabJobDefKeys(jobDef).reduce(
+    (acc, key) =>
+      jobDef[key] !== undefined
+        ? Object.assign(acc, { [key]: jobDef[key] })
+        : acc,
+    {} as GitlabJobDef,
+  );

package/src/pipeline/index.ts

@@ -1,2 +1,4 @@
 export * from "./createChildPipeline";
-export * from "./createJobs";
+export * from "./createJobsForComponent";
+export * from "./generatePipelineFiles";
+export * from "./gitlab/gitlabReleaseJobs";