@catladder/pipeline 1.153.0 → 1.154.0

package/src/pipeline/gitlab/createGitlabJobs.ts

@@ -1,7 +1,13 @@
 import { isObject, merge } from "lodash";
 import { getInjectVarsScript } from "../../bash/getInjectVarsScript";
 import { BASE_RETRY } from "../../defaults";
-import type { ComponentContext, GitlabJobDef, GitlabRule } from "../../types";
+import type {
+  ComponentContext,
+  Context,
+  GitlabJobDef,
+  GitlabRule,
+  WorkspaceContext,
+} from "../../types";
 import type { CatladderJob, CatladderJobNeed } from "../../types/jobs";
 import { notNil } from "../../utils";
 import { collapseableSection } from "../../utils/gitlab";
@@ -11,31 +17,40 @@ import type { AllCatladderJobs } from "../createAllJobs";
 export type AllGitlabJobs = {
   name: string;
   gitlabJob: GitlabJobDef;
-  context: ComponentContext;
+  context: Context;
 }[];
 
 export const GITLAB_ENVIRONMENT_URL_VARIABLE = "CL_GITLAB_ENVIRONMENT_URL";
-const getFullJobName = (
-  name: string,
-  componentName: string,
-  env?: string | null,
-) => {
+const getFullJobName = ({
+  type,
+  name,
+  baseName,
+  allJobs,
+  env,
+}: {
+  type: "component" | "workspace";
+  name: string;
+  baseName: string;
+  allJobs: AllCatladderJobs;
+  env?: string | null;
+}) => {
+  const shouldAddIcon = allJobs.workspaces.length > 0;
+  const icon = type === "component" ? "🔹" : "🔸";
+  const prefix = shouldAddIcon ? icon + " " : "";
   if (env) {
-    return `${componentName} ${name} | ${env} `;
+    return `${prefix}${baseName} ${name} | ${env} `;
   }
-  return `${componentName} ${name}`;
+  return `${prefix}${baseName} ${name}`;
 };
 
-const getFullReferencedJobName = (
+const getFullReferencedJobNameFromComponent = (
   referencedJobName: string,
   componentName: string,
   env: string,
   allJobs: AllCatladderJobs,
 ) => {
-  const referencedJob = allJobs
-    .find(
-      (j) => j.context.componentName === componentName && j.context.env === env,
-    )
+  const referencedJob = allJobs.components
+    .find((j) => j.context.name === componentName && j.context.env === env)
     ?.jobs?.find((j) => j.name === referencedJobName);
 
   if (!referencedJob) {
@@ -44,15 +59,50 @@ const getFullReferencedJobName = (
     );
   }
   const envToSet = referencedJob.envMode !== "none" ? env : null;
-  return getFullJobName(referencedJobName, componentName, envToSet);
+  return getFullJobName({
+    type: "component",
+    name: referencedJobName,
+    baseName: componentName,
+    env: envToSet,
+    allJobs,
+  });
+};
+
+const getFullReferencedJobNameFromWorkspace = (
+  referencedJobName: string,
+  workspaceName: string,
+  env: string,
+  allJobs: AllCatladderJobs,
+) => {
+  const referencedJob = allJobs.workspaces
+    .find((w) => w.context.name === workspaceName)
+    ?.jobs?.find((j) => j.name === referencedJobName);
+  if (!referencedJob) {
+    throw new Error(
+      `unknown job referenced: '${referencedJobName}' from workspace ${env}:${workspaceName}'`,
+    );
+  }
+  const envToSet = referencedJob.envMode !== "none" ? env : null;
+
+  return getFullJobName({
+    type: "workspace",
+    name: referencedJobName,
+    baseName: workspaceName,
+    env: envToSet,
+    allJobs,
+  });
 };
 
 const getJobName = (need: CatladderJobNeed) =>
   isObject(need) ? need.job : need;
 
 export const makeGitlabJob = (
-  context: ComponentContext,
-  {
+  context: Context,
+  job: CatladderJob<string>,
+  allJobs: AllCatladderJobs,
+  baseRules?: GitlabRule[],
+): [fullName: string, job: GitlabJobDef] => {
+  const {
     environment,
     envMode,
     needsStages,
@@ -65,70 +115,24 @@ export const makeGitlabJob = (
     variables,
     runnerVariables,
     when,
-    ...job
-  }: CatladderJob<string>,
-  allJobs: AllCatladderJobs,
-  baseRules?: GitlabRule[],
-): [fullName: string, job: GitlabJobDef] => {
+    ...rest
+  } = job;
   const stage =
     envMode === "stagePerEnv" ? `${job.stage} ${context.env}` : job.stage;
 
-  const needsFromStages: CatladderJob["needs"] = needsStages?.flatMap((n) => {
-    const referencedComponentName = context.componentName;
-    const allJobNamesFromThatStage =
-      allJobs
-        .filter(
-          (j) =>
-            j.context.componentName === referencedComponentName &&
-            j.context.env === context.env,
-        )
-        .flatMap((j) => j.jobs)
-        ?.filter((j) => j.stage === n.stage)
-        ?.map((j) => j.name) ?? [];
-
-    return allJobNamesFromThatStage.map((job) => ({
-      job,
-      artifacts: n.artifacts ?? false,
-      componentName: referencedComponentName,
-    }));
-  });
-  const cleanedNeeds: CatladderJob["needs"] = [
-    ...(needsFromStages ?? []),
-    ...(needs ?? []),
-    // pull in legacy needs from other component, which is now identical to needs
-    ...(needsOtherComponent ?? []),
-  ];
-
-  const gitlabNeeds: GitlabJobDef["needs"] = cleanedNeeds
-    ?.map((n) =>
-      isObject(n)
-        ? {
-            job: getFullReferencedJobName(
-              n.job,
-              n.componentName ?? context.componentName,
-              context.env,
-              allJobs,
-            ),
-            artifacts: n.artifacts,
-          }
-        : getFullReferencedJobName(
-            n,
-            context.componentName,
-            context.env,
-            allJobs,
-          ),
-    ) // sort in a predictable manner for snapshot tests
-    .sort((a, b) => getJobName(a).localeCompare(getJobName(b)));
-
-  const deduplicatedGitlabNeeds: GitlabJobDef["needs"] = [
-    ...new Map(gitlabNeeds.map((n) => [isObject(n) ? n.job : n, n])).values(),
-  ];
+  const deduplicatedGitlabNeeds: GitlabJobDef["needs"] = getGitlabNeeds(
+    context,
+    job,
+    allJobs,
+  );
 
-  const fullJobName = getFullJobName(
+  const fullJobName = getFullJobName({
+    type: context.type,
     name,
-    context.componentName,
-    envMode !== "none" ? context.env : undefined,
-  );
+    baseName: context.name,
+    env: envMode !== "none" ? context.env : undefined,
+    allJobs,
+  });
 
   // backwards compatibility, some may still use KUBERNETES_CPU_REQUEST, KUBERNETES_MEMORY_REQUEST, etc. in variables.
   // those should now be set in the runnerVariables as they don't work in the variables key of the catladder job, becuase those get injected
@@ -178,27 +182,27 @@ export const makeGitlabJob = (
         ? [{ when }]
         : []),
   ];
+
+  const gitlabJob: GitlabJobDef = {
+    ...rest,
+    rules: rules.length > 0 ? rules : undefined,
+    variables: {
+      ...legacyRunnerVariables,
+      ...runnerVariables,
+    },
+    script: [...varsInjectScripts, ...(script?.filter(notNil) ?? [])],
+    tags: jobTags,
+    stage,
+
+    // sort in a predictable manner for snapshot tests
+    needs: deduplicatedGitlabNeeds,
+    retry: BASE_RETRY,
+    interruptible: true,
+  };
   const modified = addGitlabEnvironment(
     context,
     environment,
-    {
-      ...job,
-      rules: rules.length > 0 ? rules : undefined,
-      variables: {
-        ...legacyRunnerVariables,
-        ...runnerVariables,
-      },
-      script: [...varsInjectScripts, ...(script?.filter(notNil) ?? [])],
-      tags: jobTags,
-      stage,
-
-      // sort in a predictable manner for snapshot tests
-      needs: deduplicatedGitlabNeeds,
-      retry: BASE_RETRY,
-      interruptible: true,
-    },
-    context.componentName,
-    context.env,
+    gitlabJob,
     allJobs,
   );
 
@@ -206,18 +210,21 @@ export const makeGitlabJob = (
 };
 
 const addGitlabEnvironment = (
-  context: ComponentContext,
-  environment: CatladderJob["environment"],
+  context: Context,
+  catladderJobEnvironment: CatladderJob["environment"],
   job: GitlabJobDef,
-  componentName: string,
-  env: string, // TODO: we could actually pull this from contxt
   allJobs: AllCatladderJobs,
 ): GitlabJobDef => {
-  if (!environment) {
+  if (!catladderJobEnvironment) {
+    return job;
+  }
+  if (context.type !== "component") {
+    // don't add enviornment for workspace jobs atm.
     return job;
   }
-  const { url, envType } = context.environment;
-  const { on_stop, ...restEnvironment } = environment;
+  const { env, name, environment } = context;
+  const { url, envType } = environment;
+  const { on_stop, ...restEnvironment } = catladderJobEnvironment;
   // those can be dynamic, so we therefore have to do this: https://docs.gitlab.com/ee/ci/environments/#set-a-dynamic-environment-url
 
   const dotEnvFile = "gitlab_environment.env";
@@ -229,8 +236,8 @@ const addGitlabEnvironment = (
   // this is NOT a bashVariable since it NEEDS to be used as a string in gitlab
   const gitlabEnvironmentName =
     envType === "review"
-      ? `${env}/$CI_COMMIT_REF_NAME/${componentName}` // FIXME: should be replaced with mr name as well
-      : `${env}/${componentName}`;
+      ? `${env}/$CI_COMMIT_REF_NAME/${name}` // FIXME: should be replaced with mr name as well
+      : `${env}/${name}`;
 
   return {
     ...job,
@@ -239,9 +246,9 @@ const addGitlabEnvironment = (
       url: `$${GITLAB_ENVIRONMENT_URL_VARIABLE}`,
       ...(on_stop
         ? {
-            on_stop: getFullReferencedJobName(
+            on_stop: getFullReferencedJobNameFromComponent(
               on_stop,
-              componentName,
+              name,
               env,
               allJobs,
             ),
@@ -263,19 +270,176 @@ export const createGitlabJobs = async (
   allJobs: AllCatladderJobs,
   baseRules?: GitlabRule[],
 ): Promise<AllGitlabJobs> => {
-  return allJobs.flatMap(({ context, jobs }) => {
-    return jobs.map((job) => {
-      const [fullJobName, gitlabJob] = makeGitlabJob(
-        context,
+  // TODO: add workspace jobs
+  return [...allJobs.workspaces, ...allJobs.components].flatMap(
+    ({ context, jobs }) => {
+      return jobs.map((job) => {
+        const [fullJobName, gitlabJob] = makeGitlabJob(
+          context,
+          job,
+          allJobs,
+          baseRules,
+        );
+        return {
+          name: fullJobName,
+          gitlabJob,
+          context,
+        };
+      });
+    },
+  );
+};
+
+function getGitlabNeeds(
+  context: Context,
+  job: CatladderJob<string>,
+  allJobs: AllCatladderJobs,
+): GitlabJobDef["needs"] {
+  const needs =
+    context.type === "workspace"
+      ? getGitlabNeedsForWorkspaceJob(context, job, allJobs)
+      : getGitlabNeedsForComponentJob(context, job, allJobs);
+
+  return deduplicateNeeds(needs);
+}
+function deduplicateNeeds(needs: GitlabJobDef["needs"]): GitlabJobDef["needs"] {
+  return needs
+    ? [...new Map(needs.map((n) => [isObject(n) ? n.job : n, n])).values()]
+    : undefined;
+}
+
+function getGitlabNeedsForComponentJob(
+  context: ComponentContext,
+  { needsStages, needs, needsOtherComponent }: CatladderJob<string>,
+  allJobs: AllCatladderJobs,
+): GitlabJobDef["needs"] {
+  const needsFromStages = needsStages?.flatMap<CatladderJobNeed>((n) => {
+    const componentName = context.name;
+    if (!n.workspaceName) {
+      const allJobNamesFromThatStage =
+        allJobs.components
+          .filter(
+            (j) =>
+              j.context.name === componentName && j.context.env === context.env,
+          )
+          .flatMap((j) => j.jobs)
+          ?.filter((j) => j.stage === n.stage)
+          ?.map((j) => j.name) ?? [];
+
+      return allJobNamesFromThatStage.map((job) => ({
+        job,
+        artifacts: n.artifacts ?? false,
+        componentName,
+      }));
+    } else {
+      const allJobNamesFromThatStage =
+        allJobs.workspaces
+          .find(
+            (w) =>
+              w.context.name === n.workspaceName &&
+              w.context.env === context.env,
+          )
+          ?.jobs?.flatMap((j) => j)
+          ?.filter((j) => j.stage === n.stage)
+          ?.map((j) => j.name) ?? [];
+
+      return allJobNamesFromThatStage.map((job) => ({
         job,
-        allJobs,
-        baseRules,
-      );
-      return {
-        name: fullJobName,
-        gitlabJob,
-        context,
-      };
-    });
+        artifacts: n.artifacts ?? false,
+        workspaceName: n.workspaceName,
+      }));
+    }
   });
-};
+  const cleanedNeeds: CatladderJob["needs"] = [
+    ...(needsFromStages ?? []),
+    ...(needs ?? []),
+    // pull in legacy needs from other component, which is now identical to needs
+    ...(needsOtherComponent ?? []),
+  ];
+
+  return cleanedNeeds
+    ?.map((n) =>
+      isObject(n)
+        ? "workspaceName" in n
+          ? {
+              job: getFullReferencedJobNameFromWorkspace(
+                n.job,
+                n.workspaceName,
+                context.env,
+                allJobs,
+              ),
+              artifacts: n.artifacts,
+            }
+          : {
+              job: getFullReferencedJobNameFromComponent(
+                n.job,
+                n.componentName ?? context.name,
+                context.env,
+                allJobs,
+              ),
+              artifacts: n.artifacts,
+            }
+        : getFullReferencedJobNameFromComponent(
+            n,
+            context.name,
+            context.env,
+            allJobs,
+          ),
+    ) // sort in a predictable manner for snapshot tests
+    .sort((a, b) => getJobName(a).localeCompare(getJobName(b)));
+}
+/**
+ *
+ *unclear whether we actually need this. So far jobs in a workspace don't have needs to other jobs from the same workspace
+ */
+function getGitlabNeedsForWorkspaceJob(
+  context: WorkspaceContext,
+  { needsStages, needs }: CatladderJob<string>,
+  allJobs: AllCatladderJobs,
+): GitlabJobDef["needs"] {
+  const needsFromStages = needsStages?.flatMap<CatladderJobNeed>((n) => {
+    const workspaceName = n.workspaceName ?? context.name;
+    const allJobNamesFromThatStage =
+      allJobs.workspaces
+        .filter(
+          (j) =>
+            j.context.name === workspaceName && j.context.env === context.env,
+        )
+        .flatMap((j) => j.jobs)
+        ?.filter((j) => j.stage === n.stage)
+        ?.map((j) => j.name) ?? [];
+
+    return allJobNamesFromThatStage.map((job) => ({
+      job,
+      artifacts: n.artifacts ?? false,
+      workspaceName: workspaceName,
+    }));
+  });
+  const cleanedNeeds: CatladderJob["needs"] = [
+    ...(needsFromStages ?? []),
+    ...(needs ?? []),
+  ];
+
+  return cleanedNeeds
+    ?.map((n) =>
+      isObject(n)
+        ? {
+            job: getFullReferencedJobNameFromWorkspace(
+              n.job,
+              "workspaceName" in n && n.workspaceName
+                ? n.workspaceName
+                : context.name,
+              context.env,
+              allJobs,
+            ),
+            artifacts: n.artifacts,
+          }
+        : getFullReferencedJobNameFromWorkspace(
+            n,
+            context.name,
+            context.env,
+            allJobs,
+          ),
+    ) // sort in a predictable manner for snapshot tests
+    .sort((a, b) => getJobName(a).localeCompare(getJobName(b)));
+}

package/src/pipeline/packageManager.ts

@@ -1,7 +1,11 @@
 import { existsSync } from "fs";
 import { join } from "path";
 import { pathEqual } from "path-equal";
-import type { Config, PackageManagerInfoComponent } from "../types";
+import type {
+  Config,
+  PackageManagerInfoBase,
+  PackageManagerInfoComponent,
+} from "../types";
 import {
   getWorkspaces,
   getYarnVersion,
@@ -12,13 +16,11 @@ export const getPackageManagerInfoForComponent = async (
   config: Config,
   componentName: string,
 ): Promise<PackageManagerInfoComponent> => {
+  const baseInfo = await getPackageManagerInfoBase();
+  const { workspaces } = baseInfo;
   // currently only supports yarn
-  const version = await getYarnVersion();
-  if (!version) throw new Error("could not get yarn version");
-  const isClassic = version.startsWith("1");
 
   const component = config.components[componentName];
-  const workspaces = await getWorkspaces(isClassic);
   const currentWorkspace = workspaces.find((w) =>
     pathEqual(component.dir, w.location),
   );
@@ -54,13 +56,27 @@ export const getPackageManagerInfoForComponent = async (
     ...currentWorkspaceDependencies,
   ];
   return {
-    type: "yarn",
-    workspaces,
-    version,
-    isClassic,
+    ...baseInfo,
     currentWorkspace,
     currentWorkspaceDependencies,
     componentIsInWorkspace,
     pathsToCopyInDocker,
   };
 };
+// TODO: memoizee
+export const getPackageManagerInfoBase =
+  async (): Promise<PackageManagerInfoBase> => {
+    // currently only supports yarn
+    const version = await getYarnVersion();
+    if (!version) throw new Error("could not get yarn version");
+    const isClassic = version.startsWith("1");
+
+    const workspaces = await getWorkspaces(isClassic);
+
+    return {
+      type: "yarn",
+      workspaces,
+      version,
+      isClassic,
+    };
+  };

package/src/types/config.ts

@@ -4,7 +4,7 @@ import type { DeployConfig } from "../deploy/types";
 import type { CatladderJob } from "./jobs";
 import type { ComponentContext } from "./context";
 import type { PartialDeep } from "./utils";
-import type { PipelineType } from "..";
+import type { PipelineType, WorkspaceBuildConfig } from "..";
 
 export const ALL_PIPELINE_TRIGGERS = [
   "mainBranch",
@@ -200,6 +200,9 @@ export type Config<C extends ConfigProps = never> = {
    *
    */
   domainCanonical?: string;
+
+  // shared workspace Builds
+  builds?: Record<string, WorkspaceBuildConfig>;
   /**
    * components (sub apps)
    */