@catladder/pipeline 1.10.2 → 1.13.0

package/src/context/index.ts

@@ -1,11 +1,15 @@
-import { merge, mergeWith } from "lodash";
 import slugify from "slugify";
 import { BUILD_TYPES } from "../build";
 import { BuildConfig } from "../build/types";
 import { DEPLOY_TYPES, getKubernetesNamespace } from "../deploy";
 import { DeployConfig } from "../deploy/types";
 import { Config, isKnowEnvType, DevLocalEnvConfig } from "../types/config";
-import { CommitInfo, Context, Environment, YarnInfo } from "../types/context";
+import {
+  CommitInfo,
+  Context,
+  Environment,
+  PackageManagerInfo,
+} from "../types/context";
 import { mergeWithMergingArrays } from "../utils";
 
 const sanitizeForEnVar = (s: string) => s.replace(/-/g, "_");
@@ -164,7 +168,7 @@ export const createContext = (
   componentName: string,
   env: string,
   commitInfo?: CommitInfo,
-  yarnInfo?: YarnInfo
+  packageManagerInfo?: PackageManagerInfo
 ): Context => {
   if (!/^[a-z0-9-]+$/.test(componentName)) {
     throw new Error(
@@ -208,6 +212,6 @@ export const createContext = (
     componentName,
     environment: getEnvironment(config, componentName, env, commitInfo),
     commitInfo,
-    yarnInfo,
+    packageManagerInfo,
   };
 };

package/src/deploy/base.ts

@@ -1,4 +1,5 @@
-import { GitlabJob, Context, GitlabJobDef } from "../types";
+import { Context } from "../types";
+import { CatladderJob } from "../types/jobs";
 
 export const DEPLOY_JOB_NAME = "🚀 Deploy";
 export const STOP_JOB_NAME = "🛑 Stop ⚠️";
@@ -9,9 +10,7 @@ const DEPLOY_RUNNER_VARIABLES = {
   KUBERNETES_MEMORY_REQUEST: "200Mi",
   KUBERNETES_MEMORY_LIMIT: "500Mi",
 };
-type JobWithoutScript = Omit<GitlabJob, "job"> & {
-  job: Omit<GitlabJobDef, "script">;
-};
+type JobWithoutScript = Omit<CatladderJob, "script">;
 export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
   const environment = {
     name: context.environment.fullName,
@@ -45,26 +44,24 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
         artifacts: false,
       },
     ], // workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/220758
-    job: {
-      rules: [
-        autoDeploy
-          ? {
-              when: "on_success",
-            }
-          : {
-              when: "manual",
-            },
-      ],
-      stage: "deploy",
-      dependencies: [],
-      variables: {
-        ...DEPLOY_RUNNER_VARIABLES,
-      },
-      environment: {
-        ...environment,
-        on_stop: STOP_JOB_NAME,
-        auto_stop_in: autoStop,
-      },
+
+    rules: [
+      autoDeploy
+        ? {
+            when: "on_success",
+          }
+        : {
+            when: "manual",
+          },
+    ],
+    stage: "deploy",
+    variables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+    },
+    environment: {
+      ...environment,
+      on_stop: STOP_JOB_NAME,
+      auto_stop_in: autoStop,
     },
   };
 };
@@ -79,29 +76,27 @@ export const getBaseDeploymentStopJob = (
   return {
     name: STOP_JOB_NAME,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
-    job: {
-      needs: [DEPLOY_JOB_NAME],
-      rules: [
-        {
-          if: "$CI_COMMIT_BRANCH =~ /^[0-9]+\\.([0-9]+|x)\\.x$/", // automatic on hotfix branches
-          when: "on_success",
-          allow_failure: true,
-        },
-        {
-          when: "manual",
-          allow_failure: true,
-        },
-      ],
-      variables: {
-        ...DEPLOY_RUNNER_VARIABLES,
-        GIT_STRATEGY: "none",
+
+    needs: [DEPLOY_JOB_NAME],
+    rules: [
+      {
+        if: "$CI_COMMIT_BRANCH =~ /^[0-9]+\\.([0-9]+|x)\\.x$/", // automatic on hotfix branches
+        when: "on_success",
+        allow_failure: true,
       },
-      stage: "stop",
-      dependencies: [],
-      environment: {
-        ...environment,
-        action: "stop",
+      {
+        when: "manual",
+        allow_failure: true,
       },
+    ],
+    variables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+      GIT_STRATEGY: "none",
+    },
+    stage: "stop",
+    environment: {
+      ...environment,
+      action: "stop",
     },
   };
 };

package/src/deploy/index.ts

@@ -1,5 +1,5 @@
-import { GitlabJobs } from "../types/gitlab-types";
 import { Context } from "../types/context";
+import { CatladderJob } from "../types/jobs";
 import { createKubernetesDeployJobs } from "./kubernetes";
 import { DeployConfig } from "./types";
 export * from "./kubernetes";
@@ -7,7 +7,7 @@ export * from "./types";
 export * from "./utils";
 export type DeployTypes = {
   [type in DeployConfig["type"]]: {
-    jobs: (context: Context) => GitlabJobs;
+    jobs: (context: Context) => CatladderJob[];
     defaults: () => Partial<Extract<DeployConfig, { type: type }>>;
   };
 };

package/src/deploy/kubernetes/deployJob.ts

@@ -1,18 +1,19 @@
-import { GitlabJobs } from "../../types/gitlab-types";
-import { Context } from "../../types/context";
-import { isOfDeployType } from "../types";
-import { getRunnerImage } from "../../runner";
-import { getBaseDeploymentJob, getBaseDeploymentStopJob } from "../base";
-import { merge } from "lodash";
 import { dump } from "js-yaml";
-import { createMongodbBaseConfig } from "./mongodb";
-import { createCloudsqlBaseConfig } from "./cloudsql";
+import { merge } from "lodash";
+import { DeployConfigKubernetesValues } from "..";
 import { getSecretVarNameForContext } from "../..";
+import { getRunnerImage } from "../../runner";
+import { Context } from "../../types/context";
+import { CatladderJob } from "../../types/jobs";
 import { mergeWithMergingArrays } from "../../utils";
-import { DeployConfigKubernetesValues } from "..";
-import { PartialDeep } from "type-fest";
+import { getBaseDeploymentJob, getBaseDeploymentStopJob } from "../base";
+import { isOfDeployType } from "../types";
+import { createCloudsqlBaseConfig } from "./cloudsql";
+import { createMongodbBaseConfig } from "./mongodb";
 
-export const createKubernetesDeployJobs = (context: Context): GitlabJobs => {
+export const createKubernetesDeployJobs = (
+  context: Context
+): CatladderJob[] => {
   const deployConfig = context.componentConfig.deploy;
   if (deployConfig === false) {
     return [];
@@ -91,28 +92,26 @@ export const createKubernetesDeployJobs = (context: Context): GitlabJobs => {
     namespace: context.environment.envVars.KUBE_NAMESPACE,
   };
   const shared = {
-    job: {
-      image: getRunnerImage("kubernetes"),
-      variables: {
-        ...context.environment.envVars,
-        HELM_EXPERIMENTAL_OCI: "1",
-        IMAGE_PULL_SECRET: `gitlab-registry-${context.componentName}`,
-        KUBE_VALUES: dump(kubeValues, {
-          lineWidth: -1,
-          quotingType: "'",
-          forceQuotes: true,
-        }),
-        HELM_GITLAB_CHART_NAME: "the-panter-chart",
-        HELM_ARGS: [
-          ...(deployConfig.debug ? ["--debug"] : []),
-          ...(deployConfig.additionalHelmArgs ?? []),
-        ].join(" "),
-        COMPONENT_NAME: context.componentName,
-        BUILD_ID: context.commitInfo?.buildId,
-        // TODO: unify with docker build stage
-        IMAGE_NAME: context.environment.shortName + "/" + context.componentName,
-        IMAGE_TAG: "$CI_COMMIT_SHA",
-      },
+    image: getRunnerImage("kubernetes"),
+    variables: {
+      ...context.environment.envVars,
+      HELM_EXPERIMENTAL_OCI: "1",
+      IMAGE_PULL_SECRET: `gitlab-registry-${context.componentName}`,
+      KUBE_VALUES: dump(kubeValues, {
+        lineWidth: -1,
+        quotingType: "'",
+        forceQuotes: true,
+      }),
+      HELM_GITLAB_CHART_NAME: "the-panter-chart",
+      HELM_ARGS: [
+        ...(deployConfig.debug ? ["--debug"] : []),
+        ...(deployConfig.additionalHelmArgs ?? []),
+      ].join(" "),
+      COMPONENT_NAME: context.componentName,
+      BUILD_ID: context.commitInfo?.buildId,
+      // TODO: unify with docker build stage
+      IMAGE_NAME: context.environment.shortName + "/" + context.componentName,
+      IMAGE_TAG: "$CI_COMMIT_SHA",
     },
   };
 
@@ -136,24 +135,20 @@ export const createKubernetesDeployJobs = (context: Context): GitlabJobs => {
   ];
   return [
     merge({}, baseDeploymentJob, shared, {
-      job: {
-        script: [
-          ...connectContext,
-          "kubernetesCreateSecret",
-          "kubernetesDeploy",
-          "echo deployment successful 😻",
-        ],
-        environment: {
-          kubernetes: kubernetesEnvironment,
-        },
+      script: [
+        ...connectContext,
+        "kubernetesCreateSecret",
+        "kubernetesDeploy",
+        "echo deployment successful 😻",
+      ],
+      environment: {
+        kubernetes: kubernetesEnvironment,
       },
     }),
     merge({}, baseStopJob, shared, {
-      job: {
-        script: [...connectContext, "kubernetesDelete"],
-        environment: {
-          kubernetes: kubernetesEnvironment,
-        },
+      script: [...connectContext, "kubernetesDelete"],
+      environment: {
+        kubernetes: kubernetesEnvironment,
       },
     }),
   ];

package/src/pipeline/createChildPipeline.test.ts

@@ -26,7 +26,8 @@ describe("createChildPipeline", () => {
     };
 
     it("creates a pipeline for a single app on the main branch", async () => {
-      const { image, stages, workflow, ...jobs } = await createChildPipeline(
+      const { image, jobs } = await createChildPipeline(
+        "gitlab",
         "mainBranch",
         config
       );

package/src/pipeline/createChildPipeline.ts

@@ -1,24 +1,25 @@
 import { getAllEnvsByTrigger, getAllEnvsInAllComponents } from "../config";
-import { createJobs } from "./createJobs";
 import { RULES_ALWAYS } from "../rules";
 import { getRunnerImage } from "../runner";
+import { GitlabPipeline, Pipeline, PipelineJob, PipelineType } from "../types";
 import { Config, PipelineTrigger } from "../types/config";
-import { GitlabJobDef } from "../types/gitlab-types";
+import { createJobs } from "./createJobs";
+
 const baseStages = ["setup", "test", "build", "deploy", "verify", "stop"];
-export const createChildPipeline = async (
+export const createChildPipeline = async <T extends PipelineType>(
+  type: T,
   trigger: PipelineTrigger,
   config: Config
-) => {
+): Promise<Pipeline<T>> => {
   const components = Object.keys(config.components);
 
   // 2. write the triggering pipeline
-
-  const jobs = await components.reduce<Promise<Record<string, GitlabJobDef>>>(
+  const jobs = await components.reduce<Promise<Record<string, PipelineJob<T>>>>(
     async (acc, componentName) => {
       const envs = getAllEnvsByTrigger(config, componentName, trigger);
       return {
         ...(await acc),
-        ...(await createJobs(envs, config, componentName, trigger)),
+        ...(await createJobs(type, envs, config, componentName, trigger)),
       };
     },
     Promise.resolve({})
@@ -37,18 +38,19 @@ export const createChildPipeline = async (
     ],
     []
   );
-
-  const childPipeline = {
-    image: getRunnerImage("jobs-default"), // default image
-    variables: {
-      FF_USE_FASTZIP: "true",
-    },
-    workflow: {
-      rules: RULES_ALWAYS,
-    },
-    stages,
-    ...jobs,
-  };
-
-  return childPipeline as typeof childPipeline & Record<string, GitlabJobDef>;
+  if (type === "gitlab") {
+    const pipeline: GitlabPipeline = {
+      image: getRunnerImage("jobs-default"), // default image
+      variables: {
+        FF_USE_FASTZIP: "true",
+      },
+      workflow: {
+        rules: RULES_ALWAYS,
+      },
+      stages,
+      jobs,
+    };
+    return pipeline as Pipeline<T>;
+  }
+  throw new Error(`${type} is not supported`);
 };

package/src/pipeline/createJobs.ts

@@ -3,13 +3,15 @@ import { isObject } from "lodash";
 import { BUILD_TYPES } from "../build";
 import { createContext } from "../context";
 import { DEPLOY_TYPES } from "../deploy";
+import { PipelineJob, PipelineType } from "../types";
 import { Config, PipelineTrigger } from "../types/config";
-import { Context, CommitInfo } from "../types/context";
-import { GitlabJob, GitlabJobDef, GitlabJobs } from "../types/gitlab-types";
+import { CommitInfo, Context } from "../types/context";
+import { CatladderJob } from "../types/jobs";
 import { notNil } from "../utils";
-import { getYarnInfo } from "./yarnInfo";
+import { makeGitlabJob } from "./gitlab/makeGitlabJob";
+import { getPackageManagerInfo } from "./packageManager";
 
-const createRawJobs = (context: Context): GitlabJobs => {
+const createRawJobs = (context: Context): CatladderJob[] => {
   if (context.componentConfig.deploy === false) {
     return [];
   }
@@ -35,7 +37,7 @@ const getFullReferencedJobName = (
   referencedJobName: string,
   componentName: string,
   env: string,
-  allRawJobs: GitlabJobs
+  allRawJobs: CatladderJob[]
 ) => {
   const referencedJob = allRawJobs.find((j) => j.name === referencedJobName);
   if (!referencedJob) {
@@ -47,18 +49,17 @@ const getFullReferencedJobName = (
 // replaces references to other jobs with the full name
 // the full name contains the componentname and the env name (if any)
 const replaceReferences = (
-  job: GitlabJob,
+  job: CatladderJob,
   componentName: string,
   env: string,
-  allRawJobs: GitlabJobs
-): GitlabJobDef => {
-  const def = job.job;
+  allRawJobs: CatladderJob[]
+): CatladderJob<string> => {
   const stage =
-    job.envMode === "stagePerEnv" ? `${def.stage} ${env}` : def.stage;
+    job.envMode === "stagePerEnv" ? `${job.stage} ${env}` : job.stage;
   return {
-    ...def,
+    ...job,
     stage,
-    needs: def.needs?.map((n) =>
+    needs: job.needs?.map((n) =>
       isObject(n)
         ? {
             job: getFullReferencedJobName(
@@ -71,25 +72,22 @@ const replaceReferences = (
           }
         : getFullReferencedJobName(n, componentName, env, allRawJobs)
     ),
-    environment: def.environment?.on_stop
+    environment: job.environment?.on_stop
       ? {
-          ...def.environment,
+          ...job.environment,
           on_stop: getFullReferencedJobName(
-            def.environment.on_stop,
+            job.environment.on_stop,
             componentName,
             env,
             allRawJobs
           ),
         }
-      : def.environment,
-    dependencies: def.dependencies?.map((n) =>
-      getFullReferencedJobName(n, componentName, env, allRawJobs)
-    ),
+      : job.environment,
   };
 };
 
 // this can be removed once https://gitlab.com/gitlab-org/gitlab/-/issues/220758 is resolved
-const addStageNeeds = (jobs: GitlabJobs): GitlabJobs => {
+const addStageNeeds = (jobs: CatladderJob[]): CatladderJob[] => {
   // when a job defines needsStages, we add these as needs
   return jobs.map((job) => {
     if (!job.needsStages || job.needsStages.length === 0) {
@@ -98,9 +96,7 @@ const addStageNeeds = (jobs: GitlabJobs): GitlabJobs => {
 
     const neededJobs = jobs
       .map((j) => {
-        const neededStage = job.needsStages?.find(
-          (s) => j.job.stage === s.stage
-        );
+        const neededStage = job.needsStages?.find((s) => j.stage === s.stage);
         if (neededStage) {
           return {
             job: j.name,
@@ -111,19 +107,19 @@ const addStageNeeds = (jobs: GitlabJobs): GitlabJobs => {
       .filter(notNil);
     return {
       ...job,
-      job: {
-        ...job.job,
-        needs: [...(job.job.needs ?? []), ...neededJobs],
-      },
+
+      needs: [...(job.needs ?? []), ...neededJobs],
     };
   });
 };
-export const createJobs = async (
+
+export const createJobs = async <T extends PipelineType>(
+  type: T,
   envs: string[],
   config: Config,
   componentName: string,
   trigger: PipelineTrigger
-): Promise<Record<string, GitlabJobDef>> => {
+): Promise<Record<string, PipelineJob<T>>> => {
   const commitInfo: CommitInfo = {
     refName: process.env.CI_COMMIT_REF_NAME ?? "unknown",
     refSlug: process.env.CI_COMMIT_REF_SLUG ?? "unknown",
@@ -134,7 +130,7 @@ export const createJobs = async (
     trigger,
   };
 
-  const yarnInfo = await getYarnInfo(config, componentName);
+  const packageManagerInfo = await getPackageManagerInfo(config, componentName);
 
   return envs.reduce((acc, env) => {
     const context = createContext(
@@ -142,22 +138,35 @@ export const createJobs = async (
       componentName,
       env,
       commitInfo,
-      yarnInfo
+      packageManagerInfo
     );
     const jobs = addStageNeeds(createRawJobs(context));
 
-    return {
+    const result = {
       ...acc,
-      ...jobs.reduce((acc, job) => {
-        return {
-          ...acc,
-          [getFullJobName(
-            job.name,
-            componentName,
-            job.envMode !== "none" ? env : undefined
-          )]: replaceReferences(job, componentName, env, jobs),
-        };
+      ...jobs.reduce<Record<string, PipelineJob<T>>>((acc, job) => {
+        const jobWithResolvedReferences = replaceReferences(
+          job,
+          componentName,
+          env,
+          jobs
+        );
+        const jobName = getFullJobName(
+          job.name,
+          componentName,
+          job.envMode !== "none" ? env : undefined
+        );
+        if (type === "gitlab") {
+          return {
+            ...acc,
+            [jobName]: makeGitlabJob(
+              jobWithResolvedReferences
+            ) as PipelineJob<T>,
+          };
+        }
+        throw new Error("not supported");
       }, {}),
     };
+    return result;
   }, {});
 };

package/src/pipeline/gitlab/makeGitlabJob.ts

@@ -0,0 +1,16 @@
+import { BASE_RETRY } from "../../defaults";
+import { GitlabJobDef } from "../../types";
+import { CatladderJob } from "../../types/jobs";
+
+export const makeGitlabJob = ({
+  envMode,
+  needsStages,
+  name,
+  ...rest
+}: CatladderJob<string>): GitlabJobDef => {
+  return {
+    ...rest,
+    retry: BASE_RETRY,
+    interruptible: true,
+  };
+};

package/src/pipeline/packageManager.ts

@@ -0,0 +1,99 @@
+import { exec } from "child-process-promise";
+import { Config, PackageManagerInfo } from "../types";
+import { pathEqual } from "path-equal";
+import memoizee from "memoizee";
+import { join } from "path";
+import { existsSync } from "fs";
+
+const execOrFail = async (cmd: string, onFail: string): Promise<string> => {
+  try {
+    return await exec(cmd).then((r) => r.stdout);
+  } catch (e) {
+    return onFail ?? null;
+  }
+};
+
+const getYarnVersion = memoizee(
+  async () => {
+    return await execOrFail("yarn --version", "");
+  },
+  { promise: true }
+);
+
+const getWorkspaces = memoizee(
+  async (isClassic: boolean): Promise<PackageManagerInfo["workspaces"]> => {
+    return isClassic
+      ? Object.values(
+          JSON.parse(
+            JSON.parse(await execOrFail("yarn workspaces --json info", "{}"))
+              ?.data ?? "{}"
+          )
+        )
+      : JSON.parse(
+          `[${(await execOrFail("yarn workspaces list --json --verbose", ""))
+            .trim()
+            .split("\n")
+            .join(",")}]`
+        );
+  },
+  { promise: true }
+);
+export const getPackageManagerInfo = async (
+  config: Config,
+  componentName: string
+): Promise<PackageManagerInfo> => {
+  // currently only supports yarn
+  const version = await getYarnVersion();
+  if (!version) throw new Error("could not get yarn version");
+  const isClassic = version.startsWith("1");
+
+  const component = config.components[componentName];
+  const workspaces = await getWorkspaces(isClassic);
+  const currentWorkspace = workspaces.find((w) =>
+    pathEqual(component.dir, w.location)
+  );
+  const componentIsInWorkspace = Boolean(currentWorkspace);
+  const workspaceRoot = "."; // currently we assume the root folder, later on we might support nested workspaces
+  const packageJson = join(component.dir, "package.json");
+  const workspacePackageJson = componentIsInWorkspace
+    ? join(workspaceRoot, "package.json")
+    : null;
+
+  const lockFile = componentIsInWorkspace
+    ? join(workspaceRoot, "yarn.lock")
+    : join(component.dir, "yarn.lock");
+  const configFiles = [".yarnrc", ".yarnrc.yml", ".npmrc", ".yarn"]; // ".yarn" is yarn 2 folder
+  const rcFiles = (
+    componentIsInWorkspace
+      ? configFiles
+      : configFiles.map((f) => join(component.dir, f))
+  ).filter((f) => existsSync(f));
+
+  // get all folders that this workspace depend on
+  // we will later copy them into the docker build
+  const workspaceDependencies = currentWorkspace
+    ? ([
+        ...currentWorkspace.workspaceDependencies,
+        ...currentWorkspace.mismatchedWorkspaceDependencies,
+      ]
+        .map((name) => workspaces.find((w) => w.name === name)?.location)
+        .filter(Boolean) as string[])
+    : [];
+
+  const pathsToCopyInDocker = [
+    packageJson,
+    ...(workspacePackageJson ? [workspacePackageJson] : []),
+    lockFile,
+    ...rcFiles,
+    ...workspaceDependencies,
+  ];
+  return {
+    type: "yarn",
+    workspaces,
+    version,
+    isClassic,
+    currentWorkspace,
+    componentIsInWorkspace,
+    pathsToCopyInDocker,
+  };
+};