@catladder/pipeline 0.0.0-node16-ddfae617 → 0.0.0-perf-cli-reduce-bundle-size-with-tsconfig-optimization-3556aaa4

package/src/deploy/utils.ts

@@ -1,8 +1,9 @@
-import { Config, Context } from "../types";
-import { DeployConfigKubernetesCluster, isOfDeployType } from "./types";
+import type { Config, ComponentContext } from "../types";
+import type { DeployConfigKubernetesCluster } from "./types";
+import { isOfDeployType } from "./types";
 
 export const getFullKubernetesClusterName = (
-  cluster: DeployConfigKubernetesCluster
+  cluster: DeployConfigKubernetesCluster,
 ) => {
   if (cluster.type === "gcloud") {
     return `gke_${cluster.projectId}_${cluster.region}_${cluster.name}`;
@@ -11,17 +12,20 @@ export const getFullKubernetesClusterName = (
 
 export const getKubernetesNamespace = (
   config: Pick<Config, "customerName" | "appName">,
-  env: string
+  env: string,
 ) => {
   return `${config.customerName}-${config.appName}-${env}`;
 };
 
-export const contextIsStoppable = (context: Context) => {
-  const deployConfig = context.componentConfig.deploy;
+export const contextIsStoppable = (context: ComponentContext) => {
+  const deployConfig = context.deploy?.config;
   if (isOfDeployType(deployConfig, "kubernetes")) {
     return true;
   }
 
+  if (isOfDeployType(deployConfig, "google-cloudrun")) {
+    return true;
+  }
   if (isOfDeployType(deployConfig, "custom") && deployConfig.stopScript) {
     return true;
   }

package/src/index.ts

@@ -1,4 +1,4 @@
-export * from "./pipeline/";
+export * from "./pipeline";
 export * from "./types";
 export * from "./rules";
 export * from "./runner";
@@ -6,3 +6,4 @@ export * from "./config";
 export * from "./context";
 export * from "./build";
 export * from "./deploy";
+export * from "./utils/writeFiles";

package/src/packageInfos.ts

@@ -0,0 +1,14 @@
+// we can't import the package.json directly, because the bundling process will inline it
+// but the version is only set during deployement, when its already bundled
+// instead we need to use require.main.path to get the path of the main module, which is actually the file in the bin
+// this will point to the root package.json
+let ownPkg = { version: "0.0.0" };
+try {
+  const path = require.main?.path + "/../package.json";
+
+  ownPkg = require(path);
+} catch (e) {
+  // ignore, currently in the gitlab-includes, this doesn't work
+}
+
+export default ownPkg;

package/src/pipeline/__tests__/__snapshots__/getPipelineStages.test.ts.snap

@@ -0,0 +1,69 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`getPipelineStages() should return all envs for SIMPLE_CONFIG1 1`] = `
+[
+  "setup",
+  "setup dev",
+  "setup review",
+  "setup stage",
+  "setup prod",
+  "setup dev2",
+  "setup review2",
+  "setup stage2",
+  "setup prod2",
+  "test",
+  "test dev",
+  "test review",
+  "test stage",
+  "test prod",
+  "test dev2",
+  "test review2",
+  "test stage2",
+  "test prod2",
+  "build",
+  "build dev",
+  "build review",
+  "build stage",
+  "build prod",
+  "build dev2",
+  "build review2",
+  "build stage2",
+  "build prod2",
+  "deploy",
+  "deploy dev",
+  "deploy review",
+  "deploy stage",
+  "deploy prod",
+  "deploy dev2",
+  "deploy review2",
+  "deploy stage2",
+  "deploy prod2",
+  "verify",
+  "verify dev",
+  "verify review",
+  "verify stage",
+  "verify prod",
+  "verify dev2",
+  "verify review2",
+  "verify stage2",
+  "verify prod2",
+  "rollback",
+  "rollback dev",
+  "rollback review",
+  "rollback stage",
+  "rollback prod",
+  "rollback dev2",
+  "rollback review2",
+  "rollback stage2",
+  "rollback prod2",
+  "stop",
+  "stop dev",
+  "stop review",
+  "stop stage",
+  "stop prod",
+  "stop dev2",
+  "stop review2",
+  "stop stage2",
+  "stop prod2",
+]
+`;

package/src/pipeline/__tests__/getPipelineStages.test.ts

@@ -0,0 +1,63 @@
+import { type DeployConfigKubernetesCluster } from "../..";
+import type { Config } from "../../types";
+import { getPipelineStages } from "../getPipelineStages";
+
+describe("getPipelineStages()", () => {
+  const cluster: DeployConfigKubernetesCluster = {
+    type: "gcloud",
+    name: "mega-cluster",
+    projectId: "super-google-project",
+    region: "ch-blabla",
+  };
+  const SIMPLE_CONFIG1: Config = {
+    appName: "my-app",
+    customerName: "pan",
+    components: {
+      app1: {
+        dir: "dir1",
+        build: {
+          type: "node",
+        },
+        deploy: { type: "kubernetes", cluster },
+      },
+      app2: {
+        dir: "dir2",
+        build: {
+          type: "node",
+        },
+        deploy: { type: "kubernetes", cluster },
+        env: {
+          dev2: {
+            type: "dev",
+          },
+          review2: {
+            type: "review",
+          },
+          stage2: {
+            type: "stage",
+          },
+          prod2: {
+            type: "prod",
+          },
+        },
+      },
+      app3: {
+        dir: "dir2",
+        build: {
+          type: "node",
+        },
+        deploy: { type: "kubernetes", cluster },
+        env: {
+          dev: false,
+          review: false,
+          stage: false,
+          prod: false,
+        },
+      },
+    },
+  };
+
+  it("should return all envs for SIMPLE_CONFIG1", () => {
+    expect(getPipelineStages(SIMPLE_CONFIG1)).toMatchSnapshot();
+  });
+});

package/src/pipeline/createAllJobs.ts

@@ -0,0 +1,72 @@
+import type { CreateAllComponentsContextProps } from "../context/createAllComponentsContext";
+import { createAllComponentsContext } from "../context/createAllComponentsContext";
+import { createWorkspaceContext } from "../context/createWorkspaceContext";
+import type { ComponentContext, WorkspaceContext } from "../types";
+import { componentContextHasWorkspaceBuild } from "../types";
+import type { CatladderJob } from "../types/jobs";
+import { createJobsForComponentContext } from "./createJobsForComponent";
+import { createJobsForWorkspace } from "./createJobsForWorkspace";
+
+export type AllCatladderJobs = {
+  workspaces: Array<{
+    context: WorkspaceContext;
+    jobs: Array<CatladderJob>;
+  }>;
+  components: Array<{
+    context: ComponentContext;
+    jobs: Array<CatladderJob>;
+  }>;
+};
+
+export const createAllJobs = async ({
+  config,
+  trigger,
+  pipelineType,
+}: CreateAllComponentsContextProps): Promise<AllCatladderJobs> => {
+  const allComponentsContext = await createAllComponentsContext({
+    config,
+    trigger,
+    pipelineType,
+  });
+
+  return {
+    workspaces: await Promise.all(
+      Object.keys(config.builds ?? {}).map(async (workspaceName) => {
+        const componentsInAllEnvs = allComponentsContext.filter(
+          (context) =>
+            componentContextHasWorkspaceBuild(context) &&
+            context.build.workspaceName === workspaceName,
+        );
+
+        const allEnvs = [...new Set(componentsInAllEnvs.map(({ env }) => env))];
+
+        return await Promise.all(
+          allEnvs.map(async (env) => {
+            const workspaceContext = await createWorkspaceContext({
+              components: componentsInAllEnvs.filter(
+                ({ env: componentEnv }) => componentEnv === env,
+              ),
+              workspaceName,
+              config,
+              trigger,
+              pipelineType,
+              env,
+            });
+
+            return {
+              context: workspaceContext,
+              jobs: createJobsForWorkspace(workspaceContext),
+            };
+          }),
+        );
+      }),
+    ).then((f) => f.flat()),
+
+    components: allComponentsContext.map((context) => {
+      return {
+        context,
+        jobs: createJobsForComponentContext(context),
+      };
+    }),
+  };
+};

package/src/pipeline/createChildPipeline.ts

@@ -1,56 +1,37 @@
-import { getAllEnvsByTrigger, getAllEnvsInAllComponents } from "../config";
 import { RULES_ALWAYS } from "../rules";
-import { getRunnerImage } from "../runner";
-import { GitlabPipeline, Pipeline, PipelineJob, PipelineType } from "../types";
-import { Config, PipelineTrigger } from "../types/config";
-import { createJobs } from "./createJobs";
 
-const baseStages = ["setup", "test", "build", "deploy", "verify", "stop"];
+import type { Pipeline, PipelineType } from "../types";
+import type { Config, PipelineTrigger } from "../types/config";
+import { createAllJobs } from "./createAllJobs";
+import { getPipelineStages } from "./getPipelineStages";
+import { createGitlabJobs } from "./gitlab/createGitlabJobs";
+import { createGitlabPipelineWithDefaults } from "./gitlab/createGitlabPipeline";
+
 export const createChildPipeline = async <T extends PipelineType>(
-  type: T,
+  pipelineType: T,
   trigger: PipelineTrigger,
-  config: Config
+  config: Config,
 ): Promise<Pipeline<T>> => {
-  const components = Object.keys(config.components);
-
-  // 2. write the triggering pipeline
-  const jobs = await components.reduce<Promise<Record<string, PipelineJob<T>>>>(
-    async (acc, componentName) => {
-      const envs = getAllEnvsByTrigger(config, componentName, trigger);
-      return {
-        ...(await acc),
-        ...(await createJobs(type, envs, config, componentName, trigger)),
-      };
-    },
-    Promise.resolve({})
-  );
+  const jobs = await createAllJobs({
+    config,
+    trigger,
+    pipelineType,
+  });
 
-  // while technically not required, we group different envs in its own stage
-  // each job from `createJobs` that is defined as `envMode: "stagePerEnv"` will have `deploy dev`, etc. instead of just `deploy`
-  // this is just so that it looks nicer in gitlab and makes running mutliple manual tasks more easy to use
+  const stages = getPipelineStages(config);
 
-  const allEnvs = getAllEnvsInAllComponents(config);
-  const stages = baseStages.reduce<string[]>(
-    (acc, baseStage) => [
-      ...acc,
-      baseStage,
-      ...allEnvs.map((e) => `${baseStage} ${e}`),
-    ],
-    []
-  );
-  if (type === "gitlab") {
-    const pipeline: GitlabPipeline = {
-      image: getRunnerImage("jobs-default"), // default image
-      variables: {
-        FF_USE_FASTZIP: "true",
-      },
+  if (pipelineType === "gitlab") {
+    const gitlabJobs = await createGitlabJobs(jobs);
+    return createGitlabPipelineWithDefaults({
       workflow: {
         rules: RULES_ALWAYS,
       },
       stages,
-      jobs,
-    };
-    return pipeline as Pipeline<T>;
+      jobs: Object.fromEntries(
+        gitlabJobs.map(({ gitlabJob, name }) => [name, gitlabJob]),
+      ),
+      variables: config.runnerVariables,
+    }) as Pipeline<T>;
   }
-  throw new Error(`${type} is not supported`);
+  throw new Error(`${pipelineType} is not supported`);
 };

package/src/pipeline/createJobsForComponent.ts

@@ -0,0 +1,35 @@
+import { BUILD_TYPES } from "../build";
+import { DEPLOY_TYPES } from "../deploy";
+import type { ComponentContext } from "../types/context";
+import type { CatladderJob } from "../types/jobs";
+
+const injectDefaultVarsInCustomJobs = (
+  context: ComponentContext,
+  jobs: CatladderJob[],
+) =>
+  jobs.map(({ variables, ...job }) => ({
+    variables: {
+      ...(context.environment.envVars ?? {}),
+      ...(variables ?? {}),
+    },
+    ...job,
+  }));
+const getCustomJobs = (context: ComponentContext) => {
+  if (!context.customJobs) {
+    return [];
+  }
+  const rawJobs = context.customJobs;
+  return injectDefaultVarsInCustomJobs(context, rawJobs);
+};
+export const createJobsForComponentContext = (
+  context: ComponentContext,
+): CatladderJob[] => {
+  const buildJobs = BUILD_TYPES[context.build.buildType].jobs(context);
+  const deployJobs =
+    context.componentConfig.deploy !== false
+      ? DEPLOY_TYPES[context.componentConfig.deploy.type].jobs(context)
+      : [];
+
+  const customJobs = getCustomJobs(context);
+  return [...buildJobs, ...deployJobs, ...customJobs];
+};

package/src/pipeline/createJobsForWorkspace.ts

@@ -0,0 +1,12 @@
+import { WORKSPACE_BUILD_TYPES } from "../build";
+import type { WorkspaceContext } from "../types/context";
+import type { CatladderJob } from "../types/jobs";
+
+export const createJobsForWorkspace = (
+  context: WorkspaceContext,
+): CatladderJob[] => {
+  const buildJobs =
+    WORKSPACE_BUILD_TYPES[context.build.buildType].jobs(context);
+
+  return buildJobs;
+};

package/src/pipeline/createMainPipeline.ts

@@ -0,0 +1,111 @@
+import {
+  RULE_IS_MAIN_BRANCH,
+  RULE_IS_MERGE_REQUEST,
+  RULE_IS_TAGGED_RELEASE,
+  RULE_NEVER_ON_RELEASE_COMMIT,
+} from "../rules";
+import type {
+  ComponentContext,
+  GitlabJobDef,
+  GitlabRule,
+  Pipeline,
+  PipelineTrigger,
+  PipelineType,
+  WorkspaceContext,
+} from "../types";
+import { ALL_PIPELINE_TRIGGERS, type Config } from "../types/config";
+import { createAllJobs } from "./createAllJobs";
+import { getPipelineStages } from "./getPipelineStages";
+import { createGitlabJobs } from "./gitlab/createGitlabJobs";
+import { createGitlabPipelineWithDefaults } from "./gitlab/createGitlabPipeline";
+import { getGitlabReleaseJobs } from "./gitlab/gitlabReleaseJobs";
+
+export const createMainPipeline = async <T extends PipelineType>(
+  pipelineType: T,
+  config: Config,
+): Promise<Pipeline<T>> => {
+  const stages = getPipelineStages(config);
+
+  if (pipelineType === "gitlab") {
+    // for all triggers create jobs and add base rules
+
+    const allJobsPerTrigger = await Promise.all(
+      ALL_PIPELINE_TRIGGERS.map(
+        async (trigger) =>
+          await createGitlabJobs(
+            await createAllJobs({ config, trigger, pipelineType }),
+            getGitlabRulesForTrigger(trigger),
+          ),
+      ),
+    ).then((j) => j.flat());
+
+    const allWorkspaceJobs = allJobsPerTrigger
+      .filter((j) => j.context.type === "workspace") // sort by componentName in the same order as they appear in the config
+      // this is purely for better readability in git diffs when you add new components
+      .sort((a, b) => {
+        const workspaceNames = Object.keys(config.builds ?? {});
+        const aIndex = workspaceNames.findIndex(
+          (c) => c === (a.context as WorkspaceContext).name,
+        );
+        const bIndex = workspaceNames.findIndex(
+          (c) => c === (b.context as WorkspaceContext).name,
+        );
+        return aIndex - bIndex;
+      });
+
+    const allComponentJobs = allJobsPerTrigger
+      .filter((j) => j.context.type === "component")
+      // sort by componentName in the same order as they appear in the config
+      // this is purely for better readability in git diffs when you add new components
+      .sort((a, b) => {
+        const componentNames = Object.keys(config.components);
+        const aIndex = componentNames.findIndex(
+          (c) => c === (a.context as ComponentContext).name,
+        );
+        const bIndex = componentNames.findIndex(
+          (c) => c === (b.context as ComponentContext).name,
+        );
+        return aIndex - bIndex;
+      });
+    const allJobs = [...allWorkspaceJobs, ...allComponentJobs].reduce(
+      (acc, { gitlabJob, name }) => {
+        // merge jobs, if a job is already there, merge the rules
+        // this is currently needed because of envMode: "none", which creates the same job for all triggers, so it can appear multiple times
+        if (acc[name]) {
+          acc[name].rules = [
+            ...(acc[name].rules ?? []),
+            ...(gitlabJob.rules ?? []),
+          ];
+        } else {
+          acc[name] = gitlabJob;
+        }
+
+        return acc;
+      },
+      {} as { [key: string]: GitlabJobDef },
+    );
+
+    return createGitlabPipelineWithDefaults({
+      stages: [...stages, "release"],
+      jobs: {
+        ...allJobs,
+        ...getGitlabReleaseJobs(),
+      },
+      variables: config.runnerVariables,
+    }) as Pipeline<T>;
+  }
+  throw new Error(`${pipelineType} is not supported`);
+};
+function getGitlabRulesForTrigger(trigger: PipelineTrigger): GitlabRule[] {
+  // mainBranch: on push to main branch
+  // mr: on merge request
+  // taggedRelease: on tag
+  switch (trigger) {
+    case "mainBranch":
+      return [RULE_NEVER_ON_RELEASE_COMMIT, RULE_IS_MAIN_BRANCH];
+    case "mr":
+      return [RULE_IS_MERGE_REQUEST];
+    case "taggedRelease":
+      return [RULE_IS_TAGGED_RELEASE];
+  }
+}

package/src/pipeline/generatePipelineFiles.ts

@@ -0,0 +1,76 @@
+import type { Config, PipelineMode, PipelineType } from "../types";
+import { writeYamlfile } from "../utils/writeFiles";
+import { createChildPipeline } from "./createChildPipeline";
+import { createMainPipeline } from "./createMainPipeline";
+import { getPipelineTriggerForGitlabChildPipeline } from "./gitlab/getPipelineTriggerForGitlabChildPipeline";
+import { sortGitLabJobDefProps } from "./gitlab/sortGitLabJobDefProps";
+
+export async function generateChildPipelineContent<T extends PipelineType>(
+  config: Config,
+  pipelineType: T,
+) {
+  const trigger = getPipelineTriggerForGitlabChildPipeline();
+  const { jobs, image, stages, variables, workflow, ...pipelineRest } =
+    await createChildPipeline(pipelineType, trigger, config);
+
+  const jobsWithSortedProps = Object.fromEntries(
+    Object.entries(jobs).map(([jobName, job]) => [
+      jobName,
+      sortGitLabJobDefProps(job),
+    ]),
+  );
+
+  return {
+    image,
+    stages,
+    variables,
+    workflow,
+    ...pipelineRest,
+    // jobs need to be spread into main YAML, because GitLab pipeline YAML has no jobs key - jobs are top level with their key as their name
+    ...jobsWithSortedProps,
+  };
+}
+
+export async function generateLocalPipelineContent<T extends PipelineType>(
+  config: Config,
+  pipelineType: T,
+) {
+  const { jobs, image, stages, variables, workflow, ...pipelineRest } =
+    await createMainPipeline(pipelineType, config);
+
+  const jobsWithSortedProps = Object.fromEntries(
+    Object.entries(jobs).map(([jobName, job]) => [
+      jobName,
+      sortGitLabJobDefProps(job),
+    ]),
+  );
+
+  return {
+    image,
+    stages,
+    variables,
+    workflow,
+    ...pipelineRest,
+    // jobs need to be spread into main YAML, because GitLab pipeline YAML has no jobs key - jobs are top level with their key as their name
+    ...jobsWithSortedProps,
+  };
+}
+
+export async function generatePipelineFiles<T extends PipelineType>(
+  config: Config,
+  pipelineType: T,
+  mode: PipelineMode<T>,
+) {
+  if (mode === "childpipeline") {
+    const pipelineContent = await generateChildPipelineContent(
+      config,
+      pipelineType,
+    );
+    return writeYamlfile(`__pipeline.yml`, pipelineContent);
+  }
+  const pipelineContent = await generateLocalPipelineContent(
+    config,
+    pipelineType,
+  );
+  return writeYamlfile(`.gitlab-ci.yml`, pipelineContent);
+}

package/src/pipeline/getPipelineStages.ts

@@ -0,0 +1,23 @@
+import { getAllEnvsInAllComponents } from "../config";
+import type { Config } from "../types/config";
+import { BASE_STAGES } from "../types/jobs";
+
+/**
+ * while technically not required, we group different envs in its own stage
+ * each job from `createJobs` that is defined as `envMode: "stagePerEnv"` will have `deploy dev`, etc. instead of just `deploy`
+ * this is just so that it looks nicer in gitlab and makes running mutliple manual tasks more easy to use
+ */
+export function getPipelineStages(config: Config) {
+  const allEnvs = getAllEnvsInAllComponents(config).filter(
+    (e) => e !== "local",
+  );
+  const stages = BASE_STAGES.reduce<string[]>(
+    (acc, baseStage) => [
+      ...acc,
+      baseStage,
+      ...allEnvs.map((e) => `${baseStage} ${e}`),
+    ],
+    [],
+  );
+  return stages;
+}