@catladder/pipeline 2.0.5 → 2.2.0

package/examples/__utils__/helpers.ts

@@ -1,27 +1,11 @@
 import { stringify } from "yaml";
 
 import type { Config } from "../../src";
-import {
-  createChildPipeline,
-  generateLocalPipelineContent,
-  yamlStringifyOptions,
-} from "../../src";
-
-const ALL_TRIGGERS = ["mainBranch", "taggedRelease", "mr"] as const;
-
-export const createAllPipelines = async (config: Config) =>
-  Object.fromEntries(
-    await Promise.all(
-      ALL_TRIGGERS.map(async (trigger) => [
-        trigger,
-        await createChildPipeline("gitlab", trigger, config),
-      ]),
-    ),
-  );
+import { getGitlabCompletePipeline, yamlStringifyOptions } from "../../src";
 
 export const createYamlLocalPipeline = async (
   config: Config,
 ): Promise<string> => {
-  const pipelineContent = await generateLocalPipelineContent(config, "gitlab");
+  const pipelineContent = await getGitlabCompletePipeline(config);
   return stringify(pipelineContent, yamlStringifyOptions);
 };

package/examples/rails-k8s-with-worker-dockerfile.test.ts

@@ -1,4 +1,4 @@
-import { mkdirSync, rmSync, writeFileSync } from "fs";
+import { mkdir, rm, writeFile } from "fs/promises";
 import { createYamlLocalPipeline } from "./__utils__/helpers";
 import config from "./rails-k8s-with-worker";
 import { merge } from "lodash";
@@ -6,8 +6,15 @@ import { join } from "path";
 
 it("matches snapshot with a Dockerfile", async () => {
   const appDir = ".temp-with-dockerfile";
-  mkdirSync(appDir);
-  writeFileSync(join(appDir, "Dockerfile"), "");
+  try {
+    await rm(appDir, {
+      recursive: true,
+    });
+  } catch (e) {
+    // ignore
+  }
+  await mkdir(appDir);
+  await writeFile(join(appDir, "Dockerfile"), "");
   expect(
     await createYamlLocalPipeline(
       merge(config, {
@@ -20,7 +27,7 @@ it("matches snapshot with a Dockerfile", async () => {
     ),
   ).toMatchSnapshot();
 
-  rmSync(appDir, {
+  await rm(appDir, {
     recursive: true,
   });
 });

package/package.json

@@ -53,7 +53,7 @@
     }
   ],
   "license": "MIT",
-  "version": "2.0.5",
+  "version": "2.2.0",
   "scripts": {
     "build:tsc": "yarn tsc",
     "build": "yarn build:compile && yarn build:inline-variables",
@@ -69,28 +69,27 @@
   "main": "dist/index.js",
   "sideEffects": false,
   "devDependencies": {
-    "@babel/cli": "^7.16.0",
-    "@babel/core": "^7.0.0-0",
-    "@babel/plugin-syntax-typescript": "^7.16.7",
-    "@babel/preset-typescript": "^7.16.5",
-    "@jest/globals": "^29.3.1",
-    "@types/child-process-promise": "^2.2.2",
-    "@types/jest": "^27.4.0",
-    "@types/lodash": "^4.14.185",
-    "@types/node": "^20.14.2",
+    "@babel/cli": "^7.24.8",
+    "@babel/core": "^7.25.2",
+    "@babel/plugin-syntax-typescript": "^7.25.4",
+    "@babel/preset-typescript": "^7.24.7",
+    "@jest/globals": "^29.7.0",
+    "@types/child-process-promise": "^2.2.6",
+    "@types/jest": "^29.5.12",
+    "@types/lodash": "^4.17.7",
+    "@types/node": "^22.5.1",
     "babel-plugin-transform-inline-environment-variables": "^0.4.3",
     "child-process-promise": "^2.2.1",
-    "glob": "^10.4.2",
+    "glob": "^11.0.0",
     "json-schema-to-typescript": "^13.0.2",
     "lodash": "^4.17.21",
-    "memoizee": "^0.4.15",
-    "path-equal": "1.1.3",
-    "slugify": "^1.6.3",
-    "tsx": "^4.16.2",
+    "memoizee": "^0.4.17",
+    "path-equal": "1.2.5",
+    "slugify": "^1.6.6",
     "typescript": "^5.4.5",
-    "yaml": "^2.4.5"
+    "yaml": "^2.5.0"
   },
   "dependencies": {
-    "tsx": "^4.16.2"
+    "tsx": "^4.19.0"
   }
 }

package/src/build/cache/createJobCache.ts

@@ -30,7 +30,7 @@ export const createJobCacheFromCacheConfigs = (
                 const baseDir =
                   c.pathMode === "absolute"
                     ? ""
-                    : c.baseDir ?? context.build.dir;
+                    : (c.baseDir ?? context.build.dir);
                 return join(baseDir, p);
               }),
             ),
@@ -41,7 +41,7 @@ export const createJobCacheFromCacheConfigs = (
   const advancedCaches = advancedCacheDefs.map(
     ({ key, paths, policy, scope, pathMode, buildDir, ...rest }) => {
       const baseDir =
-        pathMode === "absolute" ? "" : buildDir ?? context.build.dir;
+        pathMode === "absolute" ? "" : (buildDir ?? context.build.dir);
       const transformedKey =
         scope === "global"
           ? key

package/src/build/sbom.ts

@@ -22,14 +22,14 @@ export const createSbomBuildJob = (context: ComponentContext): CatladderJob => {
     isStandaloneBuildConfig(buildConfig) &&
     buildConfig.type === "custom" &&
     buildConfig.sbom !== false
-      ? buildConfig.sbom?.jobImage ?? defaultImage
+      ? (buildConfig.sbom?.jobImage ?? defaultImage)
       : defaultImage;
 
   const script =
     isStandaloneBuildConfig(buildConfig) &&
     buildConfig.type === "custom" &&
     buildConfig.sbom !== false
-      ? ensureArrayOrNull(buildConfig.sbom?.command) ?? defaultScript
+      ? (ensureArrayOrNull(buildConfig.sbom?.command) ?? defaultScript)
       : defaultScript;
 
   return {

package/src/catladder-gitlab.ts

@@ -1,17 +1,10 @@
 import { readConfig } from "./config";
 import { generatePipelineFiles } from "./pipeline/generatePipelineFiles";
-import type { PipelineMode } from "./types";
 
-const mode = process.argv[2] || "local";
-
-const config = readConfig().then(async (result) => {
+readConfig().then(async (result) => {
   if (!result?.config) {
     throw new Error("no catladder config found");
   }
 
-  await generatePipelineFiles(
-    result.config,
-    "gitlab",
-    mode as PipelineMode<"gitlab">,
-  );
+  await generatePipelineFiles(result.config, "gitlab");
 });

package/src/config/readConfig.ts

@@ -1,6 +1,5 @@
 import { existsSync, readFileSync } from "fs";
 
-// eslint-disable-next-line @typescript-eslint/no-var-requires
 const tsx = require("tsx/cjs/api");
 
 import { parse } from "yaml";

package/src/context/createComponentContext.ts

@@ -76,7 +76,7 @@ export const createComponentContext = async (
     dir,
     // also copy workspace dependencies in monorepo if packages are shared and they create build artifacts
     ...(mode === "all"
-      ? packageManagerInfo.currentWorkspaceDependencies ?? []
+      ? (packageManagerInfo.currentWorkspaceDependencies ?? [])
       : []),
   ];
   const _getBuildContext = (): BuildContextComponent => {

package/src/context/getEnvironmentVariables.ts

@@ -134,7 +134,7 @@ export const getEnvironmentVariables = async (
   );
 
   const publicEnvVars =
-    options.shouldResolveReferences ?? true
+    (options.shouldResolveReferences ?? true)
       ? await resolveAllReferences(
           publicEnvVarsUnresolved,
           async (otherComponentName) => {

package/src/deploy/base/deploy.ts

@@ -62,11 +62,11 @@ export const createDeployJob = (
         ? []
         : [{ job: SBOM_BUILD_JOB_NAME, artifacts: true }]),
       ...(deployConfig
-        ? deployConfig.waitFor?.map((c) => ({
+        ? (deployConfig.waitFor?.map((c) => ({
             componentName: c,
             job: DEPLOY_JOB_NAME,
             artifacts: false,
-          })) ?? []
+          })) ?? [])
         : []),
     ],
     // we don't want to deploy when there is a broken test
@@ -117,7 +117,7 @@ export const createDeployJob = (
     runnerVariables: {
       ...DEPLOY_RUNNER_VARIABLES,
       ...(jobDefinition.runnerVariables ?? {}),
-      ...(deployConfig ? deployConfig.runnerVariables ?? {} : {}),
+      ...(deployConfig ? (deployConfig.runnerVariables ?? {}) : {}),
     },
     environment: isStoppable
       ? {

package/src/deploy/cloudRun/createJobs/cloudRunServices.ts

@@ -25,7 +25,7 @@ export const getServiceDeployScript = (
   const customConfig = service !== true ? service : undefined;
   const command =
     service !== true
-      ? service?.command ?? context.build.config.startCommand
+      ? (service?.command ?? context.build.config.startCommand)
       : undefined;
 
   const commandArray = command

package/src/pipeline/createMainPipeline.ts

@@ -6,7 +6,6 @@ import {
 } from "../rules";
 import type {
   ComponentContext,
-  GitlabJobDef,
   GitlabRule,
   Pipeline,
   PipelineTrigger,
@@ -16,6 +15,7 @@ import type {
 import { ALL_PIPELINE_TRIGGERS, type Config } from "../types/config";
 import { createAllJobs } from "./createAllJobs";
 import { getPipelineStages } from "./getPipelineStages";
+import type { GitlabJobWithContext } from "./gitlab/createGitlabJobs";
 import { createGitlabJobs } from "./gitlab/createGitlabJobs";
 import { createGitlabPipelineWithDefaults } from "./gitlab/createGitlabPipeline";
 import { getGitlabReleaseJobs } from "./gitlab/gitlabReleaseJobs";
@@ -40,7 +40,7 @@ export const createMainPipeline = async <T extends PipelineType>(
     ).then((j) => j.flat());
 
     const allWorkspaceJobs = allJobsPerTrigger
-      .filter((j) => j.context.type === "workspace") // sort by componentName in the same order as they appear in the config
+      .filter((j) => j.context?.type === "workspace") // sort by componentName in the same order as they appear in the config
       // this is purely for better readability in git diffs when you add new components
       .sort((a, b) => {
         const workspaceNames = Object.keys(config.builds ?? {});
@@ -54,7 +54,7 @@ export const createMainPipeline = async <T extends PipelineType>(
       });
 
     const allComponentJobs = allJobsPerTrigger
-      .filter((j) => j.context.type === "component")
+      .filter((j) => j.context?.type === "component")
       // sort by componentName in the same order as they appear in the config
       // this is purely for better readability in git diffs when you add new components
       .sort((a, b) => {
@@ -68,28 +68,36 @@ export const createMainPipeline = async <T extends PipelineType>(
         return aIndex - bIndex;
       });
     const allJobs = [...allWorkspaceJobs, ...allComponentJobs].reduce(
-      (acc, { gitlabJob, name }) => {
+      (acc, { gitlabJob, name, context }) => {
         // merge jobs, if a job is already there, merge the rules
         // this is currently needed because of envMode: "none", which creates the same job for all triggers, so it can appear multiple times
         if (acc[name]) {
-          acc[name].rules = [
-            ...(acc[name].rules ?? []),
+          acc[name].gitlabJob.rules = [
+            ...(acc[name].gitlabJob.rules ?? []),
             ...(gitlabJob.rules ?? []),
           ];
         } else {
-          acc[name] = gitlabJob;
+          acc[name] = { context, gitlabJob };
         }
 
         return acc;
       },
-      {} as { [key: string]: GitlabJobDef },
+      {} as { [key: string]: GitlabJobWithContext },
     );
 
     return createGitlabPipelineWithDefaults({
       stages: [...stages, "release"],
       jobs: {
         ...allJobs,
-        ...getGitlabReleaseJobs(),
+        ...Object.fromEntries(
+          Object.entries(getGitlabReleaseJobs()).map(([name, gitlabJob]) => [
+            name,
+            {
+              gitlabJob,
+              context: null,
+            },
+          ]),
+        ),
       },
       variables: config.runnerVariables,
     }) as Pipeline<T>;

package/src/pipeline/generatePipelineFiles.ts

@@ -1,76 +1,100 @@
-import type { Config, PipelineMode, PipelineType } from "../types";
+import { mkdir, rm } from "fs/promises";
+import { dirname } from "path";
+import type { Config, GitlabJobDef, PipelineType } from "../types";
 import { writeYamlfile } from "../utils/writeFiles";
-import { createChildPipeline } from "./createChildPipeline";
 import { createMainPipeline } from "./createMainPipeline";
-import { getPipelineTriggerForGitlabChildPipeline } from "./gitlab/getPipelineTriggerForGitlabChildPipeline";
 import { sortGitLabJobDefProps } from "./gitlab/sortGitLabJobDefProps";
 
-export async function generateChildPipelineContent<T extends PipelineType>(
+const CATLADDER_GENERATED_FOLDER = ".catladder-generated";
+
+const GITLAB_GENERATED_FOLDER = CATLADDER_GENERATED_FOLDER + "/gitlab";
+
+type YamlFile = {
+  path: string;
+  content: Record<string, unknown>;
+};
+export async function generatePipelineFiles<T extends PipelineType>(
   config: Config,
   pipelineType: T,
 ) {
-  const trigger = getPipelineTriggerForGitlabChildPipeline();
-  const { jobs, image, stages, variables, workflow, ...pipelineRest } =
-    await createChildPipeline(pipelineType, trigger, config);
-
-  const jobsWithSortedProps = Object.fromEntries(
-    Object.entries(jobs).map(([jobName, job]) => [
-      jobName,
-      sortGitLabJobDefProps(job),
-    ]),
-  );
+  if (pipelineType !== "gitlab") {
+    throw new Error("Pipeline type not supported");
+  }
+  const includes = await getGitlabPipelineIncludes(config);
 
-  return {
-    image,
-    stages,
-    variables,
-    workflow,
-    ...pipelineRest,
-    // jobs need to be spread into main YAML, because GitLab pipeline YAML has no jobs key - jobs are top level with their key as their name
-    ...jobsWithSortedProps,
+  const mainFile: YamlFile = {
+    path: ".gitlab-ci.yml",
+    content: {
+      include: includes.map((i) => i.path),
+    },
   };
-}
 
-export async function generateLocalPipelineContent<T extends PipelineType>(
-  config: Config,
-  pipelineType: T,
-) {
+  const files = [mainFile, ...includes];
+
+  // first clean up the folder
+  await rm(GITLAB_GENERATED_FOLDER, { recursive: true });
+  // write files
+  await Promise.all(
+    files.map(async ({ path, content }) => {
+      await mkdir(dirname(path), { recursive: true });
+      await writeYamlfile(path, content);
+    }),
+  );
+}
+async function getGitlabPipelineIncludes(config: Config) {
   const { jobs, image, stages, variables, workflow, ...pipelineRest } =
-    await createMainPipeline(pipelineType, config);
+    await createMainPipeline("gitlab", config);
+  // we will create 1 include per component or workspace
+  // this is for better readability in git diffs and to avoid problems with yaml files beeing too large
+  // group by context
+  const groups = Object.entries(jobs).reduce(
+    (acc, [jobName, { gitlabJob, context }]) => {
+      const group = !context
+        ? "global-jobs"
+        : context?.type + "/" + context.name;
+      if (!acc[group]) {
+        acc[group] = {};
+      }
 
-  const jobsWithSortedProps = Object.fromEntries(
-    Object.entries(jobs).map(([jobName, job]) => [
-      jobName,
-      sortGitLabJobDefProps(job),
-    ]),
+      acc[group][jobName] = sortGitLabJobDefProps(gitlabJob); // also sort properties for more consistent diffing
+      return acc;
+    },
+    {} as Record<string, Record<string, GitlabJobDef>>,
   );
 
-  return {
-    image,
-    stages,
-    variables,
-    workflow,
-    ...pipelineRest,
-    // jobs need to be spread into main YAML, because GitLab pipeline YAML has no jobs key - jobs are top level with their key as their name
-    ...jobsWithSortedProps,
+  const componentIncludes = Object.entries(groups).map(([group, jobs]) => {
+    return {
+      path: GITLAB_GENERATED_FOLDER + "/" + group + ".yaml",
+      content: jobs,
+    };
+  });
+
+  const mainInclude: YamlFile = {
+    path: GITLAB_GENERATED_FOLDER + "/main.yaml",
+    content: {
+      image,
+      stages,
+      variables,
+      workflow,
+      ...pipelineRest,
+    },
   };
+
+  const includes = [mainInclude, ...componentIncludes];
+  return includes;
 }
 
-export async function generatePipelineFiles<T extends PipelineType>(
-  config: Config,
-  pipelineType: T,
-  mode: PipelineMode<T>,
-) {
-  if (mode === "childpipeline") {
-    const pipelineContent = await generateChildPipelineContent(
-      config,
-      pipelineType,
-    );
-    return writeYamlfile(`__pipeline.yml`, pipelineContent);
-  }
-  const pipelineContent = await generateLocalPipelineContent(
-    config,
-    pipelineType,
-  );
-  return writeYamlfile(`.gitlab-ci.yml`, pipelineContent);
+/**
+ *
+ * for testing purposes
+ */
+export async function getGitlabCompletePipeline(config: Config) {
+  const includes = await getGitlabPipelineIncludes(config);
+
+  return includes.reduce((acc, { content }) => {
+    return {
+      ...acc,
+      ...content, // merge all includes into one object
+    };
+  }, {});
 }

package/src/pipeline/gitlab/createGitlabJobs.ts

@@ -14,11 +14,11 @@ import { collapseableSection } from "../../utils/gitlab";
 import { removeUndefined } from "../../utils/removeUndefined";
 import type { AllCatladderJobs } from "../createAllJobs";
 
-export type AllGitlabJobs = {
-  name: string;
+export type GitlabJobWithContext = {
   gitlabJob: GitlabJobDef;
-  context: Context;
-}[];
+  context: Context | null;
+};
+export type AllGitlabJobs = (GitlabJobWithContext & { name: string })[];
 
 export const GITLAB_ENVIRONMENT_URL_VARIABLE = "CL_GITLAB_ENVIRONMENT_URL";
 const getFullJobName = ({

package/src/pipeline/gitlab/gitlabReleaseJobs.ts

@@ -9,15 +9,13 @@ export const getGitlabReleaseJobs = () => {
     ["create release"]: {
       stage: "release",
       image: getRunnerImage("semantic-release"),
-      script: ["semanticRelease"],
-      after_script: [EXPIRED_TOKEN_HELP],
+      script: ["semanticRelease", EXPIRED_TOKEN_HELP],
       rules: RULES_RELEASE,
     },
     ["⚠️ force create release"]: {
       stage: "release",
       image: getRunnerImage("semantic-release"),
-      script: ["semanticRelease"],
-      after_script: [EXPIRED_TOKEN_HELP],
+      script: ["semanticRelease", EXPIRED_TOKEN_HELP],
       needs: [],
       rules: RULES_MANUAL_RELEASE,
     },

package/src/pipeline/index.ts

@@ -1,4 +1,3 @@
-export * from "./createChildPipeline";
 export * from "./createJobsForComponent";
 export * from "./generatePipelineFiles";
 export * from "./gitlab/gitlabReleaseJobs";

package/src/types/gitlab-types.ts

@@ -1,3 +1,4 @@
+import type { GitlabJobWithContext } from "../pipeline/gitlab/createGitlabJobs";
 import type {
   Artifacts as GitlabCiArtifacts,
   JobTemplate,
@@ -65,6 +66,6 @@ export interface GitlabPipeline extends Pick<JobTemplate, "variables"> {
     rules: GitlabRule[];
   };
   stages: string[];
-  jobs: Record<string, GitlabJobDef>;
+  jobs: Record<string, GitlabJobWithContext>;
   before_script?: string[];
 }

package/src/types/pipeline.ts

@@ -2,10 +2,6 @@ import type { GitlabJobDef, GitlabPipeline } from "./gitlab-types";
 
 export type PipelineType = "gitlab";
 
-export type PipelineMode<T extends PipelineType> = T extends "gitlab"
-  ? "childpipeline" | "local"
-  : "local";
-
 export type PipelineJob<T extends PipelineType> = T extends "gitlab"
   ? GitlabJobDef
   : never;

package/dist/pipeline/createChildPipeline.d.ts

@@ -1,3 +0,0 @@
-import type { Pipeline } from "../types";
-import type { Config, PipelineTrigger } from "../types/config";
-export declare const createChildPipeline: <T extends "gitlab">(pipelineType: T, trigger: PipelineTrigger, config: Config) => Promise<Pipeline<T>>;