@catladder/pipeline 0.0.0-more-collapse-deeb54d7 → 0.0.0-multiple-gitlab-includes-1a40efc0

package/src/build/types.ts

@@ -1,7 +1,42 @@
 import type { Artifacts, EnvVars, GitlabJobImage } from "../types";
 import type { Services } from "../types/gitlab-ci-yml";
 
-import type { CatladderJob } from "../types/jobs";
+import type { CatladderJob, CatladderJobCache } from "../types/jobs";
+
+export type CacheConfigAdvanced = Omit<CatladderJobCache, "paths" | "key"> & {
+  /**
+   * if you set a key, the cache is scoped to the key. Be aware that there is a limit in gitlab of 4 caches.
+   *
+   * see https://gitlab.com/gitlab-org/gitlab/-/issues/421962
+   */
+  key: Required<CatladderJobCache>["key"];
+
+  /**
+   * whether the cache is scoped to the context (component or workspace) or the build directory.
+   *
+   * defaults to context
+   */
+  scope?: "context" | "buildDir" | "global";
+} & CacheConfigSimple;
+
+export type CacheConfigSimple = {
+  /**
+   * paths to cache. All Paths are relative to the context dir.
+   */
+  paths: string[];
+
+  /**
+   * whether paths are relative to the base dir. defaults to relative.
+   */
+  pathMode?: "relative" | "absolute";
+
+  /**
+   * custom base dir
+   */
+  baseDir?: string;
+};
+
+export type CacheConfig = CacheConfigSimple | CacheConfigAdvanced;
 
 export type BuildConfigArtifactsReports = {
   /**
@@ -13,6 +48,20 @@ export type BuildConfigArtifactsReports = {
   junit?: string[];
 };
 
+export type WithCacheConfig = {
+  /*
+  cache config. All Paths are relative to the context dir.
+  */
+  cache?: CacheConfig | CacheConfig[];
+
+  /**
+   * customize cache for the job
+   *
+   * @deprecated use cache
+   */
+  jobCache?: CatladderJob["cache"];
+};
+
 export type TestJobCustom = {
   command?: string | string[];
   jobImage?: GitlabJobImage;
@@ -40,39 +89,18 @@ export type BuildConfigBase = {
    * Can curently not reference other variables from other components
    */
   jobVars?: EnvVars;
+
   /**
-   * additional env vars for the buid jobs
+   * additional vars only for the runner.
+   * Also if you use services: that require env vars, you need to set them here.
    *
-   * @deprecated use jobVars
    */
-  extraVars?: Record<string, string>;
+  runnerVariables?: Record<string, string>;
   /**
    * define the build command
    */
   buildCommand?: string | string[] | null;
 
-  /**
-   * customize docker build
-   */
-  docker?: {
-    /**
-     * Custom Dockerfile lines integrated in the generated Dockerfile before the standard build steps.
-     *
-     * - [runner-images/docker-build/scripts/ensureNodeDockerfile](https://git.panter.ch/catladder/catladder/-/tree/main/runner-images/docker-build/scripts/ensureNodeDockerfile)
-     * - [runner-images/docker-build/scripts/ensureNginxDockerfile](https://git.panter.ch/catladder/catladder/-/tree/main/runner-images/docker-build/scripts/ensureNginxDockerfile)
-     * - [runner-images/docker-build/scripts/ensureMeteorDockerfile](https://git.panter.ch/catladder/catladder/-/tree/main/runner-images/docker-build/scripts/ensureMeteorDockerfile)
-     */
-    additionsBegin?: string[];
-    /**
-     * Custom Dockerfile lines integrated in the generated Dockerfile after the standard build steps.
-     *
-     * - [runner-images/docker-build/scripts/ensureNodeDockerfile](https://git.panter.ch/catladder/catladder/-/tree/main/runner-images/docker-build/scripts/ensureNodeDockerfile)
-     * - [runner-images/docker-build/scripts/ensureNginxDockerfile](https://git.panter.ch/catladder/catladder/-/tree/main/runner-images/docker-build/scripts/ensureNginxDockerfile)
-     * - [runner-images/docker-build/scripts/ensureMeteorDockerfile](https://git.panter.ch/catladder/catladder/-/tree/main/runner-images/docker-build/scripts/ensureMeteorDockerfile)
-     */
-    additionsEnd?: string[];
-  };
-
   /**
    * customize lint, set false to disable
    */
@@ -102,6 +130,8 @@ export type BuildConfigBase = {
 
   /**
    * customize cache for the job
+   *
+   * @deprecated use cache
    */
   jobCache?: CatladderJob["cache"];
 
@@ -113,7 +143,7 @@ export type BuildConfigBase = {
    * custom image to use
    */
   jobImage?: GitlabJobImage;
-};
+} & WithCacheConfig;
 
 export type BuildConfigNodeBase = BuildConfigBase & {
   /**
@@ -125,11 +155,16 @@ export type BuildConfigNodeBase = BuildConfigBase & {
 
 export type BuildConfigNode = {
   type: "node";
+  docker?: Omit<BuildConfigDockerBuiltInNode, "type"> | BuildConfigDocker;
 } & BuildConfigNodeBase;
 
 export type BuildConfigNodeStatic = BuildConfigNodeBase & {
+  /**
+   * @deprecated use type: "node" and docker: { type: "nginx" } instead
+   */
   type: "node-static";
   startCommand?: never;
+  docker?: Omit<BuildConfigDockerBuiltInNgninx, "type"> | BuildConfigDocker;
 };
 
 export type BuildConfigMeteor = BuildConfigNodeBase & {
@@ -139,23 +174,64 @@ export type BuildConfigMeteor = BuildConfigNodeBase & {
    * This is only required if you have custom scripts in your image
    */
   installScripts?: boolean;
+  docker?: Omit<BuildConfigDockerBuiltInMeteor, "type"> | BuildConfigDocker;
 };
 
-export type BuildConfigCustomDocker = BuildConfigBase["docker"] &
-  (
-    | {
-        /**
-         * use the built-in nginx image for simple static apps
-         */
-        type: "nginx";
-      }
-    | {
-        /**
-         * custom docker build, expect that a Dockerfile in your directory
-         */
-        type: "custom";
-      }
-  );
+export type BuildConfigNodeLike =
+  | BuildConfigNode
+  | BuildConfigNodeStatic
+  | BuildConfigStorybook
+  | BuildConfigMeteor;
+
+type BuildConfigDockerWithAdditions = {
+  /**
+   * Custom Dockerfile lines integrated in the generated Dockerfile before the standard build steps.
+   *
+   */
+  additionsBegin?: string[];
+  /**
+   * Custom Dockerfile lines integrated in the generated Dockerfile after the standard build steps.
+   *
+   */
+  additionsEnd?: string[];
+};
+
+type BuildConfigDockerBuiltInNode = {
+  type: "node";
+} & BuildConfigDockerWithAdditions;
+
+type BuildConfigDockerBuiltInMeteor = {
+  type: "meteor";
+} & BuildConfigDockerWithAdditions;
+
+type BuildConfigDockerBuiltInNgninx = {
+  /**
+   * use the built-in nginx image for simple static apps
+   */
+  type: "nginx";
+} & BuildConfigDockerWithAdditions;
+type BuildConfigDockerBuiltIn =
+  | BuildConfigDockerBuiltInNgninx
+  | BuildConfigDockerBuiltInNode
+  | BuildConfigDockerBuiltInMeteor;
+
+type BuildConfigDockerCustom = {
+  /**
+   * custom docker build, expect that a Dockerfile in your directory
+   */
+  type: "custom";
+
+  /**
+   * where to run the docker build, defaults to the root of the repo.
+   *
+   * Depending on your Dockerfile you may want to change it to "component"
+   */
+  buildContextLocation?: "root" | "component";
+};
+
+export type BuildConfigDocker =
+  | BuildConfigDockerBuiltIn
+  | BuildConfigDockerCustom;
 
 export type BuildConfigCustom = Omit<
   BuildConfigBase,
@@ -168,7 +244,7 @@ export type BuildConfigCustom = Omit<
    */
   jobServices?: Services;
 
-  docker: BuildConfigCustomDocker;
+  docker: BuildConfigDocker;
 
   /**
    * custom lint, disabled when not set
@@ -236,10 +312,24 @@ export type BuildConfigRails = BuildConfigBase & {
 };
 
 export type BuildConfigStorybook = BuildConfigNodeBase & {
+  /**
+   * @deprecated use type: "node" and docker: { type: "nginx" } instead
+   */
   type: "storybook";
   startCommand?: never;
 };
-export type BuildConfig =
+
+export type BuildConfigFromWorkspace = {
+  from: string;
+  docker?: BuildConfigDocker;
+  startCommand?: string;
+  /**
+   * additional paths for artifacts,
+   * by default "dist" and ".next" are allways included
+   */
+  artifactsPaths?: string[];
+} & WithCacheConfig;
+export type BuildConfigStandalone =
   | BuildConfigNode
   | BuildConfigNodeStatic
   | BuildConfigStorybook
@@ -247,16 +337,69 @@ export type BuildConfig =
   | BuildConfigCustom
   | BuildConfigRails;
 
-export type BuildConfigType = BuildConfig["type"];
+export type BuildConfig = BuildConfigFromWorkspace | BuildConfigStandalone;
+
+export type BuildConfigStandaloneType = BuildConfigStandalone["type"];
 
-export type BuildConfigGeneric<T extends BuildConfigType> = Extract<
+export type BuildConfigGeneric<T extends BuildConfigStandaloneType> = Extract<
   BuildConfig,
   { type: T }
 >;
 
-export const isOfBuildType = <T extends Array<BuildConfigType>>(
+export const isStandaloneBuildConfig = (
+  t: BuildConfig | false,
+): t is BuildConfigStandalone => {
+  if (!t) return false;
+  return !("from" in t);
+};
+
+export const isOfBuildType = <T extends Array<BuildConfigStandaloneType>>(
   t: BuildConfig,
   ...types: T
 ): t is Extract<BuildConfig, { type: T[number] }> => {
+  if (!isStandaloneBuildConfig(t)) return false;
   return types.includes(t.type);
 };
+
+export type WorkspaceBuildConfigBase = {
+  dir?: string;
+  /**
+   * customize lint, set false to disable
+   */
+  lint?: false | TestJobCustom;
+
+  /**
+   * customize test, set false to disable
+   */
+  test?: false | TestJobCustom;
+
+  /**
+   * customize audit, set false to disable
+   */
+  audit?: false | TestJobCustom;
+  /**
+   * additional vars only for the runner.
+   * Also if you use services: that require env vars, you need to set them here.
+   *
+   */
+  runnerVariables?: Record<string, string>;
+
+  /**
+   * additional CI/CD artifacts reports,
+   * use to display information in merge requests, pipeline views and security dashboards.
+   */
+  artifactsReports?: BuildConfigArtifactsReports;
+
+  jobImage?: GitlabJobImage;
+  /**
+   * tags for the underlying job runner (e.g gitlab)
+   */
+  jobTags?: string[];
+} & WithCacheConfig;
+
+export type WorkspaceBuildConfigNode = {
+  type: "node";
+  buildCommand?: string | string[];
+} & WorkspaceBuildConfigBase;
+
+export type WorkspaceBuildConfig = WorkspaceBuildConfigNode;

package/src/catladder-gitlab.ts

@@ -1,59 +1,10 @@
-import { writeFileSync } from "fs";
-import { dump } from "js-yaml";
-import { readConfigSync } from "./config";
-import { PIPELINE_IMAGE_TAG } from "./constants";
-import { createChildPipeline } from "./pipeline";
-import type { PipelineTrigger } from "./types";
+import { readConfig } from "./config";
+import { generatePipelineFiles } from "./pipeline/generatePipelineFiles";
 
-const {
-  CI_MERGE_REQUEST_ID,
-  CI_COMMIT_TAG,
-  CI_COMMIT_BRANCH,
-  CI_DEFAULT_BRANCH,
-} = process.env;
-
-const isDefaultBranch =
-  Boolean(CI_DEFAULT_BRANCH) && CI_COMMIT_BRANCH === CI_DEFAULT_BRANCH;
-const isHotfixBranch = CI_COMMIT_BRANCH
-  ? /^[0-9]+\.([0-9]+|x)\.x$/.test(CI_COMMIT_BRANCH)
-  : false;
-const isMergeRequest = Boolean(CI_MERGE_REQUEST_ID);
-const isTaggedRelease = Boolean(CI_COMMIT_TAG);
-
-console.info(`catladder version ${PIPELINE_IMAGE_TAG}`);
-
-const trigger: PipelineTrigger | null =
-  isMergeRequest || isHotfixBranch
-    ? "mr"
-    : isDefaultBranch
-    ? "mainBranch"
-    : isTaggedRelease
-    ? "taggedRelease"
-    : null;
-if (trigger) {
-  const config = readConfigSync()?.config;
-  if (!config) {
+readConfig().then(async (result) => {
+  if (!result?.config) {
     throw new Error("no catladder config found");
   }
-  createChildPipeline("gitlab", trigger, config).then(
-    ({ jobs, ...mainPipeline }) => {
-      // need to spread out the jobs
-      writeFileSync(`__pipeline.yml`, dump({ ...jobs, ...mainPipeline }), {
-        encoding: "utf-8",
-      });
-    }
-  );
-} else {
-  throw new Error(
-    "no matching trigger: " +
-      JSON.stringify(
-        {
-          isMergeRequest,
-          isDefaultBranch,
-          isTaggedRelease,
-        },
-        null,
-        2
-      )
-  );
-}
+
+  await generatePipelineFiles(result.config, "gitlab");
+});

package/src/config/__tests__/__snapshots__/getAllEnvs.test.ts.snap

@@ -0,0 +1,31 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`getAllEnvs() should return all envs for app1 1`] = `
+[
+  "dev",
+  "review",
+  "stage",
+  "prod",
+  "local",
+]
+`;
+
+exports[`getAllEnvs() should return all envs for app2 1`] = `
+[
+  "dev",
+  "review",
+  "stage",
+  "prod",
+  "local",
+  "dev2",
+  "review2",
+  "stage2",
+  "prod2",
+]
+`;
+
+exports[`getAllEnvs() should return all envs for app3 1`] = `
+[
+  "local",
+]
+`;

package/src/config/__tests__/__snapshots__/getAllEnvsInAllComponents.test.ts.snap

@@ -0,0 +1,15 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`getAllEnvsInAllComponents() should return all envs for SIMPLE_CONFIG1 1`] = `
+[
+  "dev",
+  "review",
+  "stage",
+  "prod",
+  "local",
+  "dev2",
+  "review2",
+  "stage2",
+  "prod2",
+]
+`;

package/src/config/__tests__/getAllEnvs.test.ts

@@ -0,0 +1,71 @@
+import { type DeployConfigKubernetesCluster } from "../..";
+import type { Config } from "../../types";
+import { getAllEnvs } from "../configruedEnvs";
+
+describe("getAllEnvs()", () => {
+  const cluster: DeployConfigKubernetesCluster = {
+    type: "gcloud",
+    name: "mega-cluster",
+    projectId: "super-google-project",
+    region: "ch-blabla",
+  };
+  const SIMPLE_CONFIG: Config = {
+    appName: "my-app",
+    customerName: "pan",
+    components: {
+      app1: {
+        dir: "dir1",
+        build: {
+          type: "node",
+        },
+        deploy: { type: "kubernetes", cluster },
+      },
+      app2: {
+        dir: "dir2",
+        build: {
+          type: "node",
+        },
+        deploy: { type: "kubernetes", cluster },
+        env: {
+          dev2: {
+            type: "dev",
+          },
+          review2: {
+            type: "review",
+          },
+          stage2: {
+            type: "stage",
+          },
+          prod2: {
+            type: "prod",
+          },
+        },
+      },
+      app3: {
+        dir: "dir2",
+        build: {
+          type: "node",
+        },
+        deploy: { type: "kubernetes", cluster },
+        env: {
+          dev: false,
+          review: false,
+          stage: false,
+          prod: false,
+        },
+      },
+    },
+  };
+
+  it("should return all envs for app1", () => {
+    expect(getAllEnvs(SIMPLE_CONFIG, "app1")).toMatchSnapshot();
+  });
+
+  it("should return all envs for app2", () => {
+    expect(getAllEnvs(SIMPLE_CONFIG, "app2")).toMatchSnapshot();
+  });
+
+  it("should return all envs for app3", () => {
+    expect(getAllEnvs(SIMPLE_CONFIG, "app3")).toMatchSnapshot();
+  });
+});

package/src/config/__tests__/getAllEnvsInAllComponents.test.ts

@@ -0,0 +1,63 @@
+import { type DeployConfigKubernetesCluster } from "../..";
+import type { Config } from "../../types";
+import { getAllEnvsInAllComponents } from "../configruedEnvs";
+
+describe("getAllEnvsInAllComponents()", () => {
+  const cluster: DeployConfigKubernetesCluster = {
+    type: "gcloud",
+    name: "mega-cluster",
+    projectId: "super-google-project",
+    region: "ch-blabla",
+  };
+  const SIMPLE_CONFIG1: Config = {
+    appName: "my-app",
+    customerName: "pan",
+    components: {
+      app1: {
+        dir: "dir1",
+        build: {
+          type: "node",
+        },
+        deploy: { type: "kubernetes", cluster },
+      },
+      app2: {
+        dir: "dir2",
+        build: {
+          type: "node",
+        },
+        deploy: { type: "kubernetes", cluster },
+        env: {
+          dev2: {
+            type: "dev",
+          },
+          review2: {
+            type: "review",
+          },
+          stage2: {
+            type: "stage",
+          },
+          prod2: {
+            type: "prod",
+          },
+        },
+      },
+      app3: {
+        dir: "dir2",
+        build: {
+          type: "node",
+        },
+        deploy: { type: "kubernetes", cluster },
+        env: {
+          dev: false,
+          review: false,
+          stage: false,
+          prod: false,
+        },
+      },
+    },
+  };
+
+  it("should return all envs for SIMPLE_CONFIG1", () => {
+    expect(getAllEnvsInAllComponents(SIMPLE_CONFIG1)).toMatchSnapshot();
+  });
+});

package/src/config/configruedEnvs.ts

@@ -4,25 +4,25 @@ import { DEFAULT_ENV_TYPES, getEnvTypesByTrigger } from "../types";
 const getConfiguredAndDefaultEnvs = (
   config: Config,
   componentName: string,
-  envTypes: EnvType[]
+  envTypes: EnvType[],
 ) => {
   const configuredEnvs = config.components[componentName].env ?? {};
   // the default envs have the same name as the env types
   // these can be disabled with settimg them to `false`
   // this is the list of all not disabled envs. These are always returned
   const enabledDefaultEnvs = envTypes.filter(
-    (e) => configuredEnvs[e] !== false
+    (e) => configuredEnvs[e] !== false,
   );
 
   const configuredCustomEnvs = Object.entries(
-    config.components[componentName].env ?? {}
+    config.components[componentName].env ?? {},
   )
     .filter(
       ([, config]) =>
         config &&
         "type" in config &&
         config.type &&
-        envTypes.includes(config.type)
+        envTypes.includes(config.type),
     )
     .map(([envName]) => envName);
 
@@ -36,7 +36,7 @@ export const getAllEnvs = (config: Config, componentName: string) => {
 export const getAllEnvsInAllComponents = (config: Config) => {
   return [
     ...new Set(
-      Object.keys(config.components).flatMap((c) => getAllEnvs(config, c))
+      Object.keys(config.components).flatMap((c) => getAllEnvs(config, c)),
     ),
   ];
 };
@@ -44,7 +44,7 @@ export const getAllEnvsInAllComponents = (config: Config) => {
 export const getAllEnvsByTrigger = (
   config: Config,
   componentName: string,
-  trigger: PipelineTrigger
+  trigger: PipelineTrigger,
 ) => {
   const envTypesByTrigger = getEnvTypesByTrigger(trigger);
   return getConfiguredAndDefaultEnvs(config, componentName, envTypesByTrigger);

package/src/config/readConfig.ts

@@ -1,6 +1,8 @@
 import { existsSync, readFileSync } from "fs";
-import { register } from "ts-node";
-import { load } from "js-yaml";
+
+const tsx = require("tsx/cjs/api");
+
+import { parse } from "yaml";
 import type { Config } from "../types";
 
 // allows us to load ts files
@@ -8,39 +10,30 @@ import type { Config } from "../types";
 const fullPath = (directory: string, ext: string) =>
   directory + "/catladder." + ext;
 
-function requireUncached(module: string) {
-  delete require.cache[require.resolve(module)];
-  return require(module);
-}
-
-export const readConfigSync = (
-  directory: string = process.cwd()
-): { config: Config; path: string; ext: string } | null => {
-  register({
-    cwd: directory,
-    transpileOnly: true,
-    compilerOptions: {
-      module: "commonjs",
-    },
-  });
-
+export const readConfig = async (
+  directory: string = process.cwd(),
+): Promise<{ config: Config; path: string; ext: string } | null> => {
   const found = ["ts", "js", "yml", "yaml"].find((extension) =>
-    existsSync(fullPath(directory, extension))
+    existsSync(fullPath(directory, extension)),
   );
+
   if (found) {
     const filePath = fullPath(directory, found);
     if (found === "ts" || found === "js") {
+      const result = await tsx.require(filePath, directory);
+
+      // weird: if target project is esm, result is under result.default, but if its commonjs, its under result.default.default
+      const config = result.default.default || result.default;
       return {
         path: filePath,
         ext: found,
-        // eslint-disable-next-line @typescript-eslint/no-var-requires
-        config: requireUncached(filePath).default,
+        config,
       };
     } else {
       return {
         path: filePath,
         ext: found,
-        config: load(readFileSync(filePath, { encoding: "utf-8" })) as Config,
+        config: parse(readFileSync(filePath, { encoding: "utf-8" })) as Config,
       };
     }
   }

package/src/constants.ts

@@ -1,2 +1,5 @@
-export const PIPELINE_IMAGE_TAG = process.env.INLINE_PIPELINE_IMAGE_TAG || "";
-export const DOCKER_REGISTRY = process.env.INLINE_DOCKER_REGISTRY || "";
+export const PIPELINE_IMAGE_TAG =
+  process.env.INLINE_PIPELINE_IMAGE_TAG || "latest";
+export const DOCKER_REGISTRY =
+  process.env.INLINE_DOCKER_REGISTRY ||
+  "git.panter.ch:5001/catladder/catladder";