@catladder/pipeline 0.0.0-node16-ddfae617 → 0.0.0-perf-cli-reduce-bundle-size-with-tsconfig-optimization-3556aaa4

package/src/pipeline/packageManager.ts

@@ -1,56 +1,28 @@
-import { exec } from "child-process-promise";
-import { Config, PackageManagerInfo } from "../types";
-import { pathEqual } from "path-equal";
-import memoizee from "memoizee";
-import { join } from "path";
 import { existsSync } from "fs";
+import { join } from "path";
+import { pathEqual } from "path-equal";
+import type {
+  Config,
+  PackageManagerInfoBase,
+  PackageManagerInfoComponent,
+} from "../types";
+import {
+  getWorkspaces,
+  getYarnVersion,
+  getWorkspaceDependencies,
+} from "./yarn/yarnUtils";
 
-const execOrFail = async (cmd: string, onFail: string): Promise<string> => {
-  try {
-    return await exec(cmd).then((r) => r.stdout);
-  } catch (e) {
-    return onFail ?? null;
-  }
-};
-
-const getYarnVersion = memoizee(
-  async () => {
-    return await execOrFail("yarn --version", "");
-  },
-  { promise: true }
-);
-
-const getWorkspaces = memoizee(
-  async (isClassic: boolean): Promise<PackageManagerInfo["workspaces"]> => {
-    return isClassic
-      ? Object.values(
-          JSON.parse(
-            JSON.parse(await execOrFail("yarn workspaces --json info", "{}"))
-              ?.data ?? "{}"
-          )
-        )
-      : JSON.parse(
-          `[${(await execOrFail("yarn workspaces list --json --verbose", ""))
-            .trim()
-            .split("\n")
-            .join(",")}]`
-        );
-  },
-  { promise: true }
-);
-export const getPackageManagerInfo = async (
+export const getPackageManagerInfoForComponent = async (
   config: Config,
-  componentName: string
-): Promise<PackageManagerInfo> => {
+  componentName: string,
+): Promise<PackageManagerInfoComponent> => {
+  const baseInfo = await getPackageManagerInfoBase();
+  const { workspaces } = baseInfo;
   // currently only supports yarn
-  const version = await getYarnVersion();
-  if (!version) throw new Error("could not get yarn version");
-  const isClassic = version.startsWith("1");
 
   const component = config.components[componentName];
-  const workspaces = await getWorkspaces(isClassic);
   const currentWorkspace = workspaces.find((w) =>
-    pathEqual(component.dir, w.location)
+    pathEqual(component.dir, w.location),
   );
   const componentIsInWorkspace = Boolean(currentWorkspace);
   const workspaceRoot = "."; // currently we assume the root folder, later on we might support nested workspaces
@@ -63,37 +35,48 @@ export const getPackageManagerInfo = async (
     ? join(workspaceRoot, "yarn.lock")
     : join(component.dir, "yarn.lock");
   const configFiles = [".yarnrc", ".yarnrc.yml", ".npmrc", ".yarn"]; // ".yarn" is yarn 2 folder
-  const rcFiles = (
-    componentIsInWorkspace
-      ? configFiles
-      : configFiles.map((f) => join(component.dir, f))
-  ).filter((f) => existsSync(f));
+  // copy all those configFiles from workspace root and/or component folder
+
+  const configFilePaths = [
+    ...configFiles,
+    ...configFiles.map((f) => join(component.dir, f)),
+  ].filter((f) => existsSync(f));
 
   // get all folders that this workspace depend on
   // we will later copy them into the docker build
-  const workspaceDependencies = currentWorkspace
-    ? ([
-        ...currentWorkspace.workspaceDependencies,
-        ...currentWorkspace.mismatchedWorkspaceDependencies,
-      ]
-        .map((name) => workspaces.find((w) => w.name === name)?.location)
-        .filter(Boolean) as string[])
+  const currentWorkspaceDependencies = currentWorkspace
+    ? getWorkspaceDependencies(currentWorkspace, workspaces)
     : [];
 
   const pathsToCopyInDocker = [
     packageJson,
     ...(workspacePackageJson ? [workspacePackageJson] : []),
     lockFile,
-    ...rcFiles,
-    ...workspaceDependencies,
+    ...configFilePaths,
+    ...currentWorkspaceDependencies,
   ];
   return {
-    type: "yarn",
-    workspaces,
-    version,
-    isClassic,
+    ...baseInfo,
     currentWorkspace,
+    currentWorkspaceDependencies,
     componentIsInWorkspace,
     pathsToCopyInDocker,
   };
 };
+// TODO: memoizee
+export const getPackageManagerInfoBase =
+  async (): Promise<PackageManagerInfoBase> => {
+    // currently only supports yarn
+    const version = await getYarnVersion();
+    if (!version) throw new Error("could not get yarn version");
+    const isClassic = version.startsWith("1");
+
+    const workspaces = await getWorkspaces(isClassic);
+
+    return {
+      type: "yarn",
+      workspaces,
+      version,
+      isClassic,
+    };
+  };

package/src/pipeline/yarn/yarnUtils.ts

@@ -0,0 +1,66 @@
+import { exec } from "child-process-promise";
+import memoizee from "memoizee";
+import type { YarnWorkspace } from "../../types";
+import { jsonParseOrThrow } from "../../utils/jsonParse";
+
+const execOrFail = async (cmd: string, onFail: string): Promise<string> => {
+  try {
+    return await exec(cmd).then((r) => r.stdout);
+  } catch (e) {
+    return onFail ?? null;
+  }
+};
+// export for mocking
+export const getYarnVersion = memoizee(
+  async () => {
+    return await execOrFail("yarn --version", "");
+  },
+  { promise: true },
+);
+
+// export for mocking
+export const getWorkspaces = memoizee(
+  async (isClassic: boolean): Promise<Array<YarnWorkspace>> => {
+    return isClassic
+      ? Object.values(
+          jsonParseOrThrow(
+            jsonParseOrThrow(
+              await execOrFail("yarn workspaces --json info", "{}"),
+            )?.data ?? "{}",
+          ),
+        )
+      : jsonParseOrThrow(
+          `[${(await execOrFail("yarn workspaces list --json --verbose", ""))
+            .trim()
+            .split("\n")
+            .join(",")}]`,
+        );
+  },
+  { promise: true },
+);
+
+// recursivly get all workspace dependencies
+export const getWorkspaceDependencies = (
+  ws: YarnWorkspace,
+  allWorkspaces: YarnWorkspace[],
+): string[] => {
+  return ws
+    ? ([...ws.workspaceDependencies, ...ws.mismatchedWorkspaceDependencies]
+        .flatMap((location) => {
+          // we have to do this recursivly
+
+          const otherWorkspace = allWorkspaces.find(
+            (w) => w.location === location,
+          );
+
+          if (otherWorkspace) {
+            return [
+              ...getWorkspaceDependencies(otherWorkspace, allWorkspaces),
+              otherWorkspace.location,
+            ];
+          }
+          return [];
+        })
+        .filter(Boolean) as string[])
+    : [];
+};

package/src/rules/index.ts

@@ -1,37 +1,70 @@
-import { GitlabRule } from "../types";
+import type { GitlabRule } from "../types";
+export const RULE_CONDITION_MAIN_BRANCH =
+  "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH";
 
-const NEVER_ON_RELEASE_COMMIT: GitlabRule = {
+export const RULE_IS_MAIN_BRANCH: GitlabRule = {
+  if: RULE_CONDITION_MAIN_BRANCH,
+};
+export const RULE_NEVER_ON_RELEASE_COMMIT: GitlabRule = {
   if: "$CI_COMMIT_MESSAGE =~ /^chore\\(release\\).*/",
   when: "never",
 };
+
+export const RULE_NEVER_ON_SCHEDULE: GitlabRule = {
+  if: '$CI_PIPELINE_SOURCE  == "schedule"',
+  when: "never",
+};
+
+export const RULE_IS_MERGE_REQUEST: GitlabRule = {
+  if: "$CI_MERGE_REQUEST_ID",
+};
+
+export const RULE_IS_TAGGED_RELEASE: GitlabRule = {
+  if: "$CI_COMMIT_TAG",
+};
+
 export const RULES_ALWAYS: GitlabRule[] = [
-  {
-    if: "$CI_COMMIT_TAG",
-  },
-  NEVER_ON_RELEASE_COMMIT,
+  RULE_IS_TAGGED_RELEASE,
+  RULE_NEVER_ON_RELEASE_COMMIT,
+  RULE_NEVER_ON_SCHEDULE,
   {
     if: "$CI_COMMIT_BRANCH =~ /^[0-9]+.([0-9]+|x).x$/",
   },
   {
     if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH",
   },
-  {
-    if: "$CI_MERGE_REQUEST_ID",
-  },
+  RULE_IS_MERGE_REQUEST,
 ];
 
+export const RULE_CONDITION_HOTFIX_BRANCH =
+  "$CI_COMMIT_BRANCH =~ /^[0-9]+.([0-9]+|x).x$/";
+
 export const RULES_RELEASE: GitlabRule[] = [
-  NEVER_ON_RELEASE_COMMIT,
+  RULE_NEVER_ON_RELEASE_COMMIT,
+  RULE_NEVER_ON_SCHEDULE,
   {
-    if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $AUTO_RELEASE == "true"',
+    if: RULE_CONDITION_MAIN_BRANCH + ' && $AUTO_RELEASE == "true"',
     when: "on_success",
   },
   {
-    if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH",
+    if: RULE_CONDITION_MAIN_BRANCH,
+    when: "manual",
+  },
+  {
+    if: RULE_CONDITION_HOTFIX_BRANCH,
+    when: "manual",
+  },
+];
+
+export const RULES_MANUAL_RELEASE: GitlabRule[] = [
+  RULE_NEVER_ON_RELEASE_COMMIT,
+  RULE_NEVER_ON_SCHEDULE,
+  {
+    if: RULE_CONDITION_MAIN_BRANCH,
     when: "manual",
   },
   {
-    if: "$CI_COMMIT_BRANCH =~ /^[0-9]+.([0-9]+|x).x$/", // hotfix branches
+    if: RULE_CONDITION_HOTFIX_BRANCH,
     when: "manual",
   },
 ];

package/src/runner/index.ts

@@ -1,12 +1,13 @@
 import { PIPELINE_IMAGE_TAG, DOCKER_REGISTRY } from "../constants";
 
-type RunnerImageName =
+export type RunnerImageName =
   | "jobs-default"
   | "jobs-meteor"
   | "jobs-testing-chrome"
   | "kubernetes"
   | "base-pipeline"
   | "docker-build"
+  | "gcloud"
   | "semantic-release";
 export const getRunnerImage = (imageName: RunnerImageName) =>
   DOCKER_REGISTRY + "/" + imageName + ":" + PIPELINE_IMAGE_TAG;

package/src/types/config.ts

@@ -1,9 +1,17 @@
-import { BuildConfig } from "../build/types";
-import { DeployConfig } from "../deploy/types";
-import { PartialDeep } from "type-fest";
-import { CatladderJob } from "./jobs";
-import { Context } from "./context";
-export type PipelineTrigger = "mainBranch" | "mr" | "taggedRelease";
+import type { BuildConfig } from "../build/types";
+import type { DeployConfig } from "../deploy/types";
+
+import type { CatladderJob } from "./jobs";
+import type { ComponentContext } from "./context";
+import type { PartialDeep } from "./utils";
+import type { PipelineType, WorkspaceBuildConfig } from "..";
+
+export const ALL_PIPELINE_TRIGGERS = [
+  "mainBranch",
+  "mr",
+  "taggedRelease",
+] as const;
+export type PipelineTrigger = (typeof ALL_PIPELINE_TRIGGERS)[number];
 
 /**
  * all env types with their trigger.
@@ -11,7 +19,7 @@ export type PipelineTrigger = "mainBranch" | "mr" | "taggedRelease";
  */
 export const ENV_TYPES = {
   dev: {
-    triggers: ["mainBranch", "taggedRelease"], // we also trigger dev on tagged release, so that the versions match
+    triggers: ["mainBranch"],
   },
   review: {
     triggers: ["mr"],
@@ -35,7 +43,7 @@ export const ENV_TYPES = {
 export const getEnvTypesByTrigger = (trigger: PipelineTrigger) =>
   Object.entries(ENV_TYPES)
     .filter(([, e]) =>
-      (e.triggers as readonly PipelineTrigger[]).includes(trigger)
+      (e.triggers as readonly PipelineTrigger[]).includes(trigger),
     )
     .map(([e]) => e as EnvType);
 
@@ -59,6 +67,7 @@ export type EnvVars = {
    * You can reuse vars from other components in the same project using ${componentName:variableName}
    * EXAMPLE: A_VAR: "the other var is ${api:OTHER_VAR}"
    */
+
   public?: Record<string, any>;
   /**
    * secret env vars. These vars can be managed with catladder/cli
@@ -104,7 +113,12 @@ export type EnvConfig<E extends EnvType = EnvType> = {
   host?: string;
 } & PartialDeep<DefaultEnvConfig>;
 
-export type Env = {
+export type EnvConfigWithComponent = EnvConfig<EnvType> & ComponentConfig;
+
+export type CustomEnv = EnvConfig & {
+  type: EnvType;
+};
+export type Env<C extends ConfigProps = never> = {
   /**
    * local is a special env that is only used in local development
    */
@@ -114,13 +128,16 @@ export type Env = {
   stage?: EnvConfig<"stage"> | false;
   review?: EnvConfig<"review"> | false;
   prod?: EnvConfig<"prod"> | false;
-} & Record<string, EnvConfig | false>;
-
-export type ComponentConfig = {
+  // unfortunatly, typescript does not properly support objects with a mix of known and unknown properties.
+  // for backwards compatiblity we allow unknown props, but we cannot type it (typescript problem)
+  // however, we now support providing custom envs through a generic parameter of `Config`
+} & Record<C["CustomEnvs"], CustomEnv> &
+  Record<string, any>;
+export type ComponentConfig<C extends ConfigProps = never> = {
   /**
    * specify environment configurations
    */
-  env?: Env;
+  env?: Env<C>;
   /**
    * the directory of the component (e.g. where the package.json or similar is located). You can set "." if you only have one app.
    */
@@ -134,10 +151,41 @@ export type ComponentConfig = {
    * to let us know about why you need to use custom jobs.
    * This feedback will help us to generalize use cases
    */
-  customJobs?: CatladderJob[] | ((context: Context) => CatladderJob[]);
+  customJobs?:
+    | CatladderJob[]
+    | ((context: Omit<ComponentContext, "customJobs">) => CatladderJob[]);
+
+  /**
+   * whether to create a .env
+   *
+   * Be careful, this will overwrite any existing .env file!
+   *
+   * if set to true it will create .env locally and during build.
+   * Be careful: the .env file usually ends up in the build artifacts.
+   * Use this for apps and clients where env does not contain any secrets
+   *
+   *
+   * if set to "local" it will only create .env file locally.
+   * Use this in combination with @dotenvx/dotenvx or node 20 to start apps locally with .env files
+   * During build and runtime, it relies on the env vars set in the environment
+   *
+   */
+  dotEnv?: boolean | "local";
+  /**
+   * whether to create a env.d.ts localy and during build jobs
+   *
+   * Be careful, this will overwrite any existing env.d.ts file!
+   */
+  envDTs?: boolean;
 } & DefaultEnvConfig;
 
-export type Config = {
+export type ConfigProps = {
+  CustomEnvs: string;
+};
+
+export type Config<C extends ConfigProps = never> = {
+  pipelineType?: PipelineType;
+
   /**
    * name of the customer or group
    */
@@ -152,10 +200,13 @@ export type Config = {
    *
    */
   domainCanonical?: string;
+
+  // shared workspace Builds
+  builds?: Record<string, WorkspaceBuildConfig>;
   /**
    * components (sub apps)
    */
-  components: Record<string, ComponentConfig>;
+  components: Record<string, ComponentConfig<C>>;
 
   /**
    * additional meta data (only for organisational purposes)
@@ -163,4 +214,9 @@ export type Config = {
   meta?: {
     labels?: Record<string, string>;
   };
+
+  /**
+   * additional vars only for the runner in all jobs.
+   */
+  runnerVariables?: Record<string, string>;
 };