@catladder/pipeline 0.0.0-node16-ddfae617 → 0.0.0-perf-cli-reduce-bundle-size-with-tsconfig-optimization-3556aaa4

package/src/config/configruedEnvs.ts

@@ -1,33 +1,28 @@
-import {
-  Config,
-  DEFAULT_ENV_TYPES,
-  EnvType,
-  getEnvTypesByTrigger,
-  PipelineTrigger,
-} from "../types";
+import type { Config, EnvType, PipelineTrigger } from "../types";
+import { DEFAULT_ENV_TYPES, getEnvTypesByTrigger } from "../types";
 
 const getConfiguredAndDefaultEnvs = (
   config: Config,
   componentName: string,
-  envTypes: EnvType[]
+  envTypes: EnvType[],
 ) => {
   const configuredEnvs = config.components[componentName].env ?? {};
   // the default envs have the same name as the env types
   // these can be disabled with settimg them to `false`
   // this is the list of all not disabled envs. These are always returned
   const enabledDefaultEnvs = envTypes.filter(
-    (e) => configuredEnvs[e] !== false
+    (e) => configuredEnvs[e] !== false,
   );
 
   const configuredCustomEnvs = Object.entries(
-    config.components[componentName].env ?? {}
+    config.components[componentName].env ?? {},
   )
     .filter(
       ([, config]) =>
         config &&
         "type" in config &&
         config.type &&
-        envTypes.includes(config.type)
+        envTypes.includes(config.type),
     )
     .map(([envName]) => envName);
 
@@ -41,7 +36,7 @@ export const getAllEnvs = (config: Config, componentName: string) => {
 export const getAllEnvsInAllComponents = (config: Config) => {
   return [
     ...new Set(
-      Object.keys(config.components).flatMap((c) => getAllEnvs(config, c))
+      Object.keys(config.components).flatMap((c) => getAllEnvs(config, c)),
     ),
   ];
 };
@@ -49,7 +44,7 @@ export const getAllEnvsInAllComponents = (config: Config) => {
 export const getAllEnvsByTrigger = (
   config: Config,
   componentName: string,
-  trigger: PipelineTrigger
+  trigger: PipelineTrigger,
 ) => {
   const envTypesByTrigger = getEnvTypesByTrigger(trigger);
   return getConfiguredAndDefaultEnvs(config, componentName, envTypesByTrigger);

package/src/config/readConfig.ts

@@ -1,46 +1,40 @@
 import { existsSync, readFileSync } from "fs";
-import { register } from "ts-node";
-import { load } from "js-yaml";
-import { Config } from "../types";
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const tsx = require("tsx/cjs/api");
+
+import { parse } from "yaml";
+import type { Config } from "../types";
 
 // allows us to load ts files
 
 const fullPath = (directory: string, ext: string) =>
   directory + "/catladder." + ext;
 
-function requireUncached(module: string) {
-  delete require.cache[require.resolve(module)];
-  return require(module);
-}
-
-export const readConfigSync = (
-  directory: string = process.cwd()
-): { config: Config; path: string; ext: string } | null => {
-  register({
-    cwd: directory,
-    transpileOnly: true,
-    compilerOptions: {
-      module: "commonjs",
-    },
-  });
-
+export const readConfig = async (
+  directory: string = process.cwd(),
+): Promise<{ config: Config; path: string; ext: string } | null> => {
   const found = ["ts", "js", "yml", "yaml"].find((extension) =>
-    existsSync(fullPath(directory, extension))
+    existsSync(fullPath(directory, extension)),
   );
+
   if (found) {
     const filePath = fullPath(directory, found);
     if (found === "ts" || found === "js") {
+      const result = await tsx.require(filePath, directory);
+
+      // weird: if target project is esm, result is under result.default, but if its commonjs, its under result.default.default
+      const config = result.default.default || result.default;
       return {
         path: filePath,
         ext: found,
-        // eslint-disable-next-line @typescript-eslint/no-var-requires
-        config: requireUncached(filePath).default,
+        config,
       };
     } else {
       return {
         path: filePath,
         ext: found,
-        config: load(readFileSync(filePath, { encoding: "utf-8" })) as Config,
+        config: parse(readFileSync(filePath, { encoding: "utf-8" })) as Config,
       };
     }
   }

package/src/constants.ts

@@ -1,2 +1,5 @@
-export const PIPELINE_IMAGE_TAG = process.env.INLINE_PIPELINE_IMAGE_TAG || "";
-export const DOCKER_REGISTRY = process.env.INLINE_DOCKER_REGISTRY || "";
+export const PIPELINE_IMAGE_TAG =
+  process.env.INLINE_PIPELINE_IMAGE_TAG || "latest";
+export const DOCKER_REGISTRY =
+  process.env.INLINE_DOCKER_REGISTRY ||
+  "git.panter.ch:5001/catladder/catladder";

package/src/context/{resolveReferences.test.ts → __tests__/resolveReferences.test.ts}

@@ -1,10 +1,15 @@
+import type { BashExpression } from "../../bash/BashExpression";
 import {
   resolveReferences,
   translateLegacyFromComponents,
-} from "./resolveReferences";
+} from "../resolveReferences";
 
+const unpackBashExpressions = (obj: Record<string, string | BashExpression>) =>
+  Object.fromEntries(
+    Object.entries(obj).map(([key, value]) => [key, value.toString()]),
+  );
 describe("resolveReferences", () => {
-  it("replaces occurences of ${componentName:VARIABLE_NAME}", () => {
+  it("replaces occurences of ${componentName:VARIABLE_NAME}", async () => {
     const variables = {
       a: "hello world",
       b: "a replaced value looks like this: '${api:FOO}', nice!",
@@ -17,33 +22,30 @@ describe("resolveReferences", () => {
         FOO: "foo from frontend",
       },
     };
-    const result = resolveReferences(
-      variables,
-      (componentName, variableName) => {
-        return otherVariables[componentName]?.[variableName];
-      }
-    );
+    const result = await resolveReferences(variables, async (componentName) => {
+      return otherVariables[componentName];
+    });
 
-    expect(result).toEqual({
+    expect(unpackBashExpressions(result)).toEqual({
       a: "hello world",
       b: "a replaced value looks like this: 'foo from api', nice!",
     });
   });
 
-  it("replaces self references with structure ${VARIABLE_NAME}", () => {
+  it("replaces self references with structure ${VARIABLE_NAME}", async () => {
     const variables = {
       FOO: "hello world",
       BAR: "this: ${FOO}!",
     };
-    const result = resolveReferences(variables);
+    const result = await resolveReferences(variables);
 
-    expect(result).toEqual({
+    expect(unpackBashExpressions(result)).toEqual({
       FOO: "hello world",
       BAR: "this: hello world!",
     });
   });
 
-  it("replaces self references mixed with other vars", () => {
+  it("replaces self references mixed with other vars", async () => {
     const variables = {
       FOO: "hello from ${api:FOO}",
       BAR: "this: ${FOO}!",
@@ -53,39 +55,33 @@ describe("resolveReferences", () => {
         FOO: "foo from api",
       },
     };
-    const result = resolveReferences(
-      variables,
-      (componentName, variableName) => {
-        return otherVariables[componentName]?.[variableName];
-      }
-    );
+    const result = await resolveReferences(variables, async (componentName) => {
+      return otherVariables[componentName];
+    });
 
-    expect(result).toEqual({
+    expect(unpackBashExpressions(result)).toEqual({
       FOO: "hello from foo from api",
       BAR: "this: hello from foo from api!",
     });
   });
 
-  it("keeps variables as is if not found (when null is returned)", () => {
+  it("keeps variables as is if not found (when null is returned)", async () => {
     const variables = {
       a: "hello world",
       b: "a not found value looks like this: '${api:FOO}'",
     };
     const otherVariables: Record<string, Record<string, string>> = {};
-    const result = resolveReferences(
-      variables,
-      (componentName, variableName) => {
-        return otherVariables[componentName]?.[variableName];
-      }
-    );
+    const result = await resolveReferences(variables, async (componentName) => {
+      return otherVariables[componentName];
+    });
 
-    expect(result).toEqual({
+    expect(unpackBashExpressions(result)).toEqual({
       a: "hello world",
       b: "a not found value looks like this: '${api:FOO}'",
     });
   });
 
-  it("replaces mulitple levels deep", () => {
+  it("replaces mulitple levels deep", async () => {
     const variables = {
       a: "value is ${frontend:FOO}!",
     };
@@ -97,19 +93,16 @@ describe("resolveReferences", () => {
         FOO: "hi, ${api:FOO}",
       },
     };
-    const result = resolveReferences(
-      variables,
-      (componentName, variableName) => {
-        return otherVariables[componentName]?.[variableName] ?? "";
-      }
-    );
+    const result = await resolveReferences(variables, async (componentName) => {
+      return otherVariables[componentName];
+    });
 
-    expect(result).toEqual({
+    expect(unpackBashExpressions(result)).toEqual({
       a: "value is hi, foo from api!",
     });
   });
 
-  it("prevents endless loops", () => {
+  it("prevents endless loops", async () => {
     const variables = {
       a: "value is ${frontend:FOO}!",
     };
@@ -121,14 +114,11 @@ describe("resolveReferences", () => {
         FOO: "frontend ${api:FOO}",
       },
     };
-    const result = resolveReferences(
-      variables,
-      (componentName, variableName) => {
-        return otherVariables[componentName]?.[variableName] ?? "";
-      }
-    );
+    const result = await resolveReferences(variables, async (componentName) => {
+      return otherVariables[componentName];
+    });
 
-    expect(result).toEqual({
+    expect(unpackBashExpressions(result)).toEqual({
       a: "value is frontend api ${frontend:FOO}!",
     });
   });

package/src/context/createAllComponentsContext.ts

@@ -0,0 +1,31 @@
+import { getAllEnvsByTrigger } from "../config/configruedEnvs";
+import type { ComponentContext, PipelineType } from "../types";
+import type { Config, PipelineTrigger } from "../types/config";
+import { createComponentContext } from "./createComponentContext";
+
+export type CreateAllComponentsContextProps = {
+  config: Config;
+  trigger: PipelineTrigger;
+  pipelineType: PipelineType;
+};
+
+export const createAllComponentsContext = async ({
+  config,
+  trigger,
+  pipelineType,
+}: CreateAllComponentsContextProps): Promise<Array<ComponentContext>> => {
+  return await Promise.all(
+    Object.keys(config.components).flatMap((componentName) => {
+      const envs = getAllEnvsByTrigger(config, componentName, trigger);
+      return envs.map(async (env) => {
+        return await createComponentContext({
+          config,
+          componentName,
+          env,
+          trigger,
+          pipelineType,
+        });
+      });
+    }),
+  );
+};

package/src/context/createComponentContext.ts

@@ -0,0 +1,135 @@
+import { isFunction } from "lodash";
+import { BUILD_TYPES, isStandaloneBuildConfig } from "../build";
+import type { BuildConfig } from "../build/types";
+import { DEPLOY_TYPES } from "../deploy";
+import type { DeployConfig, DeployConfigType } from "../deploy/types";
+import { getPackageManagerInfoForComponent } from "../pipeline/packageManager";
+import type { PipelineType } from "../types";
+import type { Config, PipelineTrigger } from "../types/config";
+import type {
+  BuildContext,
+  BuildContextComponent,
+  ComponentContext,
+} from "../types/context";
+import type { PartialDeep } from "../types/utils";
+import { mergeWithMergingArrays } from "../utils";
+import { getEnvironment } from "./getEnvironment";
+import { getEnvironmentContext } from "./getEnvironmentContext";
+
+export type CreateComponentContextContext = {
+  config: Config;
+  componentName: string;
+  env: string;
+  pipelineType?: PipelineType;
+  trigger?: PipelineTrigger;
+};
+
+export const createComponentContext = async (
+  ctx: CreateComponentContextContext,
+): Promise<ComponentContext> => {
+  if (!/^[a-z0-9-]+$/.test(ctx.componentName)) {
+    throw new Error(
+      "componentName may only contain lower case letters, numbers and -",
+    );
+  }
+
+  const packageManagerInfo = await getPackageManagerInfoForComponent(
+    ctx.config,
+    ctx.componentName,
+  );
+
+  const envContext = getEnvironmentContext(ctx);
+
+  const componentConfigWithoutDefaults = envContext.envConfigRaw;
+
+  const resolvedBuildType = isStandaloneBuildConfig(
+    componentConfigWithoutDefaults.build,
+  )
+    ? componentConfigWithoutDefaults.build.type
+    : ctx.config.builds?.[componentConfigWithoutDefaults.build.from].type;
+  if (!resolvedBuildType) {
+    throw new Error("build type not found, is the build config correct?");
+  }
+  const defaults: {
+    build: PartialDeep<BuildConfig>;
+    deploy: PartialDeep<DeployConfig>;
+  } = componentConfigWithoutDefaults.deploy
+    ? {
+        build: BUILD_TYPES[resolvedBuildType].defaults(envContext),
+        deploy: DEPLOY_TYPES[
+          componentConfigWithoutDefaults.deploy.type as DeployConfigType
+        ].defaults(envContext as any),
+      }
+    : {
+        build: BUILD_TYPES[resolvedBuildType].defaults(envContext),
+        deploy: {},
+      };
+
+  const componentConfig = mergeWithMergingArrays(
+    defaults,
+    componentConfigWithoutDefaults,
+  );
+
+  const environment = await getEnvironment(ctx);
+  const { deploy, build, customJobs, dir } = componentConfig;
+  const getComponentDirs: BuildContext["getComponentDirs"] = (mode) => [
+    dir,
+    // also copy workspace dependencies in monorepo if packages are shared and they create build artifacts
+    ...(mode === "all"
+      ? packageManagerInfo.currentWorkspaceDependencies ?? []
+      : []),
+  ];
+  const _getBuildContext = (): BuildContextComponent => {
+    if (isStandaloneBuildConfig(build)) {
+      return {
+        dir: dir,
+        getComponentDirs,
+        config: build,
+        buildType: build.type,
+        type: "standalone",
+      };
+    }
+    // must be shared build
+    const referencedBuild = ctx.config.builds?.[build.from];
+    if (!referencedBuild) {
+      throw new Error("build.from not found in config");
+    }
+    return {
+      dir: dir,
+      getComponentDirs,
+      config: build,
+      workspaceBuildConfig: referencedBuild,
+      workspaceName: build.from,
+      buildType: referencedBuild.type,
+      type: "fromWorkspace",
+    };
+  };
+  const buildContext: BuildContextComponent = _getBuildContext();
+  const context: Omit<ComponentContext, "customJobs"> = {
+    type: "component",
+    name: ctx.componentName,
+    componentName: ctx.componentName,
+    env: ctx.env,
+    fullConfig: ctx.config,
+    componentConfig,
+
+    build: buildContext,
+    deploy: deploy
+      ? {
+          config: deploy,
+        }
+      : null,
+
+    environment,
+    packageManagerInfo: packageManagerInfo,
+    pipelineType: ctx.pipelineType,
+    trigger: ctx.trigger,
+  };
+  const resolvedCustomJobs = isFunction(customJobs)
+    ? customJobs(context)
+    : customJobs;
+  return {
+    ...context,
+    customJobs: resolvedCustomJobs,
+  };
+};

package/src/context/createWorkspaceContext.ts

@@ -0,0 +1,56 @@
+import {
+  WORKSPACE_BUILD_TYPES,
+  type Config,
+  type PipelineTrigger,
+  type PipelineType,
+  type WorkspaceContext,
+} from "..";
+import { getPackageManagerInfoBase } from "../pipeline/packageManager";
+import { mergeWithMergingArrays } from "../utils";
+import { uniq } from "lodash";
+
+export async function createWorkspaceContext({
+  env,
+  components,
+  workspaceName,
+  config,
+  pipelineType,
+  trigger,
+}: {
+  env: string;
+  components: WorkspaceContext["components"];
+  workspaceName: string;
+  config: Config;
+  pipelineType: PipelineType;
+  trigger: PipelineTrigger;
+}): Promise<WorkspaceContext> {
+  const workspaceConfigRaw = config.builds?.[workspaceName];
+  if (!workspaceConfigRaw) {
+    throw new Error(`Workspace ${workspaceName} not found in config`);
+  }
+
+  const defaults = WORKSPACE_BUILD_TYPES[workspaceConfigRaw.type].defaults();
+
+  const workspaceConfig = mergeWithMergingArrays(defaults, workspaceConfigRaw);
+  return {
+    name: workspaceName,
+    pipelineType,
+    trigger,
+    type: "workspace",
+
+    workspaceConfig,
+
+    env,
+    components,
+    fullConfig: config,
+    packageManagerInfo: await getPackageManagerInfoBase(),
+    build: {
+      type: "workspace",
+      dir: workspaceConfig.dir ?? ".",
+      getComponentDirs: (mode) =>
+        uniq(components.flatMap((c) => c.build.getComponentDirs(mode))),
+      buildType: workspaceConfig.type,
+      config: workspaceConfig,
+    },
+  };
+}

package/src/context/getBuildInfoVariables.ts

@@ -0,0 +1,36 @@
+import { BashExpression } from "../bash/BashExpression";
+import type { BashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
+import { getBashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
+import type { EnvironmentContext } from "../types/environmentContext";
+
+const BUILD_VARIABLES = {
+  BUILD_INFO_BUILD_ID: {
+    default: new BashExpression(
+      `$(git describe --tags 2>/dev/null || git rev-parse HEAD)`,
+    ),
+  },
+  BUILD_INFO_BUILD_TIME: {
+    default: "unknown-build-time",
+    gitlab: new BashExpression(`$CI_JOB_STARTED_AT`),
+  },
+  BUILD_INFO_CURRENT_VERSION: {
+    default: new BashExpression(
+      // because we do shallow fetch, we need to ask the origin
+      `$(tag=$(git ls-remote origin "refs/tags/v*[0-9]" 2>/dev/null | cut -f 2- | sort -V | tail -1 | sed 's/refs\\/tags\\/v//'); [ -z "$tag" ] && echo "0.0.0" || echo "$tag")`,
+    ),
+  },
+} satisfies {
+  [key: string]: BashExpressionPerPipelineType;
+};
+
+export const ALL_BUILD_VARIABLES = Object.keys(BUILD_VARIABLES);
+export const getBuildInfoVariables = (ctx: EnvironmentContext) => {
+  const { pipelineType } = ctx;
+
+  return Object.fromEntries(
+    Object.entries(BUILD_VARIABLES).map(([key, value]) => [
+      key,
+      getBashExpressionPerPipelineType(value, pipelineType),
+    ]),
+  );
+};

package/src/context/getEnvConfig.ts

@@ -0,0 +1,21 @@
+import type { Config, EnvConfigWithComponent } from "../types/config";
+
+import { mergeWithMergingArrays } from "../utils";
+
+export const getEnvConfig = (
+  config: Config,
+  componentName: string,
+  env: string,
+): EnvConfigWithComponent => {
+  const defaultConfig = config.components[componentName];
+  if (!defaultConfig) {
+    throw new Error("unknown component " + componentName);
+  }
+
+  const envCustomizations = defaultConfig.env?.[env] ?? {};
+  if (envCustomizations === false) {
+    throw new Error("env is disabled: " + env + ":" + componentName);
+  }
+
+  return mergeWithMergingArrays(defaultConfig, envCustomizations);
+};

package/src/context/getEnvType.ts

@@ -0,0 +1,17 @@
+import type { EnvType } from "../types";
+import { isKnowEnvType } from "../types";
+
+export const getEnvType = (
+  env: string,
+  envConfig: {
+    type?: EnvType;
+  },
+): EnvType => {
+  if (envConfig.type) return envConfig.type;
+
+  if (isKnowEnvType(env)) {
+    return env;
+  }
+
+  throw new Error("unknown env type: " + env);
+};

package/src/context/getEnvironment.ts

@@ -0,0 +1,26 @@
+import type { CreateComponentContextContext } from "..";
+
+import type { Environment } from "../types/context";
+import { getEnvironmentContext } from "./getEnvironmentContext";
+import { getEnvironmentVariables } from "./getEnvironmentVariables";
+
+export const getEnvironment = async (
+  ctx: CreateComponentContextContext,
+): Promise<Environment> => {
+  const { env } = ctx;
+  const variables = await getEnvironmentVariables(ctx);
+
+  const envContext = getEnvironmentContext(ctx);
+
+  const envType = envContext.envType;
+
+  return {
+    envType,
+    fullName: envContext.fullName,
+    slugPrefix: envContext.environmentSlugPrefix,
+    reviewSlug: envContext.reviewSlug,
+    slug: envContext.environmentSlug,
+    shortName: env,
+    ...variables,
+  };
+};

package/src/context/getEnvironmentContext.ts

@@ -0,0 +1,53 @@
+import type { BuildConfig, CreateComponentContextContext } from "..";
+import type { StringOrBashExpression } from "../bash/BashExpression";
+import { joinBashExpressions } from "../bash/BashExpression";
+
+import type { EnvironmentContext } from "../types/environmentContext";
+import { getEnvConfig } from "./getEnvConfig";
+import { getEnvType } from "./getEnvType";
+import { getReviewSlug } from "./getReviewSlug";
+
+const getEnvironmentSlugPrefix = (
+  env: string,
+  reviewSlug: StringOrBashExpression | null,
+): StringOrBashExpression => {
+  if (reviewSlug) {
+    return joinBashExpressions([env, reviewSlug], "-");
+  }
+  return env;
+};
+
+export const getEnvironmentContext = ({
+  env,
+  componentName,
+  config,
+  pipelineType,
+}: CreateComponentContextContext): EnvironmentContext => {
+  const envConfigRaw = getEnvConfig(config, componentName, env);
+  const envType = getEnvType(env, envConfigRaw);
+  const reviewSlug = getReviewSlug(envConfigRaw, env, pipelineType);
+
+  const environmentSlugPrefix = getEnvironmentSlugPrefix(env, reviewSlug);
+
+  const environmentSlug = environmentSlugPrefix.concat(`-${componentName}`);
+
+  const fullName = joinBashExpressions(
+    [config.customerName, config.appName, environmentSlug],
+    "-",
+  );
+
+  return {
+    envConfigRaw,
+    deployConfigRaw: envConfigRaw.deploy,
+    buildConfigRaw: envConfigRaw.build,
+    environmentSlugPrefix,
+    environmentSlug,
+    reviewSlug,
+    pipelineType,
+    fullName,
+    envType,
+    componentName,
+    env,
+    fullConfig: config,
+  };
+};