@catladder/pipeline 1.153.0 → 1.154.0

package/src/build/node/yarn.ts

@@ -1,5 +1,5 @@
 import { BashExpression } from "../../bash/BashExpression";
-import type { BuildContext, ComponentContext, Context } from "../../types";
+import type { Context } from "../../types";
 import { ensureArray } from "../../utils";
 import { collapseableSection } from "../../utils/gitlab";
 
@@ -32,7 +32,7 @@ export const getYarnInstall = (
   },
 ) => {
   const postInstall =
-    "postInstall" in context.build.config
+    context.type !== "workspace" && "postInstall" in context.build.config
       ? context.build.config.postInstall
       : null;
   return [

package/src/build/rails/build.ts

@@ -1,6 +1,11 @@
-import type { ComponentContext } from "../..";
+import {
+  componentContextIsStandaloneBuild,
+  type BuildConfigRails,
+  type BuildContextStandalone,
+  type ComponentContext,
+} from "../..";
 import type { CatladderJob } from "../../types/jobs";
-import { createBuildJobs } from "../base";
+import { createComponentBuildJobs } from "../base";
 import {
   getDockerBuildDefaultScript,
   gitlabDockerLogin,
@@ -17,8 +22,13 @@ export const createRailsBuildJobs = (
     throw new Error("build type is not rails");
   }
 
+  // if its not a standalone build, we don't need to run tests
+  if (!componentContextIsStandaloneBuild(context)) {
+    throw new Error("workspace builds are not supported for rails apps");
+  }
+
   if (hasDockerfile(context)) {
-    return createBuildJobs(context, {
+    return createComponentBuildJobs(context, {
       appBuild: undefined,
       dockerBuild: {
         script: getDockerBuildDefaultScript(context),
@@ -32,7 +42,7 @@ export const createRailsBuildJobs = (
   const packEnvArgs = Object.entries(cnbConf?.buildVars ?? {})
     .map(([k, v]) => `--env '${k}${v ? `=${v}` : ""}'`)
     .join(" ");
-  return createBuildJobs(context, {
+  return createComponentBuildJobs(context, {
     appBuild: undefined,
     dockerBuild: {
       variables: {

package/src/build/rails/test.ts

@@ -1,4 +1,7 @@
-import type { ComponentContext } from "../..";
+import {
+  componentContextIsStandaloneBuild,
+  type ComponentContext,
+} from "../..";
 import type { CatladderJob } from "../../types/jobs";
 import { ensureArray, notNil } from "../../utils";
 
@@ -11,6 +14,11 @@ export const createRailsTestJobs = (
     return [];
   }
 
+  // if its not a standalone build, we don't need to run tests
+  if (!componentContextIsStandaloneBuild(context)) {
+    return [];
+  }
+
   const buildConfig = context.build.config;
 
   const base: Omit<CatladderJob, "script" | "name"> = {

package/src/build/sbom.ts

@@ -1,3 +1,4 @@
+import { isStandaloneBuildConfig } from ".";
 import type { ComponentContext } from "../types/context";
 import type { CatladderJob } from "../types/jobs";
 import { ensureArray } from "../utils";
@@ -18,12 +19,16 @@ export const createSbomBuildJob = (context: ComponentContext): CatladderJob => {
   ];
 
   const image =
-    buildConfig.type === "custom" && buildConfig.sbom !== false
+    isStandaloneBuildConfig(buildConfig) &&
+    buildConfig.type === "custom" &&
+    buildConfig.sbom !== false
       ? buildConfig.sbom?.jobImage ?? defaultImage
       : defaultImage;
 
   const script =
-    buildConfig.type === "custom" && buildConfig.sbom !== false
+    isStandaloneBuildConfig(buildConfig) &&
+    buildConfig.type === "custom" &&
+    buildConfig.sbom !== false
       ? ensureArray(buildConfig.sbom?.command) ?? defaultScript
       : defaultScript;
 

package/src/build/types.ts

@@ -129,6 +129,12 @@ export type BuildConfigMeteor = BuildConfigNodeBase & {
   docker?: Omit<BuildConfigDockerBuiltInMeteor, "type"> | BuildConfigDocker;
 };
 
+export type BuildConfigNodeLike =
+  | BuildConfigNode
+  | BuildConfigNodeStatic
+  | BuildConfigStorybook
+  | BuildConfigMeteor;
+
 type BuildConfigDockerWithAdditions = {
   /**
    * Custom Dockerfile lines integrated in the generated Dockerfile before the standard build steps.
@@ -261,7 +267,18 @@ export type BuildConfigStorybook = BuildConfigNodeBase & {
   type: "storybook";
   startCommand?: never;
 };
-export type BuildConfig =
+
+export type BuildConfigFromWorkspace = {
+  from: string;
+  docker?: BuildConfigDocker;
+  startCommand?: string;
+  /**
+   * additional paths for artifacts,
+   * by default "dist" and ".next" are allways included
+   */
+  artifactsPaths?: string[];
+};
+export type BuildConfigStandalone =
   | BuildConfigNode
   | BuildConfigNodeStatic
   | BuildConfigStorybook
@@ -269,16 +286,63 @@ export type BuildConfig =
   | BuildConfigCustom
   | BuildConfigRails;
 
-export type BuildConfigType = BuildConfig["type"];
+export type BuildConfig = BuildConfigFromWorkspace | BuildConfigStandalone;
 
-export type BuildConfigGeneric<T extends BuildConfigType> = Extract<
+export type BuildConfigStandaloneType = BuildConfigStandalone["type"];
+
+export type BuildConfigGeneric<T extends BuildConfigStandaloneType> = Extract<
   BuildConfig,
   { type: T }
 >;
 
-export const isOfBuildType = <T extends Array<BuildConfigType>>(
+export const isStandaloneBuildConfig = (
+  t: BuildConfig | false,
+): t is BuildConfigStandalone => {
+  if (!t) return false;
+  return !("from" in t);
+};
+
+export const isOfBuildType = <T extends Array<BuildConfigStandaloneType>>(
   t: BuildConfig,
   ...types: T
 ): t is Extract<BuildConfig, { type: T[number] }> => {
+  if (!isStandaloneBuildConfig(t)) return false;
   return types.includes(t.type);
 };
+
+export type WorkspaceBuildConfigBase = {
+  dir?: string;
+  /**
+   * customize lint, set false to disable
+   */
+  lint?: false | TestJobCustom;
+
+  /**
+   * customize test, set false to disable
+   */
+  test?: false | TestJobCustom;
+
+  /**
+   * customize audit, set false to disable
+   */
+  audit?: false | TestJobCustom;
+  /**
+   * additional vars only for the runner.
+   * Also if you use services: that require env vars, you need to set them here.
+   *
+   */
+  runnerVariables?: Record<string, string>;
+
+  /**
+   * additional CI/CD artifacts reports,
+   * use to display information in merge requests, pipeline views and security dashboards.
+   */
+  artifactsReports?: BuildConfigArtifactsReports;
+};
+
+export type WorkspaceBuildConfigNode = {
+  type: "node";
+  buildCommand?: string | string[];
+} & WorkspaceBuildConfigBase;
+
+export type WorkspaceBuildConfig = WorkspaceBuildConfigNode;

package/src/context/createAllComponentsContext.ts

@@ -0,0 +1,31 @@
+import { getAllEnvsByTrigger } from "../config/configruedEnvs";
+import type { ComponentContext, PipelineType } from "../types";
+import type { Config, PipelineTrigger } from "../types/config";
+import { createComponentContext } from "./createComponentContext";
+
+export type CreateAllComponentsContextProps = {
+  config: Config;
+  trigger: PipelineTrigger;
+  pipelineType: PipelineType;
+};
+
+export const createAllComponentsContext = async ({
+  config,
+  trigger,
+  pipelineType,
+}: CreateAllComponentsContextProps): Promise<Array<ComponentContext>> => {
+  return await Promise.all(
+    Object.keys(config.components).flatMap((componentName) => {
+      const envs = getAllEnvsByTrigger(config, componentName, trigger);
+      return envs.map(async (env) => {
+        return await createComponentContext({
+          config,
+          componentName,
+          env,
+          trigger,
+          pipelineType,
+        });
+      });
+    }),
+  );
+};

package/src/context/createComponentContext.ts

@@ -1,16 +1,20 @@
 import { isFunction } from "lodash";
-import { BUILD_TYPES } from "../build";
-import type { BuildConfig, BuildConfigType } from "../build/types";
+import { BUILD_TYPES, isStandaloneBuildConfig } from "../build";
+import type { BuildConfig } from "../build/types";
 import { DEPLOY_TYPES } from "../deploy";
 import type { DeployConfig, DeployConfigType } from "../deploy/types";
+import { getPackageManagerInfoForComponent } from "../pipeline/packageManager";
 import type { PipelineType } from "../types";
 import type { Config, PipelineTrigger } from "../types/config";
-import type { ComponentContext } from "../types/context";
+import type {
+  BuildContext,
+  BuildContextComponent,
+  ComponentContext,
+} from "../types/context";
 import type { PartialDeep } from "../types/utils";
 import { mergeWithMergingArrays } from "../utils";
 import { getEnvironment } from "./getEnvironment";
 import { getEnvironmentContext } from "./getEnvironmentContext";
-import { getPackageManagerInfoForComponent } from "../pipeline/packageManager";
 
 export type CreateComponentContextContext = {
   config: Config;
@@ -37,23 +41,30 @@ export const createComponentContext = async (
   const envContext = getEnvironmentContext(ctx);
 
   const componentConfigWithoutDefaults = envContext.envConfigRaw;
+
+  const resolvedBuildType = isStandaloneBuildConfig(
+    componentConfigWithoutDefaults.build,
+  )
+    ? componentConfigWithoutDefaults.build.type
+    : ctx.config.builds?.[componentConfigWithoutDefaults.build.from].type;
+  if (!resolvedBuildType) {
+    throw new Error("build type not found, is the build config correct?");
+  }
   const defaults: {
     build: PartialDeep<BuildConfig>;
     deploy: PartialDeep<DeployConfig>;
   } = componentConfigWithoutDefaults.deploy
     ? {
-        build:
-          BUILD_TYPES[
-            componentConfigWithoutDefaults.build.type as BuildConfigType
-          ].defaults(envContext),
+        build: BUILD_TYPES[resolvedBuildType].defaults(envContext),
         deploy: DEPLOY_TYPES[
           componentConfigWithoutDefaults.deploy.type as DeployConfigType
         ].defaults(envContext as any),
       }
     : {
-        build: {},
+        build: BUILD_TYPES[resolvedBuildType].defaults(envContext),
         deploy: {},
       };
+
   const componentConfig = mergeWithMergingArrays(
     defaults,
     componentConfigWithoutDefaults,
@@ -61,21 +72,54 @@ export const createComponentContext = async (
 
   const environment = await getEnvironment(ctx);
   const { deploy, build, customJobs, dir } = componentConfig;
+  const getComponentDirs: BuildContext["getComponentDirs"] = (mode) => [
+    dir,
+    // also copy workspace dependencies in monorepo if packages are shared and they create build artifacts
+    ...(mode === "all"
+      ? packageManagerInfo.currentWorkspaceDependencies ?? []
+      : []),
+  ];
+  const _getBuildContext = (): BuildContextComponent => {
+    if (isStandaloneBuildConfig(build)) {
+      return {
+        dir: dir,
+        getComponentDirs,
+        config: build,
+        buildType: build.type,
+        type: "standalone",
+      };
+    }
+    // must be shared build
+    const referencedBuild = ctx.config.builds?.[build.from];
+    if (!referencedBuild) {
+      throw new Error("build.from not found in config");
+    }
+    return {
+      dir: dir,
+      getComponentDirs,
+      config: build,
+      workspaceBuildConfig: referencedBuild,
+      workspaceName: build.from,
+      buildType: referencedBuild.type,
+      type: "fromWorkspace",
+    };
+  };
+  const buildContext: BuildContextComponent = _getBuildContext();
   const context: Omit<ComponentContext, "customJobs"> = {
+    type: "component",
+    name: ctx.componentName,
+    componentName: ctx.componentName,
+    env: ctx.env,
     fullConfig: ctx.config,
     componentConfig,
-    env: ctx.env,
 
-    build: {
-      dir: dir,
-      config: build,
-    },
+    build: buildContext,
     deploy: deploy
       ? {
           config: deploy,
         }
       : null,
-    componentName: ctx.componentName,
+
     environment,
     packageManagerInfo: packageManagerInfo,
     pipelineType: ctx.pipelineType,
@@ -89,8 +133,3 @@ export const createComponentContext = async (
     customJobs: resolvedCustomJobs,
   };
 };
-
-/**
- * @deprecated use createComponentContext instead
- */
-export const createContext = createComponentContext;

package/src/context/createWorkspaceContext.ts

@@ -0,0 +1,56 @@
+import {
+  WORKSPACE_BUILD_TYPES,
+  type Config,
+  type PipelineTrigger,
+  type PipelineType,
+  type WorkspaceContext,
+} from "..";
+import { getPackageManagerInfoBase } from "../pipeline/packageManager";
+import { mergeWithMergingArrays } from "../utils";
+import { uniq } from "lodash";
+
+export async function createWorkspaceContext({
+  env,
+  components,
+  workspaceName,
+  config,
+  pipelineType,
+  trigger,
+}: {
+  env: string;
+  components: WorkspaceContext["components"];
+  workspaceName: string;
+  config: Config;
+  pipelineType: PipelineType;
+  trigger: PipelineTrigger;
+}): Promise<WorkspaceContext> {
+  const workspaceConfigRaw = config.builds?.[workspaceName];
+  if (!workspaceConfigRaw) {
+    throw new Error(`Workspace ${workspaceName} not found in config`);
+  }
+
+  const defaults = WORKSPACE_BUILD_TYPES[workspaceConfigRaw.type].defaults();
+
+  const workspaceConfig = mergeWithMergingArrays(defaults, workspaceConfigRaw);
+  return {
+    name: workspaceName,
+    pipelineType,
+    trigger,
+    type: "workspace",
+
+    workspaceConfig,
+
+    env,
+    components,
+    fullConfig: config,
+    packageManagerInfo: await getPackageManagerInfoBase(),
+    build: {
+      type: "workspace",
+      dir: workspaceConfig.dir ?? ".",
+      getComponentDirs: (mode) =>
+        uniq(components.flatMap((c) => c.build.getComponentDirs(mode))),
+      buildType: workspaceConfig.type,
+      config: workspaceConfig,
+    },
+  };
+}

package/src/context/getBuildInfoVariables.ts

@@ -1,6 +1,7 @@
 import { BashExpression } from "../bash/BashExpression";
 import type { BashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
 import { getBashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
+import type { BuildConfig } from "../build";
 import type { EnvironmentContext } from "../types/environmentContext";
 
 const BUILD_TIME: BashExpressionPerPipelineType = {
@@ -20,7 +21,7 @@ const CURRENT_VERSION: BashExpressionPerPipelineType = {
   ),
 };
 
-export const getBuildInfoVariables = (ctx: EnvironmentContext<any, any>) => {
+export const getBuildInfoVariables = (ctx: EnvironmentContext) => {
   const { pipelineType } = ctx;
 
   return {

package/src/context/getEnvironmentContext.ts

@@ -1,4 +1,4 @@
-import type { CreateComponentContextContext } from "..";
+import type { BuildConfig, CreateComponentContextContext } from "..";
 import type { StringOrBashExpression } from "../bash/BashExpression";
 import { joinBashExpressions } from "../bash/BashExpression";
 
@@ -22,7 +22,7 @@ export const getEnvironmentContext = ({
   componentName,
   config,
   pipelineType,
-}: CreateComponentContextContext): EnvironmentContext<any, any> => {
+}: CreateComponentContextContext): EnvironmentContext => {
   const envConfigRaw = getEnvConfig(config, componentName, env);
   const envType = getEnvType(env, envConfigRaw);
   const reviewSlug = getReviewSlug(envConfigRaw, env, pipelineType);

package/src/context/getEnvironmentVariables.ts

@@ -8,7 +8,8 @@ import type { DevLocalEnvConfig } from "../types/config";
 
 import type { CreateComponentContextContext, UnspecifiedEnvVars } from "..";
 import type { StringOrBashExpression } from "../bash/BashExpression";
-import { getBashVariable, joinBashExpressions } from "../bash/BashExpression";
+import { joinBashExpressions } from "../bash/BashExpression";
+import { isStandaloneBuildConfig } from "../build/types";
 import type { EnvironmentContext } from "../types/environmentContext";
 import { getBuildInfoVariables } from "./getBuildInfoVariables";
 import { getEnvironmentContext } from "./getEnvironmentContext";
@@ -29,7 +30,7 @@ export type SecretEnvVar = {
   hidden?: boolean;
 };
 
-const getBasePredefinedVariables = (ctx: EnvironmentContext<any, any>) => {
+const getBasePredefinedVariables = (ctx: EnvironmentContext) => {
   return {
     ENV_SHORT: ctx.env,
     APP_DIR: ctx.envConfigRaw.dir,
@@ -78,7 +79,8 @@ export const getEnvironmentVariables = async (
       ENV_SHORT: "local",
       ROOT_URL: url,
       // Rails before 6.1 (mis)uses the `HOST` environment variable to specify the IP to bind to
-      ...(config.components[componentName].build.type === "rails"
+      ...(isStandaloneBuildConfig(buildConfigRaw) &&
+      buildConfigRaw.type === "rails"
         ? {}
         : { HOST: host }),
       HOST_INTERNAL: host,
@@ -101,7 +103,8 @@ export const getEnvironmentVariables = async (
     predefinedVariables = {
       ...basePredefinedVariables,
       // Rails before 6.1 (mis)uses the `HOST` environment variable to specify the IP to bind to
-      ...(config.components[componentName].build.type === "rails"
+      ...(isStandaloneBuildConfig(buildConfigRaw) &&
+      buildConfigRaw.type === "rails"
         ? {}
         : { HOST: host }),
       ROOT_URL: url,
@@ -169,7 +172,10 @@ export const getEnvironmentVariables = async (
       build: await transformJobOnlyVars(
         env,
         componentName,
-        (buildConfigRaw && buildConfigRaw.jobVars) || null,
+        (buildConfigRaw &&
+          isStandaloneBuildConfig(buildConfigRaw) &&
+          buildConfigRaw.jobVars) ||
+          null,
       ),
       deploy: await transformJobOnlyVars(
         env,
@@ -201,5 +207,4 @@ const addIndexVar = <V extends Record<string, unknown>>(
 export const getSecretVarNameForContext = (
   context: ComponentContext,
   key: string,
-) =>
-  getSecretVarName(context.environment.shortName, context.componentName, key);
+) => getSecretVarName(context.env, context.name, key);

package/src/context/getLabels.ts

@@ -10,11 +10,11 @@ const sanitize = (value?: string) => {
 export const getLabels = (context: ComponentContext) => {
   const labels = {
     "customer-name": sanitize(context.fullConfig.customerName),
-    "component-name": sanitize(context.componentName),
+    "component-name": sanitize(context.name),
     "app-name": sanitize(context.fullConfig.appName),
     "env-type": sanitize(context.environment.envType),
-    "env-name": sanitize(context.environment.shortName),
-    "build-type": sanitize(context.build.config?.type),
+    "env-name": sanitize(context.env),
+    "build-type": sanitize(context.build.buildType),
     ...(context.fullConfig.meta?.labels ?? {}),
   };
   return labels;

package/src/deploy/base/deploy.ts

@@ -3,8 +3,11 @@ import {
   requiresDockerBuild,
 } from "../../build/docker";
 import { SBOM_BUILD_JOB_NAME } from "../../build/sbom";
-import type { ComponentContext } from "../../types/context";
-import type { CatladderJob } from "../../types/jobs";
+import {
+  componentContextHasWorkspaceBuild,
+  type ComponentContext,
+} from "../../types/context";
+import type { BaseStage, CatladderJob } from "../../types/jobs";
 import { sbomDeactivated } from "../sbom";
 import { contextIsStoppable } from "../utils";
 import { STOP_JOB_NAME } from "./stop";
@@ -74,15 +77,38 @@ export const createDeployJob = (
     ],
     // we don't want to deploy when there is a broken test
     needsStages: [
-      {
-        stage: "build",
-        artifacts: hasDocker ? false : true, // we asume that no-docker deployments need build artifacts,
-      },
+      ...(componentContextHasWorkspaceBuild(context)
+        ? hasDocker // docker build is per component,
+          ? [
+              // we don't need artifacts, but have to wait for the component build
+              {
+                stage: "build" as BaseStage,
+                artifacts: false,
+              },
+            ]
+          : [
+              {
+                // pick build artifacts from workspace build
+                stage: "build" as BaseStage,
+                artifacts: true,
+                workspaceName: context.build.workspaceName,
+              },
+            ]
+        : [
+            {
+              stage: "build" as BaseStage,
+              artifacts: hasDocker ? false : true, // we asume that no-docker deployments need build artifacts,
+            },
+          ]),
       {
         stage: "test",
         artifacts: false,
+        // use test from workspace build
+        workspaceName: componentContextHasWorkspaceBuild(context)
+          ? context.build.workspaceName
+          : undefined,
       },
-    ], // workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/220758
+    ],
     when: whenDeploy === "auto" ? "on_success" : "manual",
 
     allow_failure: whenDeploy === "manual" ? true : false,

package/src/deploy/cloudRun/artifactsRegistry.ts

@@ -43,9 +43,8 @@ export const getArtifactsRegistryImageName = (
   const dockerUrl = getArtifactsRegistryDockerUrl(context);
   const gcloudImagePath = [
     dockerUrl,
-    context.environment.shortName,
-    context.componentName,
-
+    context.env,
+    context.name,
     ...(context.environment.reviewSlug && !lecacyReviewImageName
       ? [context.environment.reviewSlug]
       : []),
@@ -58,7 +57,7 @@ export const getArtifactsRegistryBuildCacheImage = (
 ) => {
   const dockerUrl = getArtifactsRegistryDockerUrl(context);
   // does not include env, so that after merge, you might get more cache hits (review-->dev)
-  const gcloudImagePath = [dockerUrl, "caches", context.componentName];
+  const gcloudImagePath = [dockerUrl, "caches", context.name];
   return gcloudImagePath.join("/");
 };
 

package/src/deploy/cloudRun/createJobs/common.ts

@@ -1,4 +1,4 @@
-import type { Context } from "../../../types/context";
+import type { ComponentContext } from "../../../types/context";
 
 import { isOfDeployType } from "../../types";
 import type { DeployConfigCloudRun } from "../../types/googleCloudRun";
@@ -29,7 +29,7 @@ export const makeLabelString = (obj: Record<string, unknown>) =>
     .map(([key, value]) => `${key}=${value}`)
     .join(",");
 
-export const getCloudRunDeployConfig = (context: Context) => {
+export const getCloudRunDeployConfig = (context: ComponentContext) => {
   const deployConfig = context.deploy?.config;
   if (!isOfDeployType(deployConfig, "google-cloudrun")) {
     // should not happen
@@ -38,7 +38,7 @@ export const getCloudRunDeployConfig = (context: Context) => {
   return deployConfig;
 };
 
-export function getCommonCloudRunArgs(context: Context) {
+export function getCommonCloudRunArgs(context: ComponentContext) {
   const deployConfig = getCloudRunDeployConfig(context);
   return {
     project: deployConfig.projectId,
@@ -46,7 +46,7 @@ export function getCommonCloudRunArgs(context: Context) {
   };
 }
 
-export function getCommonDeployArgs(context: Context) {
+export function getCommonDeployArgs(context: ComponentContext) {
   const commonArgs = getCommonCloudRunArgs(context);
   const deployConfig = getCloudRunDeployConfig(context);
   return {