@catladder/pipeline 1.165.0 → 1.166.0

package/examples/cloud-run-with-sql.ts

@@ -3,17 +3,12 @@ import type { Config } from "../src";
 const config: Config = {
   appName: "test-app",
   customerName: "pan",
-  builds: {
-    myWorkspace: {
-      type: "node",
-    },
-  },
   components: {
     api: {
       dotEnv: true,
       dir: "api",
       build: {
-        from: "myWorkspace",
+        type: "node",
       },
       deploy: {
         type: "google-cloudrun",
@@ -44,23 +39,6 @@ const config: Config = {
         },
       },
     },
-    www: {
-      dotEnv: true,
-      dir: "www",
-      build: {
-        from: "myWorkspace",
-      },
-      deploy: {
-        type: "google-cloudrun",
-        projectId: "google-project-id",
-        region: "europe-west6",
-      },
-      vars: {
-        public: {
-          API_URL: "${api:ROOT_URL}/graphql",
-        },
-      },
-    },
   },
 };
 

package/examples/workspace-api-www-custom-cache.test.ts

@@ -0,0 +1,11 @@
+import { createYamlLocalPipeline } from "./__utils__/helpers";
+import config from "./workspace-api-www-custom-cache";
+
+/**
+ * This test is auto-generated.
+ * Modifications will be overwritten on every `yarn test` run!
+ */
+
+it("matches snapshot for workspace-api-www-custom-cache local pipeline YAML", async () => {
+  expect(await createYamlLocalPipeline(config)).toMatchSnapshot();
+});

package/examples/workspace-api-www-custom-cache.ts

@@ -0,0 +1,51 @@
+import type { Config, DeployConfig } from "../src";
+
+const DEPLOY_CONFIG: DeployConfig = {
+  type: "google-cloudrun",
+  projectId: "google-project-id",
+  region: "europe-west6",
+};
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  builds: {
+    myWorkspace: {
+      type: "node",
+      cache: {
+        paths: [".my-cache"],
+      },
+    },
+  },
+  components: {
+    api: {
+      dotEnv: true,
+      dir: "api",
+      build: {
+        from: "myWorkspace",
+        cache: {
+          paths: [".component-custom-cache"],
+        },
+      },
+      deploy: DEPLOY_CONFIG,
+    },
+    www: {
+      dotEnv: true,
+      dir: "www",
+      build: {
+        from: "myWorkspace",
+      },
+      deploy: DEPLOY_CONFIG,
+      vars: {
+        public: {
+          API_URL: "${api:ROOT_URL}/graphql",
+        },
+      },
+    },
+  },
+};
+
+export default config;
+
+export const information = {
+  title: "Workspace API and WWW",
+};

package/examples/workspace-api-www.test.ts

@@ -0,0 +1,11 @@
+import { createYamlLocalPipeline } from "./__utils__/helpers";
+import config from "./workspace-api-www";
+
+/**
+ * This test is auto-generated.
+ * Modifications will be overwritten on every `yarn test` run!
+ */
+
+it("matches snapshot for workspace-api-www local pipeline YAML", async () => {
+  expect(await createYamlLocalPipeline(config)).toMatchSnapshot();
+});

package/examples/workspace-api-www.ts

@@ -0,0 +1,45 @@
+import type { Config, DeployConfig } from "../src";
+
+const DEPLOY_CONFIG: DeployConfig = {
+  type: "google-cloudrun",
+  projectId: "google-project-id",
+  region: "europe-west6",
+};
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  builds: {
+    myWorkspace: {
+      type: "node",
+    },
+  },
+  components: {
+    api: {
+      dotEnv: true,
+      dir: "api",
+      build: {
+        from: "myWorkspace",
+      },
+      deploy: DEPLOY_CONFIG,
+    },
+    www: {
+      dotEnv: true,
+      dir: "www",
+      build: {
+        from: "myWorkspace",
+      },
+      deploy: DEPLOY_CONFIG,
+      vars: {
+        public: {
+          API_URL: "${api:ROOT_URL}/graphql",
+        },
+      },
+    },
+  },
+};
+
+export default config;
+
+export const information = {
+  title: "Workspace API and WWW",
+};

package/package.json

@@ -53,7 +53,7 @@
     }
   ],
   "license": "MIT",
-  "version": "1.165.0",
+  "version": "1.166.0",
   "scripts": {
     "build:tsc": "yarn tsc",
     "build": "yarn build:compile && yarn build:inline-variables && yarn build:bundle",

package/src/build/base/createAppBuildJob.ts

@@ -5,9 +5,11 @@ import type {
   WorkspaceContext,
 } from "../..";
 import { getRunnerImage } from "../..";
+import type { JobDefintion } from "../../types/jobDefinition";
 import type { CatladderJob } from "../../types/jobs";
 import { ensureArray } from "../../utils";
 import { createBuildJobArtifacts } from "../artifacts/createBuildJobArtifact";
+import { createJobCacheFromCacheConfigs } from "../cache/createJobCache";
 import { ensureNodeVersion } from "../node/yarn";
 import {
   APP_BUILD_JOB_NAME,
@@ -19,10 +21,10 @@ import {
   writeDotEnv,
 } from "./writeDotEnv";
 
-export type AppBuildJobDefinition = Partial<Omit<CatladderJob, "artifacts">>;
+export type AppBuildJobDefinition = JobDefintion;
 export const createAppBuildJob = (
   context: ComponentContext<BuildContextStandalone> | WorkspaceContext,
-  { script, variables, runnerVariables, ...def }: AppBuildJobDefinition,
+  { script, variables, runnerVariables, cache, ...def }: AppBuildJobDefinition,
 ): CatladderJob => {
   return merge(
     {
@@ -31,7 +33,7 @@ export const createAppBuildJob = (
       stage: "build",
       image: getRunnerImage("jobs-default"),
       needs: [],
-      cache: [],
+      cache: cache ? createJobCacheFromCacheConfigs(context, cache) : undefined,
       variables: {
         ...(variables ?? {}),
         ...(context.type === "component"
@@ -61,7 +63,7 @@ export const createAppBuildJob = (
         ...(context.type === "component" ? writeBuildInfo(context) : []),
         ...ensureNodeVersion(context), // in pure node repos, we might want to have the nvmrc file in top-level
         `cd ${context.build.dir}`,
-        ...(ensureArray(script) ?? []),
+        ...ensureArray(script),
       ],
       artifacts: createBuildJobArtifacts(context),
     },

package/src/build/base/createBuildJobDefinition.ts

@@ -0,0 +1,38 @@
+import type { BuildConfigStandalone, WorkspaceBuildConfig } from "..";
+import { getRunnerImage } from "../../runner";
+import type { Context } from "../../types";
+import type { JobDefintion } from "../../types/jobDefinition";
+import { ensureArray } from "../../utils";
+import { getAllCacheConfigsFromConfig } from "../cache/getAllCacheConfigsFromConfig";
+import { NODE_RUNNER_BUILD_VARIABLES } from "../node/constants";
+
+export const createBuildJobDefinition = (
+  context: Context,
+  buildConfig: BuildConfigStandalone | WorkspaceBuildConfig,
+  customize: Pick<JobDefintion, "cache"> & {
+    prescript?: string[];
+    postscript?: string[];
+  } = {},
+): JobDefintion | undefined => {
+  if (buildConfig.buildCommand === null) return undefined;
+
+  const defaultImage = getRunnerImage("jobs-default");
+
+  return {
+    image: buildConfig.jobImage ?? defaultImage,
+    runnerVariables: NODE_RUNNER_BUILD_VARIABLES,
+    cache: [
+      ...(customize.cache ?? []),
+      ...getAllCacheConfigsFromConfig(context, buildConfig),
+    ],
+    services:
+      "jobServices" in buildConfig ? buildConfig.jobServices : undefined,
+    script: [
+      ...(customize.prescript ?? []),
+      ...ensureArray(buildConfig.buildCommand),
+      ...(customize.postscript ?? []),
+    ],
+
+    jobTags: buildConfig.jobTags,
+  };
+};

package/src/build/cache/createJobCache.ts

@@ -0,0 +1,85 @@
+import { join } from "path";
+import type {
+  BuildConfigStandalone,
+  CacheConfig,
+  CacheConfigAdvanced,
+  CacheConfigSimple,
+  WorkspaceBuildConfig,
+} from "..";
+import type { Context } from "../../types";
+import type { CatladderJobCache } from "../../types/jobs";
+import { getAllCacheConfigsFromConfig } from "./getAllCacheConfigsFromConfig";
+
+export const createJobCacheFromCacheConfigs = (
+  context: Context,
+  caches: CacheConfig[],
+): CatladderJobCache[] | undefined => {
+  if (caches.length === 0) return undefined;
+
+  const simpleCacheDefs = caches.filter(
+    (c): c is CacheConfigSimple => !("key" in c),
+  );
+  const advancedCacheDefs = caches.filter(
+    (c): c is CacheConfigAdvanced => "key" in c,
+  );
+  // simple caches are merged together into one. This is because e.g. gitlab has a limit of 4 caches, see https://gitlab.com/gitlab-org/gitlab/-/issues/421962
+  const simpleCaches =
+    simpleCacheDefs.length > 0
+      ? [
+          {
+            key: context.name + "-default",
+            policy: "pull-push",
+            paths: simpleCacheDefs.flatMap((c) =>
+              c.paths.map((p) => {
+                const baseDir =
+                  c.pathMode === "absolute"
+                    ? ""
+                    : c.baseDir ?? context.build.dir;
+                return join(baseDir, p);
+              }),
+            ),
+          },
+        ]
+      : [];
+
+  const advancedCaches = advancedCacheDefs.map(
+    ({ key, paths, policy, scope, pathMode, buildDir, ...rest }) => {
+      const baseDir =
+        pathMode === "absolute" ? "" : buildDir ?? context.build.dir;
+      const transformedKey =
+        scope === "global"
+          ? key
+          : typeof key === "string"
+            ? (scope === "buildDir" // really edge case...
+                ? baseDir
+                : context.name) +
+              "-" +
+              key
+            : {
+                ...key,
+                files: key.files?.map((f) =>
+                  pathMode === "absolute" ? f : join(baseDir, f),
+                ),
+              };
+      return {
+        key: transformedKey,
+        policy: policy ?? "pull-push",
+        paths:
+          pathMode === "absolute" ? paths : paths?.map((p) => join(baseDir, p)),
+        ...rest,
+      };
+    },
+  );
+  return [...advancedCaches, ...simpleCaches];
+};
+
+/** shortcut, used in some build types */
+export const createJobCacheFromConfig = (
+  context: Context,
+  buildConfig: BuildConfigStandalone | WorkspaceBuildConfig,
+) => {
+  return createJobCacheFromCacheConfigs(
+    context,
+    getAllCacheConfigsFromConfig(context, buildConfig),
+  );
+};

package/src/build/cache/getAllCacheConfigsFromConfig.ts

@@ -0,0 +1,33 @@
+import type { Context } from "../../types";
+import { ensureArray } from "../../utils/index";
+import type {
+  BuildConfigStandalone,
+  CacheConfig,
+  WorkspaceBuildConfig,
+} from "../types";
+import { transformLegacyJobCache } from "./transformLegacyJobCache";
+
+export const getAllCacheConfigsFromConfig = (
+  context: Context,
+  buildConfig: BuildConfigStandalone | WorkspaceBuildConfig,
+): CacheConfig[] => {
+  return [
+    ...("jobCache" in buildConfig
+      ? transformLegacyJobCache(buildConfig.jobCache)
+      : []),
+    ...ensureArray(buildConfig.cache).map((c) => ({
+      ...c,
+      context,
+    })),
+    ...(context.type === "workspace"
+      ? // also add cache configs of the components of that workspace
+        context.components.flatMap<CacheConfig>(
+          (componentContext) =>
+            ensureArray(componentContext.build.config.cache).map((c) => ({
+              ...c,
+              baseDir: componentContext.build.dir,
+            })) ?? [],
+        )
+      : []),
+  ];
+};

package/src/build/cache/transformLegacyJobCache.ts

@@ -0,0 +1,14 @@
+import type { CacheConfigAdvanced } from "..";
+import type { CatladderJobCache } from "../../types/jobs";
+import { ensureArray } from "../../utils";
+
+export function transformLegacyJobCache(
+  jobCache: CatladderJobCache | CatladderJobCache[] | undefined,
+): CacheConfigAdvanced[] {
+  return ensureArray(jobCache).map((cache) => ({
+    pathMode: "absolute",
+    ...cache,
+    key: cache.key ?? "default",
+    paths: cache.paths ?? [],
+  }));
+}

package/src/build/custom/buildJob.ts

@@ -1,16 +1,10 @@
 import type { ComponentContext } from "../../types/context";
-import { ensureArray } from "../../utils";
 import { getDockerBuildScriptWithBuiltInDockerFile } from "../docker";
 import { isOfBuildType } from "../types";
 
 import type { CatladderJob } from "../../types/jobs";
 import { createComponentBuildJobs } from "../base";
-
-const RUNNER_BUILD_VARIABLES = {
-  KUBERNETES_CPU_REQUEST: "0.45",
-  KUBERNETES_MEMORY_REQUEST: "1Gi",
-  KUBERNETES_MEMORY_LIMIT: "4Gi",
-};
+import { createBuildJobDefinition } from "../base/createBuildJobDefinition";
 
 export const createCustomBuildJobs = (
   context: ComponentContext,
@@ -24,13 +18,7 @@ export const createCustomBuildJobs = (
   return createComponentBuildJobs(context, {
     appBuild:
       buildConfig.buildCommand !== null
-        ? {
-            image: buildConfig.jobImage,
-            runnerVariables: RUNNER_BUILD_VARIABLES,
-            cache: buildConfig.jobCache,
-            services: buildConfig.jobServices,
-            script: [...(ensureArray(buildConfig.buildCommand) ?? [])],
-          }
+        ? createBuildJobDefinition(context, buildConfig)
         : undefined,
     dockerBuild: {
       script: getDockerBuildScriptWithBuiltInDockerFile(context),

package/src/build/custom/testJob.ts

@@ -1,9 +1,10 @@
+import type { Artifacts } from "../../types";
 import type { ComponentContext } from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
 import { ensureArray, notNil } from "../../utils";
-import { isOfBuildType } from "../types";
-import type { Artifacts } from "../../types";
 import { createArtifactsConfig } from "../base/createArtifactsConfig";
+import { createJobCacheFromConfig } from "../cache/createJobCache";
+import { isOfBuildType } from "../types";
 
 const RUNNER_CUSTOM_TEST_VARIABLES = {
   KUBERNETES_CPU_REQUEST: "0.45",
@@ -34,7 +35,7 @@ export const createCustomTestJobs = (
     },
     runnerVariables: RUNNER_CUSTOM_TEST_VARIABLES,
     services: buildConfig.jobServices,
-    cache: buildConfig.jobCache,
+    cache: createJobCacheFromConfig(context, buildConfig),
     stage: "test",
     needs: [],
     envMode: "none",
@@ -45,7 +46,7 @@ export const createCustomTestJobs = (
         ...base,
         image: buildConfig.audit?.jobImage ?? buildConfig.jobImage,
         cache: undefined,
-        script: [...(ensureArray(buildConfig.audit?.command) ?? [])],
+        script: [...ensureArray(buildConfig.audit?.command)],
         allow_failure: true,
         ...createArtifactsConfig(
           context.build.dir,
@@ -61,7 +62,7 @@ export const createCustomTestJobs = (
 
         ...base,
         image: buildConfig.lint?.jobImage ?? buildConfig.jobImage,
-        script: [...(ensureArray(buildConfig.lint?.command) ?? [])],
+        script: [...ensureArray(buildConfig.lint?.command)],
         ...createArtifactsConfig(
           context.build.dir,
           buildConfig.lint?.artifactsReports,
@@ -75,7 +76,7 @@ export const createCustomTestJobs = (
 
         ...base,
         image: buildConfig.test?.jobImage ?? buildConfig.jobImage,
-        script: [...(ensureArray(buildConfig.test?.command) ?? [])],
+        script: [...ensureArray(buildConfig.test?.command)],
         ...createArtifactsConfig(
           context.build.dir,
           buildConfig.test?.artifactsReports,

package/src/build/docker.ts

@@ -10,9 +10,10 @@ import {
 } from "../deploy/cloudRun/artifactsRegistry";
 import { gcloudServiceAccountLoginCommands } from "../deploy/cloudRun/utils/gcloudServiceAccountLoginCommands";
 import { getRunnerImage } from "../runner";
-import type { ComponentContext } from "../types";
+import type { ComponentContext, JobDefintion } from "../types";
 import type { CatladderJob } from "../types/jobs";
 import { collapseableSection } from "../utils/gitlab";
+import { createJobCacheFromCacheConfigs } from "./cache/createJobCache";
 
 const DOCKER_BUILD_RUNNER_REQUESTS = {
   KUBERNETES_CPU_REQUEST: "0.45",
@@ -118,16 +119,17 @@ export const getDockerJobBaseProps = (): Pick<
   };
 };
 
-export type DockerBuildJobDefinition = Partial<CatladderJob>;
+export type DockerBuildJobDefinition = JobDefintion;
 export const createDockerBuildJobBase = (
   context: ComponentContext,
-  { script, ...def }: Partial<CatladderJob>,
+  { script, cache, ...def }: JobDefintion,
 ): CatladderJob => {
   return merge(
     {
       name: DOCKER_BUILD_JOB_NAME,
       envMode: "jobPerEnv",
       stage: "build",
+      cache: cache ? createJobCacheFromCacheConfigs(context, cache) : undefined,
       ...getDockerJobBaseProps(),
       script: script || [],
     },

package/src/build/node/buildJob.ts

@@ -1,4 +1,3 @@
-import { getRunnerImage } from "../../runner";
 import type {
   BuildContextStandalone,
   WorkspaceContext,
@@ -8,20 +7,13 @@ import {
   type ComponentContext,
 } from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
-import { ensureArray } from "../../utils";
 import { createComponentBuildJobs, createWorkspaceBuildJobs } from "../base";
 import type { AppBuildJobDefinition } from "../base/createAppBuildJob";
+import { createBuildJobDefinition } from "../base/createBuildJobDefinition";
 import type { DockerBuildJobDefinition } from "../docker";
 import { getDockerBuildScriptWithBuiltInDockerFile } from "../docker";
 import type { BuildConfigDocker } from "../types";
-import { isOfBuildType } from "../types";
-import {
-  getNextCache,
-  getNodeCache,
-  getWorkspaceDefaultCaches,
-  getYarnCache,
-} from "./cache";
-import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
+import { getNextCache, getNodeCache, getYarnCache } from "./cache";
 import { getDockerAppCopyAndBuildScript, getYarnInstall } from "./yarn";
 
 export const createNodeBuildJobs = (
@@ -46,26 +38,11 @@ export const createNodeBuildJobDefinition = (
 ): AppBuildJobDefinition | undefined => {
   const buildConfig = context.build.config;
 
-  if (!isOfBuildType(buildConfig, "node", "node-static", "storybook")) {
-    throw new Error("deploy config is not node, node-static or storybook");
-  }
-
-  if (buildConfig.buildCommand === null) return undefined;
-
-  const defaultImage = getRunnerImage("jobs-default");
   const yarnInstall = getYarnInstall(context);
-
-  return {
-    image: buildConfig.jobImage ?? defaultImage,
-    runnerVariables: NODE_RUNNER_BUILD_VARIABLES,
-    cache: [
-      ...(ensureArray(buildConfig.jobCache) ?? []),
-      ...getNodeCache(context),
-      ...getNextCache(context),
-    ],
-    script: [...yarnInstall, ...(ensureArray(buildConfig.buildCommand) ?? [])],
-    jobTags: buildConfig.jobTags,
-  };
+  return createBuildJobDefinition(context, buildConfig, {
+    prescript: yarnInstall,
+    cache: [...getNodeCache(context), ...getNextCache(context)],
+  });
 };
 
 export const createNodeDockerJobDefinition = (

package/src/build/node/cache.ts

@@ -1,35 +1,31 @@
 import { uniq } from "lodash";
 import { join } from "path";
 import slugify from "slugify";
+
 import type { Context, WorkspaceContext } from "../../types/context";
-import type { GitlabJobCache } from "../../types/gitlab-types";
+import type { CacheConfig } from "../types";
 
 export const getYarnCache = (
   context: Context,
   policy = "pull-push",
-): GitlabJobCache[] => {
+): CacheConfig[] => {
   const componentIsInWorkspace =
     context.type === "component" &&
     context.packageManagerInfo.componentIsInWorkspace;
   return [
-    componentIsInWorkspace
-      ? {
-          key: "yarn",
-          policy,
-          paths: [".yarn"],
-        }
-      : {
-          key: slugify(context.build.dir) + "-yarn",
-          policy,
-          paths: [join(context.build.dir, ".yarn")],
-        },
+    {
+      scope: componentIsInWorkspace ? "global" : "buildDir",
+      key: "yarn",
+      policy,
+      paths: [".yarn"],
+    },
   ];
 };
 
 export const getNodeModulesCache = (
   context: Context,
   policy = "pull-push",
-): GitlabJobCache[] => {
+): CacheConfig[] => {
   const componentIsInWorkspace =
     context.type === "component" &&
     context.packageManagerInfo.componentIsInWorkspace;
@@ -38,6 +34,9 @@ export const getNodeModulesCache = (
   // It would slow down all pipelines whenever one adds a new dependency as it will need to download all node_modules again.
   return [
     {
+      scope: "global",
+      pathMode: "absolute",
+
       // if component is in a shared workspace, use workspace cache. use individual cache else
       key: componentIsInWorkspace
         ? "node-modules-workspace"
@@ -59,7 +58,7 @@ export const getNodeModulesCache = (
 export const getNodeCache = (
   context: Context,
   policy = "pull-push",
-): GitlabJobCache[] => {
+): CacheConfig[] => {
   return [
     ...getYarnCache(context, policy),
     ...getNodeModulesCache(context, policy),
@@ -67,15 +66,15 @@ export const getNodeCache = (
   ];
 };
 
-export const getNextCache = (context: Context): GitlabJobCache[] => {
-  const key = context.name + "-next-cache";
+export const getNextCache = (context: Context): CacheConfig[] => {
   const paths = context.build
     .getComponentDirs("direct")
     .map((c) => join(c, ".next/cache"));
 
   return [
     {
-      key,
+      pathMode: "absolute",
+      key: "next-cache",
       policy: "pull-push",
       paths,
     },
@@ -84,13 +83,13 @@ export const getNextCache = (context: Context): GitlabJobCache[] => {
 
 export const getWorkspaceDefaultCaches = (
   context: WorkspaceContext,
-): GitlabJobCache[] => {
+): CacheConfig[] => {
   return [
     {
       // turbo repo
-      key: context.name + "-turbo",
+      key: "turbo",
       policy: "pull-push",
-      paths: [join(context.build.dir, ".turbo")],
+      paths: [".turbo"],
     },
   ];
 };