@catladder/pipeline 1.165.0 → 1.166.0

package/src/build/node/meteor.ts

@@ -1,25 +1,24 @@
-import { join } from "path";
 import { getRunnerImage } from "../../runner";
-import type { GitlabJobCache } from "../../types";
 import type { ComponentContext } from "../../types/context";
 
 import type { CatladderJob } from "../../types/jobs";
 
 import { createComponentBuildJobs } from "../base";
 import { getDockerBuildScriptWithBuiltInDockerFile } from "../docker";
+import type { CacheConfig } from "../types";
 import { isOfBuildType } from "../types";
 import { getNodeCache } from "./cache";
 import { getYarnInstall } from "./yarn";
 
-const getMeteorCache = (context: ComponentContext): GitlabJobCache[] => [
+const getMeteorCache = (context: ComponentContext): CacheConfig[] => [
   {
-    key: context.name + "meteor-build-cache",
+    key: "meteor-build-cache",
     policy: "pull-push",
     paths: [
-      join(context.build.dir, ".meteor/local/resolver-result-cache.json"),
-      join(context.build.dir, ".meteor/local/plugin-cache"),
-      join(context.build.dir, ".meteor/local/isopacks"),
-      join(context.build.dir, ".meteor/local/bundler-cache/scanner"),
+      ".meteor/local/resolver-result-cache.json",
+      ".meteor/local/plugin-cache",
+      ".meteor/local/isopacks",
+      ".meteor/local/bundler-cache/scanner",
     ],
   },
 ];

package/src/build/node/testJob.ts

@@ -5,11 +5,12 @@ import {
   type ComponentContext,
 } from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
-import { ensureArray, notNil } from "../../utils";
+import { ensureArrayOrNull, notNil } from "../../utils";
 import { createArtifactsConfig } from "../base/createArtifactsConfig";
 import { getNodeCache } from "./cache";
 import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
 import { ensureNodeVersion, getYarnInstall } from "./yarn";
+import { createJobCacheFromCacheConfigs } from "../cache/createJobCache";
 
 export const createNodeTestJobs = (
   context: ComponentContext | WorkspaceContext,
@@ -55,7 +56,7 @@ export const createNodeTestJobs = (
           cache: undefined, // audit does not need yarn install and no cache
           script: [
             `cd ${context.build.dir}`,
-            ...(ensureArray(buildConfig.audit?.command) ?? [
+            ...(ensureArrayOrNull(buildConfig.audit?.command) ?? [
               context.packageManagerInfo.isClassic
                 ? "yarn audit"
                 : "yarn npm audit --environment production", // yarn 2
@@ -76,12 +77,12 @@ export const createNodeTestJobs = (
           name: "👮 lint",
           ...base,
           image: buildConfig.lint?.jobImage ?? defaultImage,
-          cache: getNodeCache(context),
+          cache: createJobCacheFromCacheConfigs(context, getNodeCache(context)),
           script: [
             ...ensureNodeVersion(context),
             `cd ${context.build.dir}`,
             ...yarnInstall,
-            ...(ensureArray(buildConfig.lint?.command) ?? ["yarn lint"]),
+            ...(ensureArrayOrNull(buildConfig.lint?.command) ?? ["yarn lint"]),
           ],
           ...createArtifactsConfig(
             context.build.dir,
@@ -98,12 +99,12 @@ export const createNodeTestJobs = (
           ...base,
           image:
             buildConfig.test?.jobImage ?? getRunnerImage("jobs-testing-chrome"),
-          cache: getNodeCache(context),
+          cache: createJobCacheFromCacheConfigs(context, getNodeCache(context)),
           script: [
             ...ensureNodeVersion(context),
             `cd ${context.build.dir}`,
             ...yarnInstall,
-            ...(ensureArray(buildConfig.test?.command) ?? ["yarn test"]),
+            ...(ensureArrayOrNull(buildConfig.test?.command) ?? ["yarn test"]),
           ],
           ...createArtifactsConfig(
             context.build.dir,

package/src/build/node/yarn.ts

@@ -45,7 +45,7 @@ export const getYarnInstall = (
       ? collapseableSection(
           "postinstall",
           "Custom post install",
-        )(ensureArray(postInstall) ?? [])
+        )(ensureArray(postInstall))
       : []),
   ];
 };

package/src/build/rails/test.ts

@@ -3,7 +3,7 @@ import {
   type ComponentContext,
 } from "../..";
 import type { CatladderJob } from "../../types/jobs";
-import { ensureArray, notNil } from "../../utils";
+import { ensureArrayOrNull, notNil } from "../../utils";
 
 export const createRailsTestJobs = (
   context: ComponentContext,
@@ -54,7 +54,7 @@ export const createRailsTestJobs = (
             buildConfig.audit?.jobImage ?? buildConfig.jobImage ?? defaultImage,
           script: [
             `cd ${context.build.dir}`,
-            ...(ensureArray(buildConfig.audit?.command) ?? [
+            ...(ensureArrayOrNull(buildConfig.audit?.command) ?? [
               "gem install bundler-audit",
               "bundle audit check",
             ]),
@@ -75,7 +75,7 @@ export const createRailsTestJobs = (
           script: [
             `cd ${context.build.dir}`,
             ...bundlerInstall,
-            ...(ensureArray(buildConfig.lint?.command) ?? [
+            ...(ensureArrayOrNull(buildConfig.lint?.command) ?? [
               "bundle exec rubocop",
             ]),
           ],
@@ -93,7 +93,7 @@ export const createRailsTestJobs = (
           script: [
             `cd ${context.build.dir}`,
             ...bundlerInstall,
-            ...(ensureArray(buildConfig.test?.command) ?? [
+            ...(ensureArrayOrNull(buildConfig.test?.command) ?? [
               "bundle exec rspec",
             ]),
           ],

package/src/build/sbom.ts

@@ -1,7 +1,7 @@
 import { isStandaloneBuildConfig } from ".";
 import type { ComponentContext } from "../types/context";
 import type { CatladderJob } from "../types/jobs";
-import { ensureArray } from "../utils";
+import { ensureArrayOrNull } from "../utils";
 
 export const SBOM_BUILD_JOB_NAME = "🧾 sbom";
 export const SBOM_FILE = "__sbom.json";
@@ -29,7 +29,7 @@ export const createSbomBuildJob = (context: ComponentContext): CatladderJob => {
     isStandaloneBuildConfig(buildConfig) &&
     buildConfig.type === "custom" &&
     buildConfig.sbom !== false
-      ? ensureArray(buildConfig.sbom?.command) ?? defaultScript
+      ? ensureArrayOrNull(buildConfig.sbom?.command) ?? defaultScript
       : defaultScript;
 
   return {

package/src/build/types.ts

@@ -1,7 +1,42 @@
 import type { Artifacts, EnvVars, GitlabJobImage } from "../types";
 import type { Services } from "../types/gitlab-ci-yml";
 
-import type { CatladderJob } from "../types/jobs";
+import type { CatladderJob, CatladderJobCache } from "../types/jobs";
+
+export type CacheConfigAdvanced = Omit<CatladderJobCache, "paths" | "key"> & {
+  /**
+   * if you set a key, the cache is scoped to the key. Be aware that there is a limit in gitlab of 4 caches.
+   *
+   * see https://gitlab.com/gitlab-org/gitlab/-/issues/421962
+   */
+  key: Required<CatladderJobCache>["key"];
+
+  /**
+   * whether the cache is scoped to the context (component or workspace) or the build directory.
+   *
+   * defaults to context
+   */
+  scope?: "context" | "buildDir" | "global";
+} & CacheConfigSimple;
+
+export type CacheConfigSimple = {
+  /**
+   * paths to cache. All Paths are relative to the context dir.
+   */
+  paths: string[];
+
+  /**
+   * whether paths are relative to the base dir. defaults to relative.
+   */
+  pathMode?: "relative" | "absolute";
+
+  /**
+   * custom base dir
+   */
+  baseDir?: string;
+};
+
+export type CacheConfig = CacheConfigSimple | CacheConfigAdvanced;
 
 export type BuildConfigArtifactsReports = {
   /**
@@ -13,6 +48,13 @@ export type BuildConfigArtifactsReports = {
   junit?: string[];
 };
 
+export type WithCacheConfig = {
+  /*
+  cache config. All Paths are relative to the context dir.
+  */
+  cache?: CacheConfig | CacheConfig[];
+};
+
 export type TestJobCustom = {
   command?: string | string[];
   jobImage?: GitlabJobImage;
@@ -87,6 +129,8 @@ export type BuildConfigBase = {
 
   /**
    * customize cache for the job
+   *
+   * @deprecated use cache
    */
   jobCache?: CatladderJob["cache"];
 
@@ -98,7 +142,7 @@ export type BuildConfigBase = {
    * custom image to use
    */
   jobImage?: GitlabJobImage;
-};
+} & WithCacheConfig;
 
 export type BuildConfigNodeBase = BuildConfigBase & {
   /**
@@ -290,7 +334,7 @@ export type BuildConfigFromWorkspace = {
    * by default "dist" and ".next" are allways included
    */
   artifactsPaths?: string[];
-};
+} & WithCacheConfig;
 export type BuildConfigStandalone =
   | BuildConfigNode
   | BuildConfigNodeStatic
@@ -351,7 +395,13 @@ export type WorkspaceBuildConfigBase = {
    * use to display information in merge requests, pipeline views and security dashboards.
    */
   artifactsReports?: BuildConfigArtifactsReports;
-};
+
+  jobImage?: GitlabJobImage;
+  /**
+   * tags for the underlying job runner (e.g gitlab)
+   */
+  jobTags?: string[];
+} & WithCacheConfig;
 
 export type WorkspaceBuildConfigNode = {
   type: "node";

package/src/types/gitlab-types.ts

@@ -1,7 +1,7 @@
 import type {
   Artifacts as GitlabCiArtifacts,
   JobTemplate,
-  Cache,
+  Cache as GitlabJobCache,
   Rules,
 } from "./gitlab-ci-yml";
 
@@ -9,7 +9,7 @@ export { Retry, Image as GitlabJobImage } from "./gitlab-ci-yml";
 
 // Reports won't show up on MRs until https://gitlab.com/groups/gitlab-org/-/epics/8205
 export type Artifacts = GitlabCiArtifacts;
-export type GitlabJobCache = Cache;
+export type { GitlabJobCache };
 export type GitlabRule = Exclude<
   Exclude<Rules, null>[number],
   string | string[] // we only use the object version

package/src/types/index.ts

@@ -1,4 +1,5 @@
 export * from "./config";
 export * from "./gitlab-types";
 export * from "./context";
+export * from "./jobDefinition";
 export * from "./pipeline";

package/src/types/jobDefinition.ts

@@ -0,0 +1,9 @@
+import type { CacheConfig } from "../build";
+import { CacheConfigAdvanced, CacheConfigSimple } from "../build";
+import type { CatladderJob } from "./jobs";
+
+export type JobDefintion = Partial<
+  Omit<CatladderJob, "artifacts" | "cache">
+> & {
+  cache?: CacheConfig[];
+};

package/src/types/jobs.ts

@@ -1,17 +1,14 @@
 import type { UnspecifiedEnvVars } from "..";
-import type {
-  BashExpression,
-  StringOrBashExpression,
-} from "../bash/BashExpression";
 import type {
   Artifacts,
+  GitlabEnvironment,
   GitlabJobCache,
-  GitlabRule,
-  GitlabJobImage,
   GitlabJobDef,
-  GitlabEnvironment,
+  GitlabJobImage,
+  GitlabRule,
 } from "./gitlab-types";
 
+export type CatladderJobCache = GitlabJobCache;
 export const BASE_STAGES = [
   "setup",
   "test",
@@ -84,7 +81,7 @@ export type CatladderJob<S = BaseStage> = {
   /**
    * cache config, we use here the same shape as gitlab itself
    */
-  cache?: GitlabJobCache | GitlabJobCache[];
+  cache?: CatladderJobCache | CatladderJobCache[];
 
   /**
    * job artifacts, we also use gitlab shape here

package/src/utils/index.ts

@@ -6,9 +6,12 @@ export function notNil<TValue>(
   return value !== null && value !== undefined;
 }
 
-export const ensureArray = <T>(s: undefined | T | T[]): T[] | null =>
+export const ensureArrayOrNull = <T>(s: undefined | T | T[]): T[] | null =>
   isNil(s) ? null : Array.isArray(s) ? s : [s];
 
+export const ensureArray = <T>(s: undefined | T | T[]): T[] =>
+  ensureArrayOrNull(s) ?? [];
+
 // see https://github.com/lodash/lodash/issues/5384
 export const mergeWithMergingArrays = <A, B>(obj1: A, obj2: B): A & B => {
   return mergeWith({}, obj1, obj2, (prop1, prop2) =>