@catladder/pipeline 1.154.2 → 1.155.0

package/examples/__snapshots__/cloud-run-with-sql.ts.snap

@@ -96,6 +96,13 @@ exports[`matches snapshot 1`] = `
             ],
             "policy": "pull-push",
           },
+          {
+            "key": "myWorkspace-turbo",
+            "paths": [
+              ".turbo",
+            ],
+            "policy": "pull-push",
+          },
         ],
         "image": "path/to/docker/jobs-default:the-version",
         "interruptible": true,
@@ -115,9 +122,6 @@ exports[`matches snapshot 1`] = `
 ENV_SHORT=dev
 APP_DIR=api
 ENV_TYPE=dev
-BUILD_INFO_BUILD_ID=$(git describe --tags 2>/dev/null || git rev-parse HEAD)
-BUILD_INFO_BUILD_TIME=$CI_JOB_STARTED_AT
-BUILD_INFO_CURRENT_VERSION=$(tag=$(git ls-remote origin "refs/tags/v*[0-9]" 2>/dev/null | cut -f 2- | sort -V | tail -1 | sed 's/refs\\/tags\\/v//'); [ -z "$tag" ] && echo "0.0.0" || echo "$tag")
 HOST=$(printf %s "pan-test-app-dev-api-$CL_dev_api_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
 ROOT_URL=https://$(printf %s "pan-test-app-dev-api-$CL_dev_api_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
 HOST_INTERNAL=$(printf %s "pan-test-app-dev-api-$CL_dev_api_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
@@ -143,9 +147,6 @@ EOF",
 ENV_SHORT=dev
 APP_DIR=www
 ENV_TYPE=dev
-BUILD_INFO_BUILD_ID=$(git describe --tags 2>/dev/null || git rev-parse HEAD)
-BUILD_INFO_BUILD_TIME=$CI_JOB_STARTED_AT
-BUILD_INFO_CURRENT_VERSION=$(tag=$(git ls-remote origin "refs/tags/v*[0-9]" 2>/dev/null | cut -f 2- | sort -V | tail -1 | sed 's/refs\\/tags\\/v//'); [ -z "$tag" ] && echo "0.0.0" || echo "$tag")
 HOST=$(printf %s "pan-test-app-dev-www-$CL_dev_www_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
 ROOT_URL=https://$(printf %s "pan-test-app-dev-www-$CL_dev_www_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
 HOST_INTERNAL=$(printf %s "pan-test-app-dev-www-$CL_dev_www_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
@@ -1038,6 +1039,13 @@ EOF
             ],
             "policy": "pull-push",
           },
+          {
+            "key": "myWorkspace-turbo",
+            "paths": [
+              ".turbo",
+            ],
+            "policy": "pull-push",
+          },
         ],
         "image": "path/to/docker/jobs-default:the-version",
         "interruptible": true,
@@ -1057,9 +1065,6 @@ EOF
 ENV_SHORT=review
 APP_DIR=api
 ENV_TYPE=review
-BUILD_INFO_BUILD_ID=$(git describe --tags 2>/dev/null || git rev-parse HEAD)
-BUILD_INFO_BUILD_TIME=$CI_JOB_STARTED_AT
-BUILD_INFO_CURRENT_VERSION=$(tag=$(git ls-remote origin "refs/tags/v*[0-9]" 2>/dev/null | cut -f 2- | sort -V | tail -1 | sed 's/refs\\/tags\\/v//'); [ -z "$tag" ] && echo "0.0.0" || echo "$tag")
 HOST=$(printf %s "pan-test-app-review-$([ -n "$CI_MERGE_REQUEST_IID" ] && echo "mr$CI_MERGE_REQUEST_IID" || { [ -n "$CI_COMMIT_REF_SLUG" ] && echo "$CI_COMMIT_REF_SLUG" || echo "unknown"; })-api-$CL_review_api_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
 ROOT_URL=https://$(printf %s "pan-test-app-review-$([ -n "$CI_MERGE_REQUEST_IID" ] && echo "mr$CI_MERGE_REQUEST_IID" || { [ -n "$CI_COMMIT_REF_SLUG" ] && echo "$CI_COMMIT_REF_SLUG" || echo "unknown"; })-api-$CL_review_api_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
 HOST_INTERNAL=$(printf %s "pan-test-app-review-$([ -n "$CI_MERGE_REQUEST_IID" ] && echo "mr$CI_MERGE_REQUEST_IID" || { [ -n "$CI_COMMIT_REF_SLUG" ] && echo "$CI_COMMIT_REF_SLUG" || echo "unknown"; })-api-$CL_review_api_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
@@ -1085,9 +1090,6 @@ EOF",
 ENV_SHORT=review
 APP_DIR=www
 ENV_TYPE=review
-BUILD_INFO_BUILD_ID=$(git describe --tags 2>/dev/null || git rev-parse HEAD)
-BUILD_INFO_BUILD_TIME=$CI_JOB_STARTED_AT
-BUILD_INFO_CURRENT_VERSION=$(tag=$(git ls-remote origin "refs/tags/v*[0-9]" 2>/dev/null | cut -f 2- | sort -V | tail -1 | sed 's/refs\\/tags\\/v//'); [ -z "$tag" ] && echo "0.0.0" || echo "$tag")
 HOST=$(printf %s "pan-test-app-review-$([ -n "$CI_MERGE_REQUEST_IID" ] && echo "mr$CI_MERGE_REQUEST_IID" || { [ -n "$CI_COMMIT_REF_SLUG" ] && echo "$CI_COMMIT_REF_SLUG" || echo "unknown"; })-www-$CL_review_www_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
 ROOT_URL=https://$(printf %s "pan-test-app-review-$([ -n "$CI_MERGE_REQUEST_IID" ] && echo "mr$CI_MERGE_REQUEST_IID" || { [ -n "$CI_COMMIT_REF_SLUG" ] && echo "$CI_COMMIT_REF_SLUG" || echo "unknown"; })-www-$CL_review_www_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
 HOST_INTERNAL=$(printf %s "pan-test-app-review-$([ -n "$CI_MERGE_REQUEST_IID" ] && echo "mr$CI_MERGE_REQUEST_IID" || { [ -n "$CI_COMMIT_REF_SLUG" ] && echo "$CI_COMMIT_REF_SLUG" || echo "unknown"; })-www-$CL_review_www_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
@@ -1949,6 +1951,13 @@ EOF
             ],
             "policy": "pull-push",
           },
+          {
+            "key": "myWorkspace-turbo",
+            "paths": [
+              ".turbo",
+            ],
+            "policy": "pull-push",
+          },
         ],
         "image": "path/to/docker/jobs-default:the-version",
         "interruptible": true,
@@ -1968,9 +1977,6 @@ EOF
 ENV_SHORT=prod
 APP_DIR=api
 ENV_TYPE=prod
-BUILD_INFO_BUILD_ID=$(git describe --tags 2>/dev/null || git rev-parse HEAD)
-BUILD_INFO_BUILD_TIME=$CI_JOB_STARTED_AT
-BUILD_INFO_CURRENT_VERSION=$(tag=$(git ls-remote origin "refs/tags/v*[0-9]" 2>/dev/null | cut -f 2- | sort -V | tail -1 | sed 's/refs\\/tags\\/v//'); [ -z "$tag" ] && echo "0.0.0" || echo "$tag")
 HOST=$(printf %s "pan-test-app-prod-api-$CL_prod_api_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
 ROOT_URL=https://$(printf %s "pan-test-app-prod-api-$CL_prod_api_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
 HOST_INTERNAL=$(printf %s "pan-test-app-prod-api-$CL_prod_api_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
@@ -1996,9 +2002,6 @@ EOF",
 ENV_SHORT=prod
 APP_DIR=www
 ENV_TYPE=prod
-BUILD_INFO_BUILD_ID=$(git describe --tags 2>/dev/null || git rev-parse HEAD)
-BUILD_INFO_BUILD_TIME=$CI_JOB_STARTED_AT
-BUILD_INFO_CURRENT_VERSION=$(tag=$(git ls-remote origin "refs/tags/v*[0-9]" 2>/dev/null | cut -f 2- | sort -V | tail -1 | sed 's/refs\\/tags\\/v//'); [ -z "$tag" ] && echo "0.0.0" || echo "$tag")
 HOST=$(printf %s "pan-test-app-prod-www-$CL_prod_www_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
 ROOT_URL=https://$(printf %s "pan-test-app-prod-www-$CL_prod_www_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
 HOST_INTERNAL=$(printf %s "pan-test-app-prod-www-$CL_prod_www_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
@@ -2072,6 +2075,13 @@ EOF",
             ],
             "policy": "pull-push",
           },
+          {
+            "key": "myWorkspace-turbo",
+            "paths": [
+              ".turbo",
+            ],
+            "policy": "pull-push",
+          },
         ],
         "image": "path/to/docker/jobs-default:the-version",
         "interruptible": true,
@@ -2091,9 +2101,6 @@ EOF",
 ENV_SHORT=stage
 APP_DIR=api
 ENV_TYPE=stage
-BUILD_INFO_BUILD_ID=$(git describe --tags 2>/dev/null || git rev-parse HEAD)
-BUILD_INFO_BUILD_TIME=$CI_JOB_STARTED_AT
-BUILD_INFO_CURRENT_VERSION=$(tag=$(git ls-remote origin "refs/tags/v*[0-9]" 2>/dev/null | cut -f 2- | sort -V | tail -1 | sed 's/refs\\/tags\\/v//'); [ -z "$tag" ] && echo "0.0.0" || echo "$tag")
 HOST=$(printf %s "pan-test-app-stage-api-$CL_stage_api_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
 ROOT_URL=https://$(printf %s "pan-test-app-stage-api-$CL_stage_api_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
 HOST_INTERNAL=$(printf %s "pan-test-app-stage-api-$CL_stage_api_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
@@ -2119,9 +2126,6 @@ EOF",
 ENV_SHORT=stage
 APP_DIR=www
 ENV_TYPE=stage
-BUILD_INFO_BUILD_ID=$(git describe --tags 2>/dev/null || git rev-parse HEAD)
-BUILD_INFO_BUILD_TIME=$CI_JOB_STARTED_AT
-BUILD_INFO_CURRENT_VERSION=$(tag=$(git ls-remote origin "refs/tags/v*[0-9]" 2>/dev/null | cut -f 2- | sort -V | tail -1 | sed 's/refs\\/tags\\/v//'); [ -z "$tag" ] && echo "0.0.0" || echo "$tag")
 HOST=$(printf %s "pan-test-app-stage-www-$CL_stage_www_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
 ROOT_URL=https://$(printf %s "pan-test-app-stage-www-$CL_stage_www_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')
 HOST_INTERNAL=$(printf %s "pan-test-app-stage-www-$CL_stage_www_GCLOUD_RUN_canonicalHostSuffix" | awk '{print tolower($0)}')

package/examples/__snapshots__/native-app.ts.snap

@@ -647,9 +647,6 @@ EOF
 ENV_SHORT=dev
 APP_DIR=app
 ENV_TYPE=dev
-BUILD_INFO_BUILD_ID=$(git describe --tags 2>/dev/null || git rev-parse HEAD)
-BUILD_INFO_BUILD_TIME=$CI_JOB_STARTED_AT
-BUILD_INFO_CURRENT_VERSION=$(tag=$(git ls-remote origin "refs/tags/v*[0-9]" 2>/dev/null | cut -f 2- | sort -V | tail -1 | sed 's/refs\\/tags\\/v//'); [ -z "$tag" ] && echo "0.0.0" || echo "$tag")
 HOST=unknown-host.example.com
 ROOT_URL=https://unknown-host.example.com
 HOST_INTERNAL=unknown-host.example.com
@@ -1605,9 +1602,6 @@ EOF
 ENV_SHORT=review
 APP_DIR=app
 ENV_TYPE=review
-BUILD_INFO_BUILD_ID=$(git describe --tags 2>/dev/null || git rev-parse HEAD)
-BUILD_INFO_BUILD_TIME=$CI_JOB_STARTED_AT
-BUILD_INFO_CURRENT_VERSION=$(tag=$(git ls-remote origin "refs/tags/v*[0-9]" 2>/dev/null | cut -f 2- | sort -V | tail -1 | sed 's/refs\\/tags\\/v//'); [ -z "$tag" ] && echo "0.0.0" || echo "$tag")
 HOST=unknown-host.example.com
 ROOT_URL=https://unknown-host.example.com
 HOST_INTERNAL=unknown-host.example.com
@@ -2722,9 +2716,6 @@ EOF
 ENV_SHORT=prod
 APP_DIR=app
 ENV_TYPE=prod
-BUILD_INFO_BUILD_ID=$(git describe --tags 2>/dev/null || git rev-parse HEAD)
-BUILD_INFO_BUILD_TIME=$CI_JOB_STARTED_AT
-BUILD_INFO_CURRENT_VERSION=$(tag=$(git ls-remote origin "refs/tags/v*[0-9]" 2>/dev/null | cut -f 2- | sort -V | tail -1 | sed 's/refs\\/tags\\/v//'); [ -z "$tag" ] && echo "0.0.0" || echo "$tag")
 HOST=unknown-host.example.com
 ROOT_URL=https://unknown-host.example.com
 HOST_INTERNAL=unknown-host.example.com
@@ -2851,9 +2842,6 @@ EOF",
 ENV_SHORT=stage
 APP_DIR=app
 ENV_TYPE=stage
-BUILD_INFO_BUILD_ID=$(git describe --tags 2>/dev/null || git rev-parse HEAD)
-BUILD_INFO_BUILD_TIME=$CI_JOB_STARTED_AT
-BUILD_INFO_CURRENT_VERSION=$(tag=$(git ls-remote origin "refs/tags/v*[0-9]" 2>/dev/null | cut -f 2- | sort -V | tail -1 | sed 's/refs\\/tags\\/v//'); [ -z "$tag" ] && echo "0.0.0" || echo "$tag")
 HOST=unknown-host.example.com
 ROOT_URL=https://unknown-host.example.com
 HOST_INTERNAL=unknown-host.example.com

package/package.json

@@ -52,7 +52,7 @@
     }
   ],
   "license": "MIT",
-  "version": "1.154.2",
+  "version": "1.155.0",
   "scripts": {
     "build:tsc": "yarn tsc",
     "build": "yarn build:compile && yarn build:inline-variables && yarn build:bundle",

package/src/build/base/writeDotEnv.ts

@@ -1,6 +1,7 @@
 import { isNil } from "lodash";
 import type { ComponentContext } from "../../types";
 import { collapseableSection } from "../../utils/gitlab";
+import { ALL_BUILD_VARIABLES } from "../../context/getBuildInfoVariables";
 
 /**
  * writes a .env file in the components folder
@@ -14,6 +15,9 @@ export const writeDotEnv = (context: ComponentContext) => {
   const keyValueString = Object.entries(envVars)
     // filter out null and undefined values
     .filter(([, value]) => !isNil(value))
+    // filter out build variables, since they may interfer with caching like turbo
+    // build variables are rarely used anyway and we may treat them differently in the future
+    .filter(([key]) => !ALL_BUILD_VARIABLES.includes(key))
     .map(
       ([key, value]) => `${key}=${value?.toString().replaceAll("\n", "\\n")}`,
     )

package/src/build/node/buildJob.ts

@@ -15,7 +15,12 @@ import type { DockerBuildJobDefinition } from "../docker";
 import { getDockerBuildScriptWithBuiltInDockerFile } from "../docker";
 import type { BuildConfigDocker } from "../types";
 import { isOfBuildType } from "../types";
-import { getNextCache, getNodeCache, getYarnCache } from "./cache";
+import {
+  getNextCache,
+  getNodeCache,
+  getWorkspaceDefaultCaches,
+  getYarnCache,
+} from "./cache";
 import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
 import { getDockerAppCopyAndBuildScript, getYarnInstall } from "./yarn";
 
@@ -57,6 +62,9 @@ export const createNodeBuildJobDefinition = (
       ...(ensureArray(buildConfig.jobCache) ?? []),
       ...getNodeCache(context),
       ...getNextCache(context),
+      ...(context.type === "workspace"
+        ? getWorkspaceDefaultCaches(context)
+        : []),
     ],
     script: [...yarnInstall, ...(ensureArray(buildConfig.buildCommand) ?? [])],
     jobTags: buildConfig.jobTags,

package/src/build/node/cache.ts

@@ -1,7 +1,7 @@
 import { uniq } from "lodash";
 import { join } from "path";
 import slugify from "slugify";
-import type { Context } from "../../types/context";
+import type { Context, WorkspaceContext } from "../../types/context";
 import type { GitlabJobCache } from "../../types/gitlab-types";
 
 export const getYarnCache = (
@@ -80,3 +80,16 @@ export const getNextCache = (context: Context): GitlabJobCache[] => {
     },
   ];
 };
+
+export const getWorkspaceDefaultCaches = (
+  context: WorkspaceContext,
+): GitlabJobCache[] => {
+  return [
+    {
+      // turbo repo
+      key: context.name + "-turbo",
+      policy: "pull-push",
+      paths: [join(context.build.dir, ".turbo")],
+    },
+  ];
+};

package/src/context/getBuildInfoVariables.ts

@@ -1,41 +1,36 @@
 import { BashExpression } from "../bash/BashExpression";
 import type { BashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
 import { getBashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
-import type { BuildConfig } from "../build";
 import type { EnvironmentContext } from "../types/environmentContext";
 
-const BUILD_TIME: BashExpressionPerPipelineType = {
-  default: "unknown-build-time",
-  gitlab: new BashExpression(`$CI_JOB_STARTED_AT`),
-};
-
-const BUILD_ID: BashExpressionPerPipelineType = {
-  default: new BashExpression(
-    `$(git describe --tags 2>/dev/null || git rev-parse HEAD)`,
-  ),
-};
-const CURRENT_VERSION: BashExpressionPerPipelineType = {
-  default: new BashExpression(
-    // because we do shallow fetch, we need to ask the origin
-    `$(tag=$(git ls-remote origin "refs/tags/v*[0-9]" 2>/dev/null | cut -f 2- | sort -V | tail -1 | sed 's/refs\\/tags\\/v//'); [ -z "$tag" ] && echo "0.0.0" || echo "$tag")`,
-  ),
+const BUILD_VARIABLES = {
+  BUILD_INFO_BUILD_ID: {
+    default: new BashExpression(
+      `$(git describe --tags 2>/dev/null || git rev-parse HEAD)`,
+    ),
+  },
+  BUILD_INFO_BUILD_TIME: {
+    default: "unknown-build-time",
+    gitlab: new BashExpression(`$CI_JOB_STARTED_AT`),
+  },
+  BUILD_INFO_CURRENT_VERSION: {
+    default: new BashExpression(
+      // because we do shallow fetch, we need to ask the origin
+      `$(tag=$(git ls-remote origin "refs/tags/v*[0-9]" 2>/dev/null | cut -f 2- | sort -V | tail -1 | sed 's/refs\\/tags\\/v//'); [ -z "$tag" ] && echo "0.0.0" || echo "$tag")`,
+    ),
+  },
+} satisfies {
+  [key: string]: BashExpressionPerPipelineType;
 };
 
+export const ALL_BUILD_VARIABLES = Object.keys(BUILD_VARIABLES);
 export const getBuildInfoVariables = (ctx: EnvironmentContext) => {
   const { pipelineType } = ctx;
 
-  return {
-    BUILD_INFO_BUILD_ID: getBashExpressionPerPipelineType(
-      BUILD_ID,
-      pipelineType,
-    ),
-    BUILD_INFO_BUILD_TIME: getBashExpressionPerPipelineType(
-      BUILD_TIME,
-      pipelineType,
-    ),
-    BUILD_INFO_CURRENT_VERSION: getBashExpressionPerPipelineType(
-      CURRENT_VERSION,
-      pipelineType,
-    ),
-  };
+  return Object.fromEntries(
+    Object.entries(BUILD_VARIABLES).map(([key, value]) => [
+      key,
+      getBashExpressionPerPipelineType(value, pipelineType),
+    ]),
+  );
 };

package/src/pipeline/yarn/yarnUtils.ts

@@ -1,6 +1,7 @@
 import { exec } from "child-process-promise";
 import memoizee from "memoizee";
-import type { PackageManagerInfoComponent, YarnWorkspace } from "../../types";
+import type { YarnWorkspace } from "../../types";
+import { jsonParseOrThrow } from "../../utils/jsonParse";
 
 const execOrFail = async (cmd: string, onFail: string): Promise<string> => {
   try {
@@ -16,17 +17,19 @@ export const getYarnVersion = memoizee(
   },
   { promise: true },
 );
+
 // export for mocking
 export const getWorkspaces = memoizee(
   async (isClassic: boolean): Promise<Array<YarnWorkspace>> => {
     return isClassic
       ? Object.values(
-          JSON.parse(
-            JSON.parse(await execOrFail("yarn workspaces --json info", "{}"))
-              ?.data ?? "{}",
+          jsonParseOrThrow(
+            jsonParseOrThrow(
+              await execOrFail("yarn workspaces --json info", "{}"),
+            )?.data ?? "{}",
           ),
         )
-      : JSON.parse(
+      : jsonParseOrThrow(
           `[${(await execOrFail("yarn workspaces list --json --verbose", ""))
             .trim()
             .split("\n")

package/src/utils/jsonParse.ts

@@ -0,0 +1,7 @@
+export const jsonParseOrThrow = (str: string): any => {
+  try {
+    return JSON.parse(str);
+  } catch (e) {
+    throw new Error(`could not parse json: ${str}`);
+  }
+};