@catladder/pipeline 3.47.1 → 4.0.0

package/src/build/node/meteor.ts

@@ -26,10 +26,11 @@ const getMeteorCache = (context: ComponentContext): CacheConfig[] => [
     ],
   },
 ];
-const getMeteorDockerInstallScripts = (
+const getMeteorDockerInstallScripts = async (
   context: ComponentContextWithBuild,
-): BashExpression => {
-  if (context.packageManagerInfo.isClassic) {
+): Promise<BashExpression> => {
+  const packageManagerInfo = await context.packageManagerInfo;
+  if (packageManagerInfo.isClassic) {
     return new BashExpression(
       `
 COPY $APP_DIR/package.json $APP_DIR/yarn.lock ./
@@ -43,7 +44,7 @@ RUN yarn --frozen-lockfile --production=true --ignore-engines
 
   // yarn >= 4 ships with built-in plugins, see https://github.com/yarnpkg/berry/pull/4253
   const doesNotShipWithBuiltInPlugins = ["2", "3"].some((v) =>
-    context.packageManagerInfo.version.startsWith(v),
+    packageManagerInfo.version.startsWith(v),
   );
   const maybeAddWorkspaceToolsCommand = doesNotShipWithBuiltInPlugins
     ? "RUN yarn plugin import workspace-tools"
@@ -59,22 +60,28 @@ RUN yarn workspaces focus --production
   );
 };
 
-export const createMeteorBuildJobs = (
+export const createMeteorBuildJobs = async (
   context: ComponentContextWithBuild,
-): CatladderJob[] => {
+): Promise<CatladderJob[]> => {
   const buildConfig = context.build.config;
 
   if (!isOfBuildType(buildConfig, "meteor")) {
     throw new Error("deploy config is not meteor");
   }
 
-  const yarnInstall = getYarnInstall(context);
+  const [yarnInstall, nodeCache, meteorInstallScripts] = await Promise.all([
+    getYarnInstall(context),
+    getNodeCache(context),
+    buildConfig.installScripts
+      ? getMeteorDockerInstallScripts(context)
+      : Promise.resolve("" as const),
+  ]);
 
   return createComponentBuildJobs(context, {
     appBuild:
-      buildConfig.buildCommand !== null
+      buildConfig.buildCommand !== null && buildConfig.buildCommand !== false
         ? {
-            cache: [...getNodeCache(context), ...getMeteorCache(context)],
+            cache: [...nodeCache, ...getMeteorCache(context)],
             image: getRunnerImage("jobs-meteor"),
             variables: {
               METEOR_DISABLE_OPTIMISTIC_CACHING: "1", // see https://forums.meteor.com/t/veeery-long-building-time-inside-docker-container/58673/17?u=macrozone
@@ -91,9 +98,7 @@ export const createMeteorBuildJobs = (
     dockerBuild: {
       script: getDockerBuildScriptWithBuiltInDockerFile(context, "meteor"),
       variables: {
-        METEOR_INSTALL_SCRIPTS: buildConfig.installScripts
-          ? getMeteorDockerInstallScripts(context)
-          : "",
+        METEOR_INSTALL_SCRIPTS: meteorInstallScripts,
       },
     },
   });

package/src/build/node/testJob.ts

@@ -12,9 +12,9 @@ import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
 import { ensureNodeVersion, getYarnInstall } from "./yarn";
 import { createJobCacheFromCacheConfigs } from "../cache/createJobCache";
 
-export const createNodeTestJobs = (
+export const createNodeTestJobs = async (
   context: ComponentContext | WorkspaceContext,
-): CatladderJob[] => {
+): Promise<CatladderJob[]> => {
   // don't run tests after release
   // TODO: this will be replaced by using rules
   if (context.trigger === "taggedRelease") {
@@ -43,7 +43,11 @@ export const createNodeTestJobs = (
     needs: [],
   };
   const buildConfig = context.build.config;
-  const yarnInstall = getYarnInstall(context);
+  const [yarnInstall, packageManagerInfo, nodeCache] = await Promise.all([
+    getYarnInstall(context),
+    context.packageManagerInfo,
+    getNodeCache(context),
+  ]);
   const auditJob: CatladderJob | null =
     buildConfig.audit !== false
       ? {
@@ -58,7 +62,7 @@ export const createNodeTestJobs = (
           script: [
             `cd ${context.build.dir}`,
             ...(ensureArrayOrNull(buildConfig.audit?.command) ?? [
-              context.packageManagerInfo.isClassic
+              packageManagerInfo.isClassic
                 ? "yarn audit --level critical"
                 : "yarn npm audit --environment production --severity critical", // yarn 2
             ]),
@@ -81,7 +85,7 @@ export const createNodeTestJobs = (
             ...(buildConfig.lint?.runnerVariables ?? {}),
           },
           image: buildConfig.lint?.jobImage ?? defaultImage,
-          cache: createJobCacheFromCacheConfigs(context, getNodeCache(context)),
+          cache: createJobCacheFromCacheConfigs(context, nodeCache),
           script: [
             ...ensureNodeVersion(context),
             `cd ${context.build.dir}`,
@@ -108,7 +112,7 @@ export const createNodeTestJobs = (
           },
           image:
             buildConfig.test?.jobImage ?? getRunnerImage("jobs-testing-chrome"),
-          cache: createJobCacheFromCacheConfigs(context, getNodeCache(context)),
+          cache: createJobCacheFromCacheConfigs(context, nodeCache),
           script: [
             ...ensureNodeVersion(context),
             `cd ${context.build.dir}`,

package/src/build/node/yarn.ts

@@ -10,8 +10,9 @@ const YARN_BERRY_PROD_INSTALL = `yarn workspaces focus --production`;
 // FIXME: check why and when rebuild is needed
 const YARN_BERRY_PROD_REBUILD = `yarn rebuild`;
 
-const getYarnInstallCommand = (context: Context) => {
-  if (context.packageManagerInfo.isClassic) {
+const getYarnInstallCommand = async (context: Context) => {
+  const packageManagerInfo = await context.packageManagerInfo;
+  if (packageManagerInfo.isClassic) {
     return YARN_INSTALL_CLASSIC;
   }
   // inline builds make debugging easier as it prints it out in the logs, instead of writing it in temp files
@@ -27,7 +28,7 @@ export const ensureNodeVersion = (context: Context) =>
     "if command -v nvm &> /dev/null && [ -f ./.nvmrc ]; then nvm install; fi",
   ]);
 
-export const getYarnInstall = (
+export const getYarnInstall = async (
   context: Context,
   options?: {
     noCustomPostInstall: boolean;
@@ -44,7 +45,7 @@ export const getYarnInstall = (
     ...collapseableSection(
       "yarninstall",
       "Yarn install",
-    )([getYarnInstallCommand(context)]),
+    )([await getYarnInstallCommand(context)]),
     ...(postInstall && !options?.noCustomPostInstall
       ? collapseableSection(
           "postinstall",
@@ -56,15 +57,16 @@ export const getYarnInstall = (
 
 const DOCKER_COPY_FILES = `COPY --chown=node:node $APP_DIR .`;
 
-export const getDockerAppCopyAndBuildScript = (
+export const getDockerAppCopyAndBuildScript = async (
   context: ComponentContextWithBuild,
 ) => {
-  if (context.packageManagerInfo.isClassic) {
+  const packageManagerInfo = await context.packageManagerInfo;
+  if (packageManagerInfo.isClassic) {
     return new BashExpression(
       `
 RUN ${YARN_INSTALL_CLASSIC} --production --ignore-scripts
 ${DOCKER_COPY_FILES}
-RUN ${YARN_INSTALL_CLASSIC} --production 
+RUN ${YARN_INSTALL_CLASSIC} --production
     `.trim(),
     );
   }
@@ -86,7 +88,7 @@ RUN ${YARN_INSTALL_CLASSIC} --production
   // yarn >= 4 ships with build in plugins, see https://github.com/yarnpkg/berry/pull/4253
   // trying to import those fail on this version
   const doesNotShipWithBuiltInPlugins = ["2", "3"].some((v) =>
-    context.packageManagerInfo.version.startsWith(v),
+    packageManagerInfo.version.startsWith(v),
   );
   const maybeAddWorkspaceToolsCommand = doesNotShipWithBuiltInPlugins
     ? "RUN yarn plugin import workspace-tools"

package/src/build/types.ts

@@ -110,7 +110,7 @@ export type BuildConfigBase = {
   /**
    * define the build command
    */
-  buildCommand?: string | string[] | null;
+  buildCommand?: string | string[] | null | false;
 
   /**
    * customize lint, set false to disable
@@ -432,7 +432,7 @@ export type WorkspaceBuildConfigBase = {
 
 export type WorkspaceBuildConfigNode = {
   type: "node";
-  buildCommand?: string | string[];
+  buildCommand?: string | string[] | null | false;
   /**
    * set docker config defaults for the node build type.
    * Currently not all options are supported.

package/src/context/createComponentContext.ts

@@ -33,7 +33,7 @@ export const createComponentContext = async (
     );
   }
 
-  const packageManagerInfo = await getPackageManagerInfoForComponent(
+  const packageManagerInfoPromise = getPackageManagerInfoForComponent(
     ctx.config,
     ctx.componentName,
   );
@@ -75,7 +75,10 @@ export const createComponentContext = async (
     componentConfigWithoutDefaults,
   );
 
-  const environment = await getEnvironment(ctx);
+  const [environment, packageManagerInfo] = await Promise.all([
+    getEnvironment(ctx),
+    packageManagerInfoPromise,
+  ]);
   const { deploy, build, customJobs, dir } = componentConfig;
   const getComponentDirs: BuildContext["getComponentDirs"] = (mode) => [
     dir,
@@ -133,7 +136,7 @@ export const createComponentContext = async (
       : null,
 
     environment,
-    packageManagerInfo: packageManagerInfo,
+    packageManagerInfo: packageManagerInfoPromise,
     pipelineType: ctx.pipelineType,
     trigger: ctx.trigger,
   };

package/src/context/createWorkspaceContext.ts

@@ -43,7 +43,7 @@ export async function createWorkspaceContext({
     env,
     components,
     fullConfig: config,
-    packageManagerInfo: await getPackageManagerInfoBase(),
+    packageManagerInfo: getPackageManagerInfoBase(),
     build: {
       type: "workspace",
       dir: workspaceConfig.dir ?? ".",

package/src/deploy/cloudRun/createJobs/getCloudRunDeployScripts.ts

@@ -66,7 +66,9 @@ export function getCloudRunDeployScripts(context: ComponentContext) {
           ),
         ),
       ...Object.entries(deployConfig.workerPools ?? {})
-        .filter(([_, workerPool]) => workerPool !== false && workerPool !== null)
+        .filter(
+          ([_, workerPool]) => workerPool !== false && workerPool !== null,
+        )
         .map(([name, workerPool]) =>
           getWorkerPoolDeployScript(
             context,

package/src/deploy/custom/deployJob.ts

@@ -7,9 +7,9 @@ import { createDeployementJobs } from "../base";
 
 import { isOfDeployType } from "../types";
 
-export const createCustomDeployJobs = (
+export const createCustomDeployJobs = async (
   context: ComponentContext,
-): CatladderJob[] => {
+): Promise<CatladderJob[]> => {
   const deployConfig = context.deploy?.config;
 
   if (!isOfDeployType(deployConfig, "custom")) {
@@ -17,7 +17,7 @@ export const createCustomDeployJobs = (
     throw new Error("deploy config is not custom");
   }
   // FIXME: custom deploy currently assumes yarn-based project
-  const yarnInstall = getYarnInstall(context, {
+  const yarnInstall = await getYarnInstall(context, {
     noCustomPostInstall: true,
   });
 

package/src/deploy/index.ts

@@ -19,7 +19,7 @@ export * from "./types";
 export * from "./utils";
 
 export type DeployTypeDefinition<D extends DeployConfig> = {
-  jobs: (context: ComponentContext) => CatladderJob[];
+  jobs: (context: ComponentContext) => CatladderJob[] | Promise<CatladderJob[]>;
   defaults: (envContext: EnvironmentContext<BuildConfig, D>) => PartialDeep<D>;
   additionalSecretKeys: (
     envContext: EnvironmentContext<BuildConfig, D>,

package/src/pipeline/createAllJobs.ts

@@ -65,19 +65,21 @@ export const createAllJobs = async ({
 
             return {
               context: workspaceContext,
-              jobs: createJobsForWorkspace(workspaceContext),
+              jobs: await createJobsForWorkspace(workspaceContext),
             };
           }),
         );
       }),
     ).then((f) => f.flat()),
 
-    components: allComponentsContext.map((context) => {
-      return {
-        context,
-        jobs: createJobsForComponentContext(context),
-      };
-    }),
+    components: await Promise.all(
+      allComponentsContext.map(async (context) => {
+        return {
+          context,
+          jobs: await createJobsForComponentContext(context),
+        };
+      }),
+    ),
     agents: await Promise.all(
       Object.keys(config.agents ?? {}).map(async (agentName) => {
         const context = await createAgentContext({ agentName, config });

package/src/pipeline/createJobsForComponent.ts

@@ -24,19 +24,19 @@ const getCustomJobs = (context: ComponentContext) => {
   const rawJobs = context.customJobs;
   return injectDefaultVarsInCustomJobs(context, rawJobs);
 };
-export const createJobsForComponentContext = (
+export const createJobsForComponentContext = async (
   context: ComponentContext,
-): CatladderJob[] => {
-  const buildJobs =
+): Promise<CatladderJob[]> => {
+  const [buildJobs, deployJobs] = await Promise.all([
     context.build.type !== "disabled"
       ? BUILD_TYPES[context.build.buildType].jobs(
           context as ComponentContextWithBuild,
         )
-      : [];
-  const deployJobs =
+      : [],
     context.componentConfig.deploy !== false
       ? DEPLOY_TYPES[context.componentConfig.deploy.type].jobs(context)
-      : [];
+      : [],
+  ]);
 
   const customJobs = getCustomJobs(context);
 

package/src/pipeline/createJobsForWorkspace.ts

@@ -2,11 +2,11 @@ import { WORKSPACE_BUILD_TYPES } from "../build";
 import type { WorkspaceContext } from "../types/context";
 import type { CatladderJob } from "../types/jobs";
 
-export const createJobsForWorkspace = (
+export const createJobsForWorkspace = async (
   context: WorkspaceContext,
-): CatladderJob[] => {
+): Promise<CatladderJob[]> => {
   const buildJobs =
-    WORKSPACE_BUILD_TYPES[context.build.buildType].jobs(context);
+    await WORKSPACE_BUILD_TYPES[context.build.buildType].jobs(context);
 
   return buildJobs;
 };

package/src/pipeline/yarn/yarnUtils.ts

@@ -1,4 +1,6 @@
 import { exec } from "child-process-promise";
+import { readFile } from "fs/promises";
+import { join } from "path";
 import memoizee from "memoizee";
 import type { YarnWorkspace } from "../../types";
 import { jsonParseOrThrow } from "../../utils/jsonParse";
@@ -10,9 +12,41 @@ const execOrFail = async (cmd: string, onFail: string): Promise<string> => {
     return onFail ?? null;
   }
 };
+
+const readPackageManagerVersion = async (
+  dir: string,
+): Promise<string | null> => {
+  try {
+    const pkg = JSON.parse(await readFile(join(dir, "package.json"), "utf-8"));
+    const match = /^yarn@(.+)$/.exec(pkg.packageManager);
+    return match?.[1] ?? null;
+  } catch {
+    return null;
+  }
+};
+
+const getGitRoot = async (): Promise<string | null> => {
+  try {
+    return (await exec("git rev-parse --show-toplevel")).stdout.trim();
+  } catch {
+    return null;
+  }
+};
+
 // export for mocking
 export const getYarnVersion = memoizee(
   async () => {
+    // Fast path: read from package.json packageManager field
+    const fromCwd = await readPackageManagerVersion(process.cwd());
+    if (fromCwd) return fromCwd;
+
+    const gitRoot = await getGitRoot();
+    if (gitRoot && gitRoot !== process.cwd()) {
+      const fromGitRoot = await readPackageManagerVersion(gitRoot);
+      if (fromGitRoot) return fromGitRoot;
+    }
+
+    // Fallback: invoke yarn CLI
     return await execOrFail("yarn --version", "");
   },
   { promise: true },

package/src/types/context.ts

@@ -90,7 +90,7 @@ export type PackageManagerInfoBase = YarnPackageManagerInfoBase;
 export type ContextBeforeConfig = {
   componentName: string;
   fullConfig: Config;
-  packageManagerInfo?: PackageManagerInfoComponent;
+  packageManagerInfo?: Promise<PackageManagerInfoComponent>;
 };
 
 export type BuildContextBase = {
@@ -175,7 +175,7 @@ export type ComponentContext<
   trigger?: PipelineTrigger;
   pipelineType?: PipelineType;
 
-  packageManagerInfo: PackageManagerInfoComponent;
+  packageManagerInfo: Promise<PackageManagerInfoComponent>;
 
   customJobs?: CatladderJob[];
 };
@@ -208,7 +208,7 @@ export type WorkspaceContext = {
   name: string;
   fullConfig: Config;
   workspaceConfig: WorkspaceBuildConfig;
-  packageManagerInfo: PackageManagerInfoBase;
+  packageManagerInfo: Promise<PackageManagerInfoBase>;
   components: Array<ComponentContext>;
   build: BuildContextWorkspace;
   trigger: PipelineTrigger;