@catladder/pipeline 3.42.0 → 3.43.0

package/package.json

@@ -53,7 +53,7 @@
     }
   ],
   "license": "MIT",
-  "version": "3.42.0",
+  "version": "3.43.0",
   "scripts": {
     "build:tsc": "yarn tsc",
     "build": "yarn build:compile && yarn build:inline-variables",

package/src/build/node/testJob.ts

@@ -60,7 +60,7 @@ export const createNodeTestJobs = (
             ...(ensureArrayOrNull(buildConfig.audit?.command) ?? [
               context.packageManagerInfo.isClassic
                 ? "yarn audit --level critical"
-                : "yarn npm audit --environment production --level critical", // yarn 2
+                : "yarn npm audit --environment production --severity critical", // yarn 2
             ]),
           ],
           allow_failure: buildConfig.audit?.allowFailure ?? true,

package/src/deploy/cloudRun/createJobs/cloudRunWorkerPools.ts

@@ -0,0 +1,75 @@
+import { getLabels } from "../../../context/getLabels";
+import type { ComponentContext } from "../../../types/context";
+
+import type { DeployConfigCloudRunWorkerPool } from "../../types/googleCloudRun";
+import { createArgsString } from "../utils/createArgsString";
+import { getCloudRunServiceOrJobArgsArg } from "../utils/getJobOrServiceArgs";
+import { getServiceName } from "../utils/getServiceName";
+import {
+  gcloudRunCmd,
+  getCommonCloudRunArgs,
+  getCommonDeployArgs,
+  makeLabelString,
+} from "./common";
+import { ENV_VARS_FILENAME } from "./constants";
+import { createVolumeConfig } from "./volumes";
+
+export const getWorkerPoolDeployScript = (
+  context: ComponentContext,
+  workerPool: DeployConfigCloudRunWorkerPool,
+  nameSuffix: string,
+) => {
+  const commonDeployArgs = getCommonDeployArgs(context);
+
+  const serviceName = getServiceName(context);
+
+  const command = workerPool.command;
+
+  const commandArray = Array.isArray(command) ? command : command.split(" ");
+
+  const fullWorkerPoolName = `${serviceName}-${nameSuffix}`;
+
+  const argsString = createArgsString(
+    {
+      command: '"' + commandArray.join(",") + '"',
+      args: getCloudRunServiceOrJobArgsArg(workerPool.args),
+      ...commonDeployArgs,
+      image: workerPool.image ?? commonDeployArgs.image,
+      labels: makeLabelString({
+        ...getLabels(context),
+        "cloud-run-worker-pool-name": fullWorkerPoolName,
+      }),
+      "env-vars-file": ENV_VARS_FILENAME,
+      instances: workerPool.instances ?? 1,
+      cpu: workerPool.cpu ?? 1,
+      memory: workerPool.memory ?? "512Mi",
+      "vpc-connector": workerPool.vpcConnector,
+      "vpc-egress": workerPool.vpcEgress,
+      network: workerPool.network,
+      subnet: workerPool.subnet,
+      gpu: workerPool.gpu,
+      "gpu-type": workerPool.gpuType,
+    },
+    ...createVolumeConfig(workerPool.volumes, "worker-pool"),
+  );
+
+  // Worker pools are in beta and require the beta command
+  return `${gcloudRunCmd("beta")} worker-pools deploy ${fullWorkerPoolName} ${argsString}`;
+};
+
+export const getWorkerPoolDeleteScript = (
+  context: ComponentContext,
+  workerPoolSuffix: string,
+) => {
+  const commonArgs = getCommonCloudRunArgs(context);
+
+  const commonArgsString = createArgsString(commonArgs);
+
+  const serviceName = getServiceName(context);
+
+  const fullWorkerPoolName = `${serviceName}-${workerPoolSuffix}`;
+
+  return [
+    `${gcloudRunCmd("beta")} worker-pools delete ${fullWorkerPoolName} ${commonArgsString}`,
+  ];
+};

package/src/deploy/cloudRun/createJobs/getCloudRunDeployScripts.ts

@@ -10,13 +10,17 @@ import { gcloudServiceAccountLoginCommands } from "../utils/gcloudServiceAccount
 import { getJobCreateScripts } from "./cloudRunJobs";
 import { getOnDeployExecuteScript } from "./execute/onDeploy";
 import { getServiceDeployScript } from "./cloudRunServices";
+import { getWorkerPoolDeployScript } from "./cloudRunWorkerPools";
 import {
   getCloudRunDeployConfig,
   setGoogleProjectNumberScript,
 } from "./common";
 import { ENV_VARS_FILENAME } from "./constants";
 import { getCreateScheduleScript } from "./execute/schedules";
-import type { DeployConfigCloudRunService } from "../../types";
+import type {
+  DeployConfigCloudRunService,
+  DeployConfigCloudRunWorkerPool,
+} from "../../types";
 
 export function getCloudRunDeployScripts(context: ComponentContext) {
   const deployConfig = getCloudRunDeployConfig(context);
@@ -61,6 +65,15 @@ export function getCloudRunDeployScripts(context: ComponentContext) {
             "-" + name,
           ),
         ),
+      ...Object.entries(deployConfig.workerPools ?? {})
+        .filter(([_, workerPool]) => workerPool !== false && workerPool !== null)
+        .map(([name, workerPool]) =>
+          getWorkerPoolDeployScript(
+            context,
+            workerPool as DeployConfigCloudRunWorkerPool,
+            name,
+          ),
+        ),
       ...getOnDeployExecuteScript(context, "postDeploy"),
     ]),
     ...collapseableSection(

package/src/deploy/cloudRun/createJobs/getCloudRunStopScripts.ts

@@ -7,6 +7,7 @@ import { getDeleteJobsScripts } from "./cloudRunJobs";
 import { getOnDeployExecuteScript } from "./execute/onDeploy";
 import { getDeleteSchedulesScript } from "./execute/schedules";
 import { getServiceDeleteScript } from "./cloudRunServices";
+import { getWorkerPoolDeleteScript } from "./cloudRunWorkerPools";
 import { getCloudRunDeployConfig } from "./common";
 
 export function getCloudRunStopScripts(context: ComponentContext) {
@@ -18,6 +19,9 @@ export function getCloudRunStopScripts(context: ComponentContext) {
     ...Object.entries(deployConfig.additionalServices ?? {})
       .filter(([_, service]) => service !== false && service !== null)
       .flatMap(([name]) => getServiceDeleteScript(context, name)),
+    ...Object.entries(deployConfig.workerPools ?? {})
+      .filter(([_, workerPool]) => workerPool !== false && workerPool !== null)
+      .flatMap(([name]) => getWorkerPoolDeleteScript(context, name)),
     ...getOnDeployExecuteScript(context, "postStop"),
     ...getDeleteSchedulesScript(context),
     ...getDeleteJobsScripts(context),

package/src/deploy/cloudRun/createJobs/volumes.ts

@@ -3,7 +3,7 @@ import type { keyValuesArg } from "../utils/createArgsString";
 
 export const createVolumeConfig = (
   volumes: DeployConfigCloudRunVolumes | undefined,
-  type: "service" | "job",
+  type: "service" | "job" | "worker-pool",
 ): keyValuesArg[] => {
   if (!volumes) {
     return [];

package/src/deploy/types/googleCloudRun.ts

@@ -436,6 +436,52 @@ export type DeployConfigCloudRunJob =
   | DeployConfigCloudRunJobNormal
   | DeployConfigCloudRunJobWithSchedule;
 
+export type DeployConfigCloudRunWorkerPool = {
+  /**
+   * command / entrypoint for the worker pool
+   */
+  command: string | string[];
+
+  /**
+   * number of instances to run. Defaults to 1.
+   * Worker pools do not support autoscaling.
+   */
+  instances?: number;
+
+  /**
+   * CPU limit. Defaults to 1
+   */
+  cpu?: Cpu;
+
+  /**
+   * memory limit. Defaults to 512Mi
+   */
+  memory?: Memory;
+
+  /**
+   * the image to use. Defaults to the image from the build.
+   */
+  image?: string;
+
+  /**
+   * args to pass to the command
+   */
+  args?: string[];
+
+  /**
+   * Number of GPUs to use. Defaults to 0
+   *
+   * This is a preview feature and not all regions support it.
+   */
+  gpu?: number;
+
+  /**
+   * gpu type to use. Refer to https://cloud.google.com/run/docs/configuring/services/gpu#gcloud for defaults
+   */
+  gpuType?: string;
+} & DeployConfigCloudRunWithVolumes &
+  DeployConfigCloudRunNetworkConfig;
+
 export type DeployConfigCloudRun = Omit<DeployConfigBase, "execute"> & {
   /**
    * cloud run deployment.
@@ -470,6 +516,15 @@ export type DeployConfigCloudRun = Omit<DeployConfigBase, "execute"> & {
     [name: string]: DeployConfigCloudRunJob | false | null;
   };
 
+  /**
+   * deploy worker pools for continuous background work.
+   * Worker pools are cost-effective alternatives to services for background processing.
+   * They don't have a load balanced endpoint and don't support autoscaling.
+   */
+  workerPools?: {
+    [name: string]: DeployConfigCloudRunWorkerPool | false | null;
+  };
+
   /**
    * add cloudSql
    */