@catladder/pipeline 3.42.1 → 3.43.0

package/examples/cloud-run-meteor-with-worker.ts

@@ -13,15 +13,14 @@ const config = {
         type: "google-cloudrun",
         projectId: "google-project-id",
         region: "europe-west6",
-        additionalServices: {
+        // worker pools are like services, but they don't have a load balanced endpoint and don't support autoscaling.
+        // But they are 40% cheaper than services.
+        workerPools: {
           // its usually better to create another catladder component for the worker
-          // but in some legacy cases it might be easier to add an extra service
+          // but in some legacy cases it might be easier to add service or worker pool
           // notice that you can't use different secrets on the worker, it will receive the same env vars as the normal service
           // you can specify extra vars though
           worker: {
-            noCpuThrottling: true,
-            maxInstances: 1,
-            minInstances: 1,
             command: [
               "/bin/sh",
               "-c",

package/examples/cloud-run-with-worker.ts

@@ -13,15 +13,14 @@ const config = {
         type: "google-cloudrun",
         projectId: "google-project-id",
         region: "europe-west6",
-        additionalServices: {
+        // worker pools are like services, but they don't have a load balanced endpoint and don't support autoscaling.
+        // But they are 40% cheaper than services.
+        workerPools: {
           // its usually better to create another catladder component for the worker
-          // but in some legacy cases it might be easier to add an extra service
+          // but in some legacy cases it might be easier to add service or worker pool
           // notice that you can't use different secrets on the worker, it will receive the same env vars as the normal service
           // you can specify extra vars though
           worker: {
-            noCpuThrottling: true,
-            maxInstances: 1,
-            minInstances: 1,
             command: "yarn start:worker",
           },
         },
@@ -29,7 +28,7 @@ const config = {
       env: {
         review: {
           deploy: {
-            additionalServices: {
+            workerPools: {
               worker: null, // disable the worker service on review
             },
           },

package/examples/cloud-run-worker-pool.test.ts

@@ -0,0 +1,12 @@
+import { it, expect } from "vitest";
+import { createYamlLocalPipeline } from "./__utils__/helpers";
+import config from "./cloud-run-worker-pool";
+
+/**
+ * This test is auto-generated.
+ * Modifications will be overwritten on every `yarn test` run!
+ */
+
+it("matches snapshot for cloud-run-worker-pool local pipeline YAML", async () => {
+  expect(await createYamlLocalPipeline(config)).toMatchSnapshot();
+});

package/examples/cloud-run-worker-pool.ts

@@ -0,0 +1,42 @@
+import type { Config } from "../src";
+
+const config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    api: {
+      dir: "api",
+      build: {
+        type: "node",
+      },
+      deploy: {
+        type: "google-cloudrun",
+        projectId: "google-project-id",
+        region: "europe-west6",
+        workerPools: {
+          worker: {
+            instances: 1, // is the default value
+            command: "yarn start:worker",
+            cpu: 1,
+            memory: "512Mi",
+          },
+        },
+      },
+      env: {
+        review: {
+          deploy: {
+            workerPools: {
+              worker: null, // disable the worker pool on review
+            },
+          },
+        },
+      },
+    },
+  },
+} satisfies Config;
+
+export default config;
+
+export const information = {
+  title: "Cloud Run: Worker Pool",
+};

package/package.json

@@ -53,7 +53,7 @@
     }
   ],
   "license": "MIT",
-  "version": "3.42.1",
+  "version": "3.43.0",
   "scripts": {
     "build:tsc": "yarn tsc",
     "build": "yarn build:compile && yarn build:inline-variables",

package/src/deploy/cloudRun/createJobs/cloudRunWorkerPools.ts

@@ -0,0 +1,75 @@
+import { getLabels } from "../../../context/getLabels";
+import type { ComponentContext } from "../../../types/context";
+
+import type { DeployConfigCloudRunWorkerPool } from "../../types/googleCloudRun";
+import { createArgsString } from "../utils/createArgsString";
+import { getCloudRunServiceOrJobArgsArg } from "../utils/getJobOrServiceArgs";
+import { getServiceName } from "../utils/getServiceName";
+import {
+  gcloudRunCmd,
+  getCommonCloudRunArgs,
+  getCommonDeployArgs,
+  makeLabelString,
+} from "./common";
+import { ENV_VARS_FILENAME } from "./constants";
+import { createVolumeConfig } from "./volumes";
+
+export const getWorkerPoolDeployScript = (
+  context: ComponentContext,
+  workerPool: DeployConfigCloudRunWorkerPool,
+  nameSuffix: string,
+) => {
+  const commonDeployArgs = getCommonDeployArgs(context);
+
+  const serviceName = getServiceName(context);
+
+  const command = workerPool.command;
+
+  const commandArray = Array.isArray(command) ? command : command.split(" ");
+
+  const fullWorkerPoolName = `${serviceName}-${nameSuffix}`;
+
+  const argsString = createArgsString(
+    {
+      command: '"' + commandArray.join(",") + '"',
+      args: getCloudRunServiceOrJobArgsArg(workerPool.args),
+      ...commonDeployArgs,
+      image: workerPool.image ?? commonDeployArgs.image,
+      labels: makeLabelString({
+        ...getLabels(context),
+        "cloud-run-worker-pool-name": fullWorkerPoolName,
+      }),
+      "env-vars-file": ENV_VARS_FILENAME,
+      instances: workerPool.instances ?? 1,
+      cpu: workerPool.cpu ?? 1,
+      memory: workerPool.memory ?? "512Mi",
+      "vpc-connector": workerPool.vpcConnector,
+      "vpc-egress": workerPool.vpcEgress,
+      network: workerPool.network,
+      subnet: workerPool.subnet,
+      gpu: workerPool.gpu,
+      "gpu-type": workerPool.gpuType,
+    },
+    ...createVolumeConfig(workerPool.volumes, "worker-pool"),
+  );
+
+  // Worker pools are in beta and require the beta command
+  return `${gcloudRunCmd("beta")} worker-pools deploy ${fullWorkerPoolName} ${argsString}`;
+};
+
+export const getWorkerPoolDeleteScript = (
+  context: ComponentContext,
+  workerPoolSuffix: string,
+) => {
+  const commonArgs = getCommonCloudRunArgs(context);
+
+  const commonArgsString = createArgsString(commonArgs);
+
+  const serviceName = getServiceName(context);
+
+  const fullWorkerPoolName = `${serviceName}-${workerPoolSuffix}`;
+
+  return [
+    `${gcloudRunCmd("beta")} worker-pools delete ${fullWorkerPoolName} ${commonArgsString}`,
+  ];
+};

package/src/deploy/cloudRun/createJobs/getCloudRunDeployScripts.ts

@@ -10,13 +10,17 @@ import { gcloudServiceAccountLoginCommands } from "../utils/gcloudServiceAccount
 import { getJobCreateScripts } from "./cloudRunJobs";
 import { getOnDeployExecuteScript } from "./execute/onDeploy";
 import { getServiceDeployScript } from "./cloudRunServices";
+import { getWorkerPoolDeployScript } from "./cloudRunWorkerPools";
 import {
   getCloudRunDeployConfig,
   setGoogleProjectNumberScript,
 } from "./common";
 import { ENV_VARS_FILENAME } from "./constants";
 import { getCreateScheduleScript } from "./execute/schedules";
-import type { DeployConfigCloudRunService } from "../../types";
+import type {
+  DeployConfigCloudRunService,
+  DeployConfigCloudRunWorkerPool,
+} from "../../types";
 
 export function getCloudRunDeployScripts(context: ComponentContext) {
   const deployConfig = getCloudRunDeployConfig(context);
@@ -61,6 +65,15 @@ export function getCloudRunDeployScripts(context: ComponentContext) {
             "-" + name,
           ),
         ),
+      ...Object.entries(deployConfig.workerPools ?? {})
+        .filter(([_, workerPool]) => workerPool !== false && workerPool !== null)
+        .map(([name, workerPool]) =>
+          getWorkerPoolDeployScript(
+            context,
+            workerPool as DeployConfigCloudRunWorkerPool,
+            name,
+          ),
+        ),
       ...getOnDeployExecuteScript(context, "postDeploy"),
     ]),
     ...collapseableSection(

package/src/deploy/cloudRun/createJobs/getCloudRunStopScripts.ts

@@ -7,6 +7,7 @@ import { getDeleteJobsScripts } from "./cloudRunJobs";
 import { getOnDeployExecuteScript } from "./execute/onDeploy";
 import { getDeleteSchedulesScript } from "./execute/schedules";
 import { getServiceDeleteScript } from "./cloudRunServices";
+import { getWorkerPoolDeleteScript } from "./cloudRunWorkerPools";
 import { getCloudRunDeployConfig } from "./common";
 
 export function getCloudRunStopScripts(context: ComponentContext) {
@@ -18,6 +19,9 @@ export function getCloudRunStopScripts(context: ComponentContext) {
     ...Object.entries(deployConfig.additionalServices ?? {})
       .filter(([_, service]) => service !== false && service !== null)
       .flatMap(([name]) => getServiceDeleteScript(context, name)),
+    ...Object.entries(deployConfig.workerPools ?? {})
+      .filter(([_, workerPool]) => workerPool !== false && workerPool !== null)
+      .flatMap(([name]) => getWorkerPoolDeleteScript(context, name)),
     ...getOnDeployExecuteScript(context, "postStop"),
     ...getDeleteSchedulesScript(context),
     ...getDeleteJobsScripts(context),

package/src/deploy/cloudRun/createJobs/volumes.ts

@@ -3,7 +3,7 @@ import type { keyValuesArg } from "../utils/createArgsString";
 
 export const createVolumeConfig = (
   volumes: DeployConfigCloudRunVolumes | undefined,
-  type: "service" | "job",
+  type: "service" | "job" | "worker-pool",
 ): keyValuesArg[] => {
   if (!volumes) {
     return [];

package/src/deploy/types/googleCloudRun.ts

@@ -436,6 +436,52 @@ export type DeployConfigCloudRunJob =
   | DeployConfigCloudRunJobNormal
   | DeployConfigCloudRunJobWithSchedule;
 
+export type DeployConfigCloudRunWorkerPool = {
+  /**
+   * command / entrypoint for the worker pool
+   */
+  command: string | string[];
+
+  /**
+   * number of instances to run. Defaults to 1.
+   * Worker pools do not support autoscaling.
+   */
+  instances?: number;
+
+  /**
+   * CPU limit. Defaults to 1
+   */
+  cpu?: Cpu;
+
+  /**
+   * memory limit. Defaults to 512Mi
+   */
+  memory?: Memory;
+
+  /**
+   * the image to use. Defaults to the image from the build.
+   */
+  image?: string;
+
+  /**
+   * args to pass to the command
+   */
+  args?: string[];
+
+  /**
+   * Number of GPUs to use. Defaults to 0
+   *
+   * This is a preview feature and not all regions support it.
+   */
+  gpu?: number;
+
+  /**
+   * gpu type to use. Refer to https://cloud.google.com/run/docs/configuring/services/gpu#gcloud for defaults
+   */
+  gpuType?: string;
+} & DeployConfigCloudRunWithVolumes &
+  DeployConfigCloudRunNetworkConfig;
+
 export type DeployConfigCloudRun = Omit<DeployConfigBase, "execute"> & {
   /**
    * cloud run deployment.
@@ -470,6 +516,15 @@ export type DeployConfigCloudRun = Omit<DeployConfigBase, "execute"> & {
     [name: string]: DeployConfigCloudRunJob | false | null;
   };
 
+  /**
+   * deploy worker pools for continuous background work.
+   * Worker pools are cost-effective alternatives to services for background processing.
+   * They don't have a load balanced endpoint and don't support autoscaling.
+   */
+  workerPools?: {
+    [name: string]: DeployConfigCloudRunWorkerPool | false | null;
+  };
+
   /**
    * add cloudSql
    */