@catladder/pipeline 2.9.3 → 2.10.1

package/examples/cloud-run-no-service.ts

@@ -17,6 +17,12 @@ const config: Config = {
         jobs: {
           "alarm-clock": {
             command: "./wake-up-call",
+          },
+        },
+        execute: {
+          "alarm-clock-scheduler": {
+            type: "job",
+            job: "alarm-clock",
             when: "schedule",
             schedule: "0 7 0 0 1-5",
           },

package/examples/cloud-run-post-stop-job.ts

@@ -18,6 +18,7 @@ const config: Config = {
         review: {
           deploy: {
             jobs: {
+              // in this example we only declare the job on the review environment.
               ["drop-db"]: {
                 command: [
                   "/bin/sh",
@@ -25,6 +26,12 @@ const config: Config = {
                   "mongosh \\$MONGO_URL --eval 'db.dropDatabase()'",
                 ],
                 image: "rtsp/mongosh:latest",
+              },
+            },
+            execute: {
+              ["drop-db"]: {
+                type: "job",
+                job: "drop-db",
                 when: "postStop",
               },
             },

package/examples/cloud-run-service-custom-vpc-connector.ts

@@ -20,7 +20,6 @@ const config: Config = {
         },
         jobs: {
           myjob: {
-            when: "manual",
             command: "echo hello",
             vpcConnector: "my-first-vpc-connector",
             vpcEgress: "all-traffic",

package/examples/cloud-run-service-custom-vpc.ts

@@ -21,7 +21,6 @@ const config: Config = {
         },
         jobs: {
           myjob: {
-            when: "manual",
             command: "echo hello",
             network: "my-network",
             subnet: "my-subnet",

package/examples/cloud-run-service-with-volumes.ts

@@ -34,7 +34,7 @@ const config: Config = {
         jobs: {
           migrate: {
             command: "migrate",
-            when: "postDeploy",
+
             volumes: {
               myMount: {
                 type: "cloud-storage",
@@ -44,6 +44,14 @@ const config: Config = {
             },
           },
         },
+
+        execute: {
+          migrate: {
+            type: "job",
+            job: "migrate",
+            when: "postDeploy",
+          },
+        },
       },
     },
   },

package/examples/cloud-run-with-sql-legacy-jobs.test.ts

@@ -0,0 +1,11 @@
+import { createYamlLocalPipeline } from "./__utils__/helpers";
+import config from "./cloud-run-with-sql-legacy-jobs";
+
+/**
+ * This test is auto-generated.
+ * Modifications will be overwritten on every `yarn test` run!
+ */
+
+it("matches snapshot for cloud-run-with-sql-legacy-jobs local pipeline YAML", async () => {
+  expect(await createYamlLocalPipeline(config)).toMatchSnapshot();
+});

package/examples/cloud-run-with-sql-legacy-jobs.ts

@@ -0,0 +1,49 @@
+import type { Config } from "../src";
+
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    api: {
+      dir: "api",
+      build: {
+        type: "node",
+      },
+      deploy: {
+        type: "google-cloudrun",
+        projectId: "google-project-id",
+        region: "europe-west6",
+        // optional, set min and max instances
+        // defaults to 0-100
+        service: {
+          minInstances: 0,
+          maxInstances: 5,
+        },
+        cloudSql: {
+          type: "unmanaged",
+          instanceConnectionName: "projectId:region:instancename",
+          dbUser: "my-user",
+        },
+        jobs: {
+          migration: {
+            when: "preDeploy",
+            command: "yarn migrate",
+            waitForCompletion: true,
+          },
+          ["send-reminders"]: {
+            when: "schedule",
+            command: "yarn job:send-reminders",
+            schedule: "0 * * * *",
+            timeout: "15m",
+          },
+        },
+      },
+    },
+  },
+};
+
+export default config;
+
+export const information = {
+  title: "Cloud Run: With SQL",
+};

package/examples/cloud-run-with-sql.ts

@@ -25,16 +25,50 @@ const config: Config = {
           dbUser: "my-user",
         },
         jobs: {
-          migration: {
-            when: "preDeploy",
+          migrate: {
             command: "yarn migrate",
+          },
+          ["send-reminders"]: {
+            command: "yarn job:send-reminders",
+            args: ["--verbose=false"],
+          },
+        },
+        execute: {
+          migrate: {
+            type: "job",
+            job: "migrate",
+            when: "preDeploy",
             waitForCompletion: true,
+            args: ["--verbose=true", "--create-db"],
+          },
+          ["send-reminders-admins"]: {
+            type: "job",
+            when: "schedule",
+            job: "send-reminders",
+            args: ["--only-admins", "--verbose=true"],
+            schedule: "0 * * * *",
           },
           ["send-reminders"]: {
+            type: "job",
+            when: "schedule",
+            job: "send-reminders",
+            schedule: "0 * * * *",
+          },
+          ["call-http-endpoint"]: {
+            type: "http",
+            url: "${ROOT_URL}/myEndpoint",
             when: "schedule",
-            command: "yarn job:send-reminders",
             schedule: "0 * * * *",
-            timeout: "15m",
+            method: "GET",
+          },
+        },
+      },
+      env: {
+        review: {
+          deploy: {
+            execute: {
+              "send-reminders": null, // disable on this env
+            },
           },
         },
       },

package/package.json

@@ -53,7 +53,7 @@
     }
   ],
   "license": "MIT",
-  "version": "2.9.3",
+  "version": "2.10.1",
   "scripts": {
     "build:tsc": "yarn tsc",
     "build": "yarn build:compile && yarn build:inline-variables",

package/src/bash/bashEscape.ts

@@ -57,6 +57,9 @@ export const escapeDoubleQuotes = (value: string | null | undefined) =>
 export const escapeSingleQuotes = (value: string | null | undefined) =>
   value?.toString().replace(/'/g, "\\'");
 
+export type EscapeForDotEnvOptions = {
+  quoteMode: "auto" | "always";
+};
 /**
  *
  * escape env vars for .env files.
@@ -75,6 +78,9 @@ export const escapeSingleQuotes = (value: string | null | undefined) =>
  */
 export const escapeForDotEnv = (
   value: VariableValue | undefined | null,
+  options: EscapeForDotEnvOptions = {
+    quoteMode: "auto",
+  },
 ): string => {
   if (value === undefined || value === null) {
     return "";
@@ -82,7 +88,7 @@ export const escapeForDotEnv = (
   if (typeof value === "string") {
     // if string contains newlines, we need to wrap it in quotes
     // we additionaly escape newlines, that give best compatibility
-    if (value.includes("\n")) {
+    if (options.quoteMode === "always" || value.includes("\n")) {
       const newlinesReplaces = value.replace(/\n/g, "\\n");
 
       // default to ", but if this is not possible, we try to use ' or `

package/src/deploy/cloudRun/createJobs/cloudRunJobs.ts

@@ -1,90 +1,36 @@
 import type { ComponentContext } from "../../../types/context";
 
-import type {
-  DeployConfigCloudRunJob,
-  DeployConfigCloudRunJobWithSchedule,
-} from "../../types/googleCloudRun";
+import { getLabels } from "../../../context/getLabels";
+import { notNil } from "../../../utils";
+import type { DeployConfigCloudRunJob } from "../../types/googleCloudRun";
 import { createArgsString } from "../utils/createArgsString";
-import { getCloudRunJobName } from "../utils/jobName";
+import { getFullJobName } from "../utils/jobName";
 import {
   gcloudRunCmd,
-  gcloudSchedulerCmd,
   getCloudRunDeployConfig,
   getCommonCloudRunArgs,
   getCommonDeployArgs,
   makeLabelString,
 } from "./common";
-import { getLabels } from "../../../context/getLabels";
-import { createVolumeConfig } from "./volumes";
-import type {
-  BashExpression,
-  StringOrBashExpression,
-} from "../../../bash/BashExpression";
 import { ENV_VARS_FILENAME } from "./constants";
-import { notNil } from "../../../utils";
-
-const getJobRunScriptForJob = (
-  context: ComponentContext,
-  jobName: StringOrBashExpression,
-  wait: boolean,
-) => {
-  const commonArgs = getCommonCloudRunArgs(context);
-
-  const commonArgsString = createArgsString(commonArgs);
-  return `${gcloudRunCmd()} jobs execute ${jobName.toString()} ${commonArgsString}${
-    wait ? " --wait" : ""
-  }`;
-};
-
-export const getDeleteSchedulesScripts = (context: ComponentContext) => {
-  const deployConfig = getCloudRunDeployConfig(context);
-  const schedules = getSchedules(context);
-  const argsString = createArgsString({
-    project: deployConfig.projectId,
-    location: deployConfig.region,
-  });
-  return schedules
-    .map(({ name }) => {
-      return [`${gcloudSchedulerCmd()} jobs delete ${name} ${argsString}`];
-    })
-    .flat();
-};
+import { createVolumeConfig } from "./volumes";
+import { getCloudRunJobArgsArg } from "./execute/onDeploy";
 
 export const getDeleteJobsScripts = (context: ComponentContext) => {
   const commonArgs = getCommonCloudRunArgs(context);
   const commonArgsString = createArgsString(commonArgs);
   const jobsWithNames = getCloudRunJobsWithNames(context);
 
-  return jobsWithNames.flatMap(({ jobName }) => [
+  return jobsWithNames.flatMap(({ fullJobName }) => [
     // first delete all job executions. Otherwise delete might fail if one of those is still running
-    `${gcloudRunCmd()} jobs executions list ${commonArgsString} --job ${jobName} --format="value(name)" | xargs -I {} ${gcloudRunCmd()} jobs executions delete {}  --quiet ${commonArgsString}`,
-    `${gcloudRunCmd()} jobs delete ${jobName} ${commonArgsString}`,
+    `${gcloudRunCmd()} jobs executions list ${commonArgsString} --job ${fullJobName} --format="value(name)" | xargs -I {} ${gcloudRunCmd()} jobs executions delete {}  --quiet ${commonArgsString}`,
+    `${gcloudRunCmd()} jobs delete ${fullJobName} ${commonArgsString}`,
   ]);
 };
 
-export const getJobRunScripts = (
-  context: ComponentContext,
-  when: DeployConfigCloudRunJob["when"],
-) => {
-  const jobsWithNames = getCloudRunJobsWithNames(context);
-
-  return jobsWithNames
-    .filter(({ job }) => job.when === when)
-    .map(({ jobName, job }) => {
-      // always wait for completion for preStop and postStop jobs
-      // since stop will delete the jobs afterwards, so they will fail
-      const waitForCompletion = ["preStop", "postStop"].includes(when)
-        ? true
-        : "waitForCompletion" in job
-          ? (job.waitForCompletion ?? false)
-          : false;
-      return getJobRunScriptForJob(context, jobName, waitForCompletion);
-    });
-};
-
 export const getJobCreateScripts = (context: ComponentContext): string[] =>
   getCloudRunJobsWithNames(context).map(
-    ({ job, jobName }, jobIndex): string => {
+    ({ job, fullJobName }, jobIndex): string => {
       const commandArray = Array.isArray(job.command)
         ? job.command
         : job.command.split(" ");
@@ -98,6 +44,7 @@ export const getJobCreateScripts = (context: ComponentContext): string[] =>
       const commonDeployArgsString = createArgsString(
         {
           command: `"${commandArray.join(",")}"`,
+          args: getCloudRunJobArgsArg(job.args),
           labels: `"${makeLabelString(getLabels(context))},cloud-run-job-name=$current_job_name"`,
           image: `"${job.image ?? commonImage}"`,
           project,
@@ -125,7 +72,7 @@ export const getJobCreateScripts = (context: ComponentContext): string[] =>
         jobIndex === 0
           ? `exist_job_names="$(\n  ${gcloudRunCmd()} jobs list --filter='metadata.name ~ ${context.env}.*${context.name}' --format='value(name)' --limit=999 --project='${project}' --region='${region}'\n)"`
           : null,
-        `current_job_name="${jobName}"`,
+        `current_job_name="${fullJobName}"`,
         'if grep "$current_job_name" <<<"$exist_job_names" >/dev/null; then',
         `  ${gcloudRunCmd()} jobs update "$current_job_name" ${commonDeployArgsString}`,
         "else",
@@ -137,104 +84,18 @@ export const getJobCreateScripts = (context: ComponentContext): string[] =>
     },
   );
 
-export const getCreateScheduleScripts = (
-  context: ComponentContext,
-): string[] => {
-  const schedules = getSchedules(context);
-  const { region: location, projectId: project } =
-    getCloudRunDeployConfig(context);
-
-  return schedules.map((scheduler, jobIndex): string => {
-    const uri = getSchedulerUrl(scheduler, context);
-
-    const argsString = createArgsString({
-      project,
-      location,
-      uri: `"$current_job_uri"`,
-      "http-method": "POST",
-      "oauth-service-account-email": `"$GCLOUD_PROJECT_NUMBER-compute@developer.gserviceaccount.com"`,
-      schedule: `"${scheduler.schedule}"`,
-      "max-retry-attempts": scheduler.maxRetryAttempts ?? 0,
-    });
-    return [
-      jobIndex === 0
-        ? `exist_scheduler_names="$(\n  ${gcloudSchedulerCmd()} jobs list --filter='httpTarget.uri ~ ${context.env}.*${context.name}' --format='value(name)' --limit=999 --location='${location}' --project='${project}'\n)"`
-        : null,
-      `current_job_uri="${uri}"`,
-      `current_scheduler_name="${scheduler.name}"`,
-      `if grep "$current_scheduler_name" <<<"$exist_scheduler_names" >/dev/null; then`,
-      `  ${gcloudSchedulerCmd()} jobs update http "$current_scheduler_name" ${argsString}`,
-      `else`,
-      `  ${gcloudSchedulerCmd()} jobs create http "$current_scheduler_name" ${argsString}`,
-      `fi`,
-    ]
-      .filter(notNil)
-      .join("\n");
-  });
-};
-
-type Scheduler = {
-  name: StringOrBashExpression;
-  maxRetryAttempts?: number;
-  schedule: string;
-} & {
-  type: "cloudRunJob";
-  jobName: StringOrBashExpression;
-};
-
-const getSchedulerUrl = (scheduler: Scheduler, context: ComponentContext) => {
-  if (scheduler.type === "cloudRunJob") {
-    const { region: location, projectId: project } =
-      getCloudRunDeployConfig(context);
-
-    const uriBase = `https://${location}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${project}/jobs`;
-
-    return `${uriBase}/${scheduler.jobName}:run`;
-  }
-
-  throw new Error(`Unknown scheduler type: ${scheduler.type}`);
-};
-
-const getSchedules = (context: ComponentContext): Scheduler[] => {
-  const jobsWithNames = getCloudRunJobsWithNames(context);
-
-  return jobsWithNames
-    .filter(
-      (
-        entry,
-      ): entry is {
-        jobName: BashExpression;
-        job: DeployConfigCloudRunJobWithSchedule;
-        jobKey: string;
-      } => entry.job.when === "schedule",
-    )
-    .map(({ job: { maxRetryAttempts, schedule }, jobName }) => {
-      const schedulerName = jobName.concat("-scheduler");
-      return {
-        name: schedulerName,
-        maxRetryAttempts,
-        schedule,
-        type: "cloudRunJob",
-        jobName,
-      };
-    });
-};
-
-const getCloudRunJobsWithNames = (context: ComponentContext) => {
+export const getCloudRunJobsWithNames = (context: ComponentContext) => {
   const deployConfig = getCloudRunDeployConfig(context);
 
-  const getFullJobName = (name: string) =>
-    getCloudRunJobName(context.environment.fullName, name);
-
   const jobsWithNames = Object.entries(deployConfig.jobs ?? {})
     // filter out disabled jobs
     .filter((entry): entry is [string, DeployConfigCloudRunJob] =>
       Boolean(entry[1]),
     )
-    .map(([jobKey, job]) => ({
-      jobName: getFullJobName(jobKey),
+    .map(([jobName, job]) => ({
+      fullJobName: getFullJobName(context, jobName),
       job,
-      jobKey,
+      jobName,
     }));
   return jobsWithNames;
 };

package/src/deploy/cloudRun/createJobs/execute/onDeploy.ts

@@ -0,0 +1,112 @@
+import type { StringOrBashExpression } from "../../../../bash";
+import type { ComponentContext } from "../../../../types/context";
+import type {
+  DeployConfigCloudRunExecuteOnDeploy,
+  DeployConfigCloudRunJob,
+} from "../../../types/googleCloudRun";
+import { createArgsString } from "../../utils/createArgsString";
+import { getFullJobName } from "../../utils/jobName";
+import { getCloudRunJobsWithNames } from "../cloudRunJobs";
+import {
+  gcloudRunCmd,
+  getCloudRunDeployConfig,
+  getCommonCloudRunArgs,
+} from "../common";
+
+type Execute = {
+  jobName: StringOrBashExpression;
+  config: DeployConfigCloudRunExecuteOnDeploy;
+};
+export const getOnDeployExecuteScript = (
+  context: ComponentContext,
+  when: DeployConfigCloudRunExecuteOnDeploy["when"],
+) => {
+  const executes = getExecutes(context);
+
+  return executes
+    .filter(({ config }) => config.when === when)
+    .map((execute) => {
+      return getJobRunScriptForExecute(context, execute);
+    });
+};
+
+const getExecutes = (context: ComponentContext): Execute[] => {
+  const deployConfig = getCloudRunDeployConfig(context);
+  return [
+    ...getLegacyExecutes(context),
+    ...Object.entries(deployConfig.execute ?? {}).flatMap(([key, value]) => {
+      if (!value || (value.when !== "schedule" && value.type !== "job")) {
+        return [];
+      }
+      return [
+        {
+          jobName: key,
+          config: value,
+        } as Execute,
+      ];
+    }),
+  ];
+};
+
+const getLegacyExecutes = (context: ComponentContext): Execute[] => {
+  const jobsWithNames = getCloudRunJobsWithNames(context);
+
+  return jobsWithNames.flatMap(({ jobName, job }) => {
+    if (
+      !job.when ||
+      !["preDeploy", "postDeploy", "preStop", "postStop"].includes(job.when)
+    ) {
+      return [];
+    }
+    return [
+      {
+        jobName,
+        config: {
+          job: jobName,
+          type: "job",
+          when: job.when,
+          ...(job.when === "preDeploy" || job.when === "postDeploy"
+            ? {
+                waitForCompletion: job.waitForCompletion,
+              }
+            : {}),
+        } as Execute["config"],
+      },
+    ];
+  });
+};
+
+const getJobRunScriptForExecute = (
+  context: ComponentContext,
+  { jobName, config }: Execute,
+) => {
+  const commonArgs = getCommonCloudRunArgs(context);
+
+  // always wait for completion for preStop and postStop jobs
+  // since stop will delete the jobs afterwards, so they will fail
+  const waitForCompletion = ["preStop", "postStop"].includes(config.when)
+    ? true // always
+    : "waitForCompletion" in config
+      ? (config.waitForCompletion ?? false) // depends on config
+      : false;
+
+  const argString = createArgsString({
+    ...commonArgs,
+    wait: waitForCompletion === true ? true : undefined,
+    args: getCloudRunJobArgsArg(config.args),
+  });
+  const fullJobName = getFullJobName(context, jobName);
+  return `${gcloudRunCmd()} jobs execute ${fullJobName.toString()} ${argString}`;
+};
+
+export const getCloudRunJobArgsArg = (
+  args:
+    | DeployConfigCloudRunExecuteOnDeploy["args"]
+    | DeployConfigCloudRunJob["args"],
+) => {
+  return args !== undefined
+    ? args.length > 0
+      ? args?.map((arg) => `"${arg}"`).join(",")
+      : ""
+    : undefined;
+};