@catladder/pipeline 2.9.2 → 2.10.0

package/src/deploy/cloudRun/createJobs/execute/schedules.ts

@@ -0,0 +1,186 @@
+import { bashEscape } from "../../../../bash";
+import type {
+  BashExpression,
+  StringOrBashExpression,
+} from "../../../../bash/BashExpression";
+import type { ComponentContext } from "../../../../types/context";
+import { notNil } from "../../../../utils";
+import type {
+  DeployConfigCloudRunExecuteWithSchedule,
+  DeployConfigCloudRunJobWithSchedule,
+} from "../../../types";
+import { getCloudRunJobExecuteUrl } from "../../utils/cloudRunExecutionUrl";
+import { createArgsString } from "../../utils/createArgsString";
+import { getFullSchedulerName } from "../../utils/jobName";
+import { getCloudRunJobsWithNames } from "../cloudRunJobs";
+import { gcloudSchedulerCmd, getCloudRunDeployConfig } from "../common";
+
+export const getDeleteSchedulesScript = (context: ComponentContext) => {
+  const deployConfig = getCloudRunDeployConfig(context);
+  const schedules = getSchedules(context);
+  const argsString = createArgsString({
+    project: deployConfig.projectId,
+    location: deployConfig.region,
+  });
+
+  return schedules
+    .map(({ fullName }) => {
+      return [`${gcloudSchedulerCmd()} jobs delete ${fullName} ${argsString}`];
+    })
+    .flat();
+};
+
+export const getCreateScheduleScript = (
+  context: ComponentContext,
+): string[] => {
+  const schedules = getSchedules(context);
+  const { region: location, projectId: project } =
+    getCloudRunDeployConfig(context);
+
+  return schedules.map(({ fullName, config }, jobIndex): string => {
+    const { uri, ...args } = getSchedulerArgs(config, context);
+
+    const argsString = createArgsString({
+      project,
+      location,
+      uri: `"$current_job_uri"`,
+      ...args,
+
+      schedule: `"${config.schedule}"`,
+      "max-retry-attempts": config.maxRetryAttempts ?? 0,
+    });
+    return [
+      jobIndex === 0
+        ? `exist_scheduler_names="$(\n  ${gcloudSchedulerCmd()} jobs list --filter='httpTarget.uri ~ ${context.env}.*${context.name}' --format='value(name)' --limit=999 --location='${location}' --project='${project}'\n)"`
+        : null,
+      `current_job_uri="${uri}"`,
+      `current_scheduler_name="${fullName}"`,
+      `if grep "$current_scheduler_name" <<<"$exist_scheduler_names" >/dev/null; then`,
+      `  ${gcloudSchedulerCmd()} jobs update http "$current_scheduler_name" ${argsString}`,
+      `else`,
+      `  ${gcloudSchedulerCmd()} jobs create http "$current_scheduler_name" ${argsString}`,
+      `fi`,
+    ]
+      .filter(notNil)
+      .join("\n");
+  });
+};
+
+const getSchedulerArgs = (
+  scheduler: DeployConfigCloudRunExecuteWithSchedule,
+  context: ComponentContext,
+): {
+  uri: string;
+  "http-method"?: string;
+  "message-body"?: string;
+  "oauth-service-account-email"?: string;
+  "oidc-service-account-email"?: string;
+} => {
+  if (scheduler.type === "job") {
+    const { projectId, region } = getCloudRunDeployConfig(context);
+    const body =
+      scheduler.args !== undefined
+        ? {
+            overrides: {
+              containerOverrides: [
+                scheduler.args?.length > 0
+                  ? {
+                      args: scheduler.args,
+                    }
+                  : {
+                      args: [],
+                      clearArgs: true, // not sure why this is neeeded, but it is
+                    },
+              ],
+            },
+          }
+        : null;
+    return {
+      uri: getCloudRunJobExecuteUrl(scheduler.job, {
+        appFullName: context.environment.fullName,
+        projectId,
+        region,
+      }),
+      "message-body": body
+        ? '"' + bashEscape(JSON.stringify(body)) + '"'
+        : undefined,
+      "http-method": "POST",
+      "oauth-service-account-email": `"$GCLOUD_PROJECT_NUMBER-compute@developer.gserviceaccount.com"`,
+    };
+  }
+  if (scheduler.type === "http") {
+    return {
+      uri: scheduler.url,
+      "message-body": scheduler.body,
+      "http-method": scheduler.method,
+      "oidc-service-account-email": `"$GCLOUD_PROJECT_NUMBER-compute@developer.gserviceaccount.com"`,
+    };
+  }
+  throw new Error(`Unknown scheduler type: ${(scheduler as any).type}`);
+};
+
+const getSchedules = (
+  context: ComponentContext,
+): {
+  name: string;
+  fullName: StringOrBashExpression;
+  config: DeployConfigCloudRunExecuteWithSchedule;
+}[] => {
+  const legacyScheduleJobs = getLegacyJobSchedules(context);
+  const schedules = getScheduledExecutes(context);
+
+  return Object.entries({ ...legacyScheduleJobs, ...schedules }).map(
+    ([name, config]) => ({
+      name,
+      fullName: getFullSchedulerName(context, name),
+      config,
+    }),
+  );
+};
+
+const getLegacyJobSchedules = (
+  context: ComponentContext,
+): Record<string, DeployConfigCloudRunExecuteWithSchedule> => {
+  const jobsWithNames = getCloudRunJobsWithNames(context);
+
+  return Object.fromEntries(
+    jobsWithNames
+      .filter(
+        (
+          entry,
+        ): entry is {
+          fullJobName: BashExpression;
+          job: DeployConfigCloudRunJobWithSchedule;
+          jobName: string;
+        } => entry.job.when === "schedule",
+      )
+      .map(({ job: { maxRetryAttempts, schedule }, jobName }) => {
+        const schedulerName = jobName.concat("-scheduler");
+        return [
+          schedulerName,
+          {
+            type: "job",
+            job: jobName,
+            maxRetryAttempts,
+            schedule,
+            when: "schedule",
+          },
+        ];
+      }),
+  );
+};
+
+const getScheduledExecutes = (
+  context: ComponentContext,
+): Record<string, DeployConfigCloudRunExecuteWithSchedule> => {
+  const deployConfig = getCloudRunDeployConfig(context);
+
+  return Object.fromEntries(
+    Object.entries(deployConfig.execute ?? {}).flatMap(([key, value]) => {
+      if (!value || value.when !== "schedule") {
+        return [];
+      }
+      return [[key, value]];
+    }),
+  );
+};

package/src/deploy/cloudRun/createJobs/getCloudRunDeployScripts.ts

@@ -8,17 +8,15 @@ import { writeBashYamlToFileScript } from "../../../bash/bashYaml";
 import { getRemoveOldRevisionsAndImagesCommand } from "../cleanup";
 import { getDatabaseCreateScript } from "../utils/database";
 import { gcloudServiceAccountLoginCommands } from "../utils/gcloudServiceAccountLoginCommands";
-import {
-  getCreateScheduleScripts,
-  getJobCreateScripts,
-  getJobRunScripts,
-} from "./cloudRunJobs";
+import { getJobCreateScripts } from "./cloudRunJobs";
+import { getOnDeployExecuteScript } from "./execute/onDeploy";
 import { getServiceDeployScript } from "./cloudRunServices";
 import {
   getCloudRunDeployConfig,
   setGoogleProjectNumberScript,
 } from "./common";
 import { ENV_VARS_FILENAME } from "./constants";
+import { getCreateScheduleScript } from "./execute/schedules";
 
 export function getCloudRunDeployScripts(context: ComponentContext) {
   const deployConfig = getCloudRunDeployConfig(context);
@@ -47,9 +45,9 @@ export function getCloudRunDeployScripts(context: ComponentContext) {
       ...(deployConfig.cloudSql
         ? getDatabaseCreateScript(context, deployConfig) // we create the db, so that we can also delete it afterwards
         : []),
-      ...getCreateScheduleScripts(context),
+      ...getCreateScheduleScript(context),
       ...getJobCreateScripts(context),
-      ...getJobRunScripts(context, "preDeploy"),
+      ...getOnDeployExecuteScript(context, "preDeploy"),
 
       ...(deployConfig.service !== false
         ? [getServiceDeployScript(context, deployConfig.service)]
@@ -58,7 +56,7 @@ export function getCloudRunDeployScripts(context: ComponentContext) {
         ([name, service]) =>
           getServiceDeployScript(context, service, "-" + name),
       ),
-      ...getJobRunScripts(context, "postDeploy"),
+      ...getOnDeployExecuteScript(context, "postDeploy"),
     ]),
     ...collapseableSection(
       "cleanup",

package/src/deploy/cloudRun/createJobs/getCloudRunStopScripts.ts

@@ -4,11 +4,9 @@ import { getDependencyTrackDeleteScript } from "../../sbom";
 import { getRemoveOldRevisionsAndImagesCommand } from "../cleanup";
 import { getDatabaseDeleteScript } from "../utils/database";
 import { gcloudServiceAccountLoginCommands } from "../utils/gcloudServiceAccountLoginCommands";
-import {
-  getDeleteJobsScripts,
-  getDeleteSchedulesScripts,
-  getJobRunScripts,
-} from "./cloudRunJobs";
+import { getDeleteJobsScripts } from "./cloudRunJobs";
+import { getOnDeployExecuteScript } from "./execute/onDeploy";
+import { getDeleteSchedulesScript } from "./execute/schedules";
 import { getServiceDeleteScript } from "./cloudRunServices";
 import { getCloudRunDeployConfig } from "./common";
 
@@ -16,13 +14,13 @@ export function getCloudRunStopScripts(context: ComponentContext) {
   const deployConfig = getCloudRunDeployConfig(context);
   return [
     ...gcloudServiceAccountLoginCommands(context),
-    ...getJobRunScripts(context, "preStop"),
+    ...getOnDeployExecuteScript(context, "preStop"),
     ...(deployConfig.service !== false ? getServiceDeleteScript(context) : []),
     ...Object.entries(deployConfig.additionalServices ?? {}).flatMap(([name]) =>
       getServiceDeleteScript(context, name),
     ),
-    ...getJobRunScripts(context, "postStop"),
-    ...getDeleteSchedulesScripts(context),
+    ...getOnDeployExecuteScript(context, "postStop"),
+    ...getDeleteSchedulesScript(context),
     ...getDeleteJobsScripts(context),
     ...(deployConfig.cloudSql && deployConfig.cloudSql.deleteDatabaseOnStop
       ? getDatabaseDeleteScript(context, deployConfig)

package/src/deploy/cloudRun/index.ts

@@ -9,11 +9,11 @@ import type { EnvironmentContext } from "../../types/environmentContext";
 import { sanitizeForBashVariable } from "../../utils/gitlab";
 import { getFullDbName } from "../cloudSql/utils";
 import { createGoogleCloudRunDeployJobs } from "./createJobs";
+import { getCloudRunJobExecuteUrl } from "./utils/cloudRunExecutionUrl";
 import {
   DATABASE_JDBC_URL,
   getDatabaseConnectionString,
 } from "./utils/database";
-import { getCloudRunJobName } from "./utils/jobName";
 
 export const GCLOUD_DEPLOY_CREDENTIALS_KEY = "GCLOUD_DEPLOY_credentialsKey";
 
@@ -103,11 +103,11 @@ export const GCLOUD_RUN_DEPLOY_TYPE: DeployTypeDefinition<DeployConfigCloudRun>
           ? Object.fromEntries(
               Object.entries(deployConfigRaw.jobs).map(([name, job]) => [
                 "CLOUD_RUN_JOB_TRIGGER_URL_" + sanitizeForBashVariable(name),
-                `https://${
-                  deployConfigRaw.region
-                }-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${
-                  deployConfigRaw.projectId
-                }/jobs/${getCloudRunJobName(fullName, name)}:run`,
+                getCloudRunJobExecuteUrl(name, {
+                  appFullName: fullName,
+                  projectId: deployConfigRaw.projectId,
+                  region: deployConfigRaw.region,
+                }),
               ]),
             )
           : {};

package/src/deploy/cloudRun/utils/cloudRunExecutionUrl.ts

@@ -0,0 +1,20 @@
+import type { StringOrBashExpression } from "../../../bash";
+import { getCloudRunJobName } from "./jobName";
+
+export function getCloudRunJobExecuteUrl(
+  jobName: string,
+  {
+    region,
+    projectId,
+    appFullName,
+  }: {
+    appFullName: StringOrBashExpression;
+    region: string;
+    projectId: string;
+  },
+): string {
+  const uriBase = `https://${region}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${projectId}/jobs`;
+  const fullJobName = getCloudRunJobName(appFullName, jobName);
+
+  return `${uriBase}/${fullJobName}:run`;
+}

package/src/deploy/cloudRun/utils/jobName.ts

@@ -1,7 +1,22 @@
 import type { StringOrBashExpression } from "../../../bash/BashExpression";
+import type { ComponentContext } from "../../../types";
 
 export const getCloudRunJobName = (
   fullAppName: StringOrBashExpression,
-  jobName: string,
+  jobName: StringOrBashExpression,
 ): StringOrBashExpression =>
   fullAppName.toLowerCase().concat("-" + jobName.toLowerCase());
+
+export const getFullJobName = (
+  context: ComponentContext,
+  name: StringOrBashExpression,
+) => getCloudRunJobName(context.environment.fullName, name);
+
+export const getCloudRunSchedulerName = (
+  fullAppName: StringOrBashExpression,
+  schedulerName: string,
+): StringOrBashExpression =>
+  fullAppName.toLowerCase().concat("-" + schedulerName.toLowerCase());
+
+export const getFullSchedulerName = (context: ComponentContext, name: string) =>
+  getCloudRunSchedulerName(context.environment.fullName, name);

package/src/deploy/types/googleCloudRun.ts

@@ -196,6 +196,11 @@ export type DeployConfigCloudRunJobBase = {
    */
   command: string | string[];
 
+  /**
+   * optional, specify args. Those get overwritten by job executions if specified there
+   */
+  args?: string[];
+
   /**
    * custom image to use. Defaults to the image from the build
    */
@@ -232,25 +237,39 @@ type DayOfWeek = string;
 type Month = string;
 export type DeployConfigCloudRunJobWithSchedule =
   DeployConfigCloudRunJobBase & {
+    /**
+     * @deprecated use `execute` instead
+     */
     when: "schedule";
+    /**
+     * @deprecated use `run` instead
+     */
     schedule: `${Minute} ${Hour} ${DayOfMonth} ${Month} ${DayOfWeek}`;
     /**
      * Max number of retries of the cloud scheduler.
      * Note: the task itself is never retried.
      *
      * defaults to 0 (no retries)
+     *
+     *  @deprecated use `run` instead
      */
     maxRetryAttempts?: 0 | 1 | 2 | 3 | 4 | 5;
   };
 
 export type DeployConfigCloudRunJobNormal = DeployConfigCloudRunJobBase & {
-  when: "manual" | "preDeploy" | "postDeploy" | "preStop" | "postStop";
   /**
    * wait for completion of the job on preDeploy and postDeploy
    *
    * has no effect on preStop and postStop (which always wait for completion)
+   *
+   *
+   * @deprecated use `run` instead
    */
   waitForCompletion?: boolean;
+  /**
+   * @deprecated use `run` instead
+   */
+  when?: "manual" | "preDeploy" | "postDeploy" | "preStop" | "postStop";
 };
 
 export type DeployConfigCloudRunWithVolumes = {
@@ -304,6 +323,11 @@ export type DeployConfigCloudRun = {
    */
   cloudSql?: DeployConfigCloudRunCloudSql | false;
 
+  /**
+   * execute jobs on deploy or execute jobs and http requests on a schedule
+   */
+  execute?: Record<string, DeployConfigCloudRunExecute | null>;
+
   debug?: boolean;
 } & DeployConfigBase;
 
@@ -318,3 +342,101 @@ export type DeployConfigCloudRunVolume = {
   mountPath: string;
   readonly?: boolean;
 };
+
+export type DeployConfigCloudRunExecuteJobBase = {
+  /**
+   * set to "job" to run a cloud run job
+   */
+  type: "job";
+  /**
+   * the cloud run job to run
+   */
+  job: string;
+
+  /**
+   * additional args to pass to the job
+   */
+  args?: string[];
+};
+
+type WithSchedule = {
+  /**
+   * run the job on a schedule
+   */
+  when: "schedule";
+  /**
+   * the cron schedule. See https://crontab.guru/
+   */
+  schedule: `${Minute} ${Hour} ${DayOfMonth} ${Month} ${DayOfWeek}`;
+
+  /**
+   * Max number of retries of the cloud scheduler.
+   * Note: the task itself is never retried.
+   *
+   * defaults to 0 (no retries)
+   */
+  maxRetryAttempts?: 0 | 1 | 2 | 3 | 4 | 5;
+};
+
+export type DeployConfigCloudRunExecuteJobScheduled =
+  DeployConfigCloudRunExecuteJobBase & WithSchedule;
+
+export type DeployConfigCloudRunExecuteOnDeploy =
+  DeployConfigCloudRunExecuteJobBase &
+    (
+      | {
+          /**
+           * run the job before or after the deploy
+           */
+          when: "preDeploy" | "postDeploy";
+          /**
+           * whether to wait for completion of the job
+           */
+          waitForCompletion?: boolean;
+        }
+      | {
+          /**
+           * run the job before or after the environment is stopped
+           */
+          when: "preStop" | "postStop";
+        }
+    );
+
+export type DeployConfigCloudRunExecuteJob =
+  | DeployConfigCloudRunExecuteJobScheduled
+  | DeployConfigCloudRunExecuteOnDeploy;
+
+export type DeployConfigCloudRunExecuteHttp = {
+  /**
+   * set to "http" to run a http request
+   */
+  type: "http";
+  /**
+   * the url to call
+   */
+  url: string;
+
+  /**
+   * the http-method to use. Defaults to "POST" (as specified by google cloud scheduler)
+   */
+  method?: "GET" | "POST" | "PUT" | "DELETE" | "PATCH";
+
+  /**
+   * the body to send
+   */
+  body?: string;
+
+  /**
+   * the headers to send
+   */
+  headers?: Record<string, string>;
+} & WithSchedule;
+
+export type DeployConfigCloudRunExecute =
+  | DeployConfigCloudRunExecuteJob
+  | DeployConfigCloudRunExecuteHttp;
+
+export type DeployConfigCloudRunExecuteWithSchedule = Extract<
+  DeployConfigCloudRunExecute,
+  WithSchedule
+>;