@catladder/pipeline 0.0.0-more-collapse-deeb54d7 → 0.0.0-multiple-gitlab-includes-1a40efc0

package/src/deploy/cloudRun/cloudRunRevisions.ts

@@ -0,0 +1,65 @@
+import type { ComponentContext } from "../../types/context";
+import { isOfDeployType } from "../types";
+import { createArgsString } from "./utils/createArgsString";
+import { getServiceName } from "./utils/getServiceName";
+import { removeFirstLinesFromCommandOutput } from "./utils/removeFirstLinesFromCommandOutput";
+
+const getListRevisionsCommand = (
+  context: ComponentContext,
+  args: {
+    filter?: string;
+    format: string;
+  },
+) => {
+  const serviceName = getServiceName(context);
+
+  const deployConfig = context.deploy?.config;
+
+  if (!deployConfig || !isOfDeployType(deployConfig, "google-cloudrun")) {
+    // should not happen
+    throw new Error("deploy config is wrong");
+  }
+  const filterRevisionArgs = {
+    project: deployConfig.projectId,
+    region: deployConfig.region,
+    // only show inactive revisions
+    service: serviceName,
+
+    limit: "unlimited", // is actualy the default
+    "sort-by": "metadata.creationTimestamp",
+    ...args,
+  };
+
+  // this prints out all inactive images of the given app
+  return `gcloud run revisions list ${createArgsString(filterRevisionArgs)}`;
+};
+
+export const getDeleteUnusedRevisionsCommands = (
+  context: ComponentContext,
+  keep: number,
+) => {
+  const deployConfig = context.deploy?.config;
+  if (!isOfDeployType(deployConfig, "google-cloudrun")) {
+    // should not happen
+    throw new Error("deploy config is wrong");
+  }
+
+  // this prints out all inactive images of the given app
+  const listAllInactiveRevisionsCmd = getListRevisionsCommand(context, {
+    format: `"value(name)"`,
+    filter: `'(status.conditions.status=False OR status.conditions.status=Unknown)'`,
+  });
+  // this removes the newest `revisionsToKeep` images
+  const listRevisionsToDeleteCmd = removeFirstLinesFromCommandOutput(
+    listAllInactiveRevisionsCmd,
+    keep,
+  );
+  const deleteRevisionCmd = `gcloud run revisions delete ${createArgsString({
+    project: deployConfig.projectId,
+    region: deployConfig.region,
+    quiet: true,
+  })} $revisionname `;
+
+  const deleteRevisionsCmd = `${listRevisionsToDeleteCmd} | while read -r revisionname; do ${deleteRevisionCmd}; done`;
+  return [deleteRevisionsCmd];
+};

package/src/deploy/cloudRun/createJobs/cloudRunJobs.ts

@@ -0,0 +1,227 @@
+import type { ComponentContext } from "../../../types/context";
+
+import type {
+  DeployConfigCloudRunJob,
+  DeployConfigCloudRunJobWithSchedule,
+} from "../../types/googleCloudRun";
+import { createArgsString } from "../utils/createArgsString";
+import { getCloudRunJobName } from "../utils/jobName";
+import {
+  gcloudRunCmd,
+  gcloudSchedulerCmd,
+  getCloudRunDeployConfig,
+  getCommonCloudRunArgs,
+  getCommonDeployArgs,
+  makeLabelString,
+} from "./common";
+import { getLabels } from "../../../context/getLabels";
+import { createVolumeConfig } from "./volumes";
+import type {
+  BashExpression,
+  StringOrBashExpression,
+} from "../../../bash/BashExpression";
+import { ENV_VARS_FILENAME } from "./constants";
+import { notNil } from "../../../utils";
+
+const getJobRunScriptForJob = (
+  context: ComponentContext,
+  jobName: StringOrBashExpression,
+  wait: boolean,
+) => {
+  const commonArgs = getCommonCloudRunArgs(context);
+
+  const commonArgsString = createArgsString(commonArgs);
+  return `${gcloudRunCmd()} jobs execute ${jobName.toString()} ${commonArgsString}${
+    wait ? " --wait" : ""
+  }`;
+};
+
+export const getDeleteSchedulesScripts = (context: ComponentContext) => {
+  const deployConfig = getCloudRunDeployConfig(context);
+  const jobsWithSchedule = getCloudRunJobsWithSchedule(context);
+  const argsString = createArgsString({
+    project: deployConfig.projectId,
+    location: deployConfig.region,
+  });
+  return jobsWithSchedule
+    .map(({ schedulerName }) => {
+      return [
+        `${gcloudSchedulerCmd()} jobs delete ${schedulerName} ${argsString}`,
+      ];
+    })
+    .flat();
+};
+
+export const getDeleteJobsScripts = (context: ComponentContext) => {
+  const commonArgs = getCommonCloudRunArgs(context);
+  const commonArgsString = createArgsString(commonArgs);
+  const jobsWithNames = getCloudRunJobsWithNames(context);
+
+  return jobsWithNames.flatMap(({ jobName }) => [
+    // first delete all job executions. Otherwise delete might fail if one of those is still running
+    `${gcloudRunCmd()} jobs executions list ${commonArgsString} --job ${jobName} --format="value(name)" | xargs -I {} ${gcloudRunCmd()} jobs executions delete {}  --quiet ${commonArgsString}`,
+    `${gcloudRunCmd()} jobs delete ${jobName} ${commonArgsString}`,
+  ]);
+};
+
+export const getJobRunScripts = (
+  context: ComponentContext,
+  when: DeployConfigCloudRunJob["when"],
+) => {
+  const jobsWithNames = getCloudRunJobsWithNames(context);
+  return jobsWithNames
+    .filter(({ job }) => job.when === when)
+    .map(({ jobName }) =>
+      getJobRunScriptForJob(
+        context,
+        jobName,
+        // wait for completin on stop jobs, since stop will delete the jobs afterwards, so they will fail
+        ["preStop", "postStop"].includes(when),
+      ),
+    );
+};
+
+export const getJobCreateScripts = (context: ComponentContext): string[] =>
+  getCloudRunJobsWithNames(context).map(
+    (
+      {
+        job: {
+          command,
+          image,
+          cpu,
+          memory = "512Mi",
+          timeout = "10m",
+          parallelism = 1,
+          volumes,
+        },
+        jobName,
+      },
+      jobIndex,
+    ): string => {
+      const commandArray = Array.isArray(command)
+        ? command
+        : command.split(" ");
+
+      const {
+        image: commonImage,
+        project,
+        region,
+        ...deployArgs
+      } = getCommonDeployArgs(context);
+      const commonDeployArgsString = createArgsString(
+        {
+          command: `"${commandArray.join(",")}"`,
+          labels: `"${makeLabelString(getLabels(context))},cloud-run-job-name=$current_job_name"`,
+          image: `"${image ?? commonImage}"`,
+          project,
+          region,
+          cpu,
+          memory,
+          parallelism,
+          "task-timeout": timeout,
+          "env-vars-file": ENV_VARS_FILENAME,
+          "max-retries": 0,
+          ...deployArgs,
+        },
+        ...createVolumeConfig(volumes, "job"),
+      );
+
+      return [
+        jobIndex === 0
+          ? `exist_job_names="$(\n  ${gcloudRunCmd()} jobs list --filter='metadata.name ~ ${context.env}.*${context.name}' --format='value(name)' --limit=999 --project='${project}' --region='${region}'\n)"`
+          : null,
+        `current_job_name="${jobName}"`,
+        'if grep "$current_job_name" <<<"$exist_job_names" >/dev/null; then',
+        `  ${gcloudRunCmd()} jobs update "$current_job_name" ${commonDeployArgsString}`,
+        "else",
+        `  ${gcloudRunCmd()} jobs create "$current_job_name" ${commonDeployArgsString}`,
+        "fi",
+      ]
+        .filter(notNil)
+        .join("\n");
+    },
+  );
+
+export const getCreateScheduleScripts = (
+  context: ComponentContext,
+): string[] => {
+  const jobsWithSchedule = getCloudRunJobsWithSchedule(context);
+  const { region: location, projectId: project } =
+    getCloudRunDeployConfig(context);
+
+  const uriBase = `https://${location}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${project}/jobs`;
+  const gcloudArgs = {
+    project,
+    location,
+    uri: `"$current_job_uri"`,
+    "http-method": "POST",
+    "oauth-service-account-email": `"$GCLOUD_PROJECT_NUMBER-compute@developer.gserviceaccount.com"`,
+  };
+
+  return jobsWithSchedule.map(
+    (
+      { job: { maxRetryAttempts, schedule }, jobName, schedulerName },
+      jobIndex,
+    ): string => {
+      const argsString = createArgsString({
+        ...gcloudArgs,
+        schedule: `"${schedule}"`,
+        "max-retry-attempts": maxRetryAttempts ?? 0,
+      });
+      return [
+        jobIndex === 0
+          ? `exist_scheduler_names="$(\n  ${gcloudSchedulerCmd()} jobs list --filter='httpTarget.uri ~ ${context.env}.*${context.name}' --format='value(name)' --limit=999 --location='${location}' --project='${project}'\n)"`
+          : null,
+        `current_job_uri="${uriBase}/${jobName}:run"`,
+        `current_scheduler_name="${schedulerName}"`,
+        `if grep "$current_scheduler_name" <<<"$exist_scheduler_names" >/dev/null; then`,
+        `  ${gcloudSchedulerCmd()} jobs update http "$current_scheduler_name" ${argsString}`,
+        `else`,
+        `  ${gcloudSchedulerCmd()} jobs create http "$current_scheduler_name" ${argsString}`,
+        `fi`,
+      ]
+        .filter(notNil)
+        .join("\n");
+    },
+  );
+};
+
+const getCloudRunJobsWithSchedule = (context: ComponentContext) => {
+  const jobsWithNames = getCloudRunJobsWithNames(context);
+
+  return jobsWithNames
+    .filter(
+      (
+        entry,
+      ): entry is {
+        jobName: BashExpression;
+        job: DeployConfigCloudRunJobWithSchedule;
+        jobKey: string;
+      } => entry.job.when === "schedule",
+    )
+    .map(({ job, jobName, jobKey }) => ({
+      job,
+      jobName,
+      jobKey,
+      schedulerName: jobName.concat("-scheduler"),
+    }));
+};
+
+const getCloudRunJobsWithNames = (context: ComponentContext) => {
+  const deployConfig = getCloudRunDeployConfig(context);
+
+  const getFullJobName = (name: string) =>
+    getCloudRunJobName(context.environment.fullName, name);
+
+  const jobsWithNames = Object.entries(deployConfig.jobs ?? {})
+    // filter out disabled jobs
+    .filter((entry): entry is [string, DeployConfigCloudRunJob] =>
+      Boolean(entry[1]),
+    )
+    .map(([jobKey, job]) => ({
+      jobName: getFullJobName(jobKey),
+      job,
+      jobKey,
+    }));
+  return jobsWithNames;
+};

package/src/deploy/cloudRun/createJobs/cloudRunServices.ts

@@ -0,0 +1,86 @@
+import { getLabels } from "../../../context/getLabels";
+import type { ComponentContext } from "../../../types/context";
+
+import type { DeployConfigCloudRunService } from "../../types/googleCloudRun";
+import { createArgsString } from "../utils/createArgsString";
+import { getServiceName } from "../utils/getServiceName";
+import {
+  gcloudRunCmd,
+  getCommonCloudRunArgs,
+  getCommonDeployArgs,
+  makeLabelString,
+} from "./common";
+import { ENV_VARS_FILENAME } from "./constants";
+import { createVolumeConfig } from "./volumes";
+
+export const getServiceDeployScript = (
+  context: ComponentContext,
+  service: DeployConfigCloudRunService | true | undefined,
+  nameSuffix?: string,
+) => {
+  const commonDeployArgs = getCommonDeployArgs(context);
+
+  const serviceName = getServiceName(context);
+
+  const customConfig = service !== true ? service : undefined;
+  const command =
+    service !== true
+      ? (service?.command ?? context.build.config.startCommand)
+      : undefined;
+
+  const commandArray = command
+    ? Array.isArray(command)
+      ? command
+      : command.split(" ")
+    : undefined;
+  const fullServiceName = serviceName.concat(nameSuffix ?? "");
+  const argsString = createArgsString(
+    {
+      // command as empty string resets it to default (uses the image's entrypoint)
+      command: commandArray ? '"' + commandArray.join(",") + '"' : '""',
+      ...commonDeployArgs,
+      labels: makeLabelString({
+        ...getLabels(context),
+        "cloud-run-service-name": fullServiceName,
+      }),
+      "env-vars-file": ENV_VARS_FILENAME,
+      "min-instances": customConfig?.minInstances ?? 0,
+      "max-instances": customConfig?.maxInstances ?? 100,
+      "cpu-throttling": customConfig?.noCpuThrottling !== true,
+      cpu: customConfig?.cpu,
+      memory: customConfig?.memory,
+      timeout: customConfig?.timeout,
+      "vpc-connector": customConfig?.vpcConnector,
+      "vpc-egress": customConfig?.vpcEgress,
+      network: customConfig?.network,
+      subnet: customConfig?.subnet,
+      "use-http2": customConfig?.http2,
+      "allow-unauthenticated": customConfig?.allowUnauthenticated ?? true,
+      ingress: customConfig?.ingress ?? "all",
+      "cpu-boost": true,
+      "execution-environment": customConfig?.executionEnvironment,
+    },
+    ...createVolumeConfig(customConfig?.volumes, "service"),
+  );
+
+  return `${gcloudRunCmd()} deploy ${fullServiceName} ${argsString}`;
+};
+
+export const getServiceDeleteScript = (
+  context: ComponentContext,
+  serviceSuffix?: string,
+) => {
+  const commonArgs = getCommonCloudRunArgs(context);
+
+  const commonArgsString = createArgsString(commonArgs);
+
+  const serviceName = getServiceName(context);
+
+  const fullServiceName = serviceSuffix
+    ? `${serviceName}-${serviceSuffix}`
+    : serviceName;
+
+  return [
+    `${gcloudRunCmd()} services delete ${fullServiceName} ${commonArgsString}`,
+  ];
+};

package/src/deploy/cloudRun/createJobs/common.ts

@@ -0,0 +1,59 @@
+import type { ComponentContext } from "../../../types/context";
+
+import { isOfDeployType } from "../../types";
+import type { DeployConfigCloudRun } from "../../types/googleCloudRun";
+import { getArtifactsRegistryImage } from "../artifactsRegistry";
+
+export const gcloudCmd = (version?: "beta") => {
+  return version ? `gcloud ${version}` : "gcloud";
+};
+export const gcloudRunCmd = (version?: "beta") => {
+  return `${gcloudCmd(version)} run`;
+};
+
+export const gcloudSchedulerCmd = (version?: "beta") => {
+  return `${gcloudCmd(version)} scheduler`;
+};
+
+export const setGoogleProjectNumberScript = (
+  deployConfig: DeployConfigCloudRun,
+) => [
+  `export GCLOUD_PROJECT_NUMBER=$(${gcloudCmd()} projects describe ${
+    deployConfig.projectId
+  } --format="value(projectNumber)")`,
+  'echo "GCLOUD_PROJECT_NUMBER: $GCLOUD_PROJECT_NUMBER"',
+];
+
+export const makeLabelString = (obj: Record<string, unknown>) =>
+  Object.entries(obj)
+    .map(([key, value]) => `${key}=${value}`)
+    .join(",");
+
+export const getCloudRunDeployConfig = (context: ComponentContext) => {
+  const deployConfig = context.deploy?.config;
+  if (!isOfDeployType(deployConfig, "google-cloudrun")) {
+    // should not happen
+    throw new Error("deploy config is wrong");
+  }
+  return deployConfig;
+};
+
+export function getCommonCloudRunArgs(context: ComponentContext) {
+  const deployConfig = getCloudRunDeployConfig(context);
+  return {
+    project: deployConfig.projectId,
+    region: deployConfig.region,
+  };
+}
+
+export function getCommonDeployArgs(context: ComponentContext) {
+  const commonArgs = getCommonCloudRunArgs(context);
+  const deployConfig = getCloudRunDeployConfig(context);
+  return {
+    image: getArtifactsRegistryImage(context),
+    ...commonArgs,
+    "set-cloudsql-instances": deployConfig.cloudSql
+      ? deployConfig.cloudSql.instanceConnectionName
+      : undefined,
+  };
+}

package/src/deploy/cloudRun/createJobs/constants.ts

@@ -0,0 +1 @@
+export const ENV_VARS_FILENAME = "____envvars.yaml";

package/src/deploy/cloudRun/createJobs/getCloudRunDeployScripts.ts

@@ -0,0 +1,71 @@
+import { omit } from "lodash";
+import type { ComponentContext } from "../../../types/context";
+import { collapseableSection } from "../../../utils/gitlab";
+import { getDependencyTrackUploadScript } from "../../sbom";
+
+import { GCLOUD_DEPLOY_CREDENTIALS_KEY } from "..";
+import { writeBashYamlToFileScript } from "../../../bash/bashYaml";
+import { getRemoveOldRevisionsAndImagesCommand } from "../cleanup";
+import { getDatabaseCreateScript } from "../utils/database";
+import { gcloudServiceAccountLoginCommands } from "../utils/gcloudServiceAccountLoginCommands";
+import {
+  getCreateScheduleScripts,
+  getJobCreateScripts,
+  getJobRunScripts,
+} from "./cloudRunJobs";
+import { getServiceDeployScript } from "./cloudRunServices";
+import {
+  getCloudRunDeployConfig,
+  setGoogleProjectNumberScript,
+} from "./common";
+import { ENV_VARS_FILENAME } from "./constants";
+
+export function getCloudRunDeployScripts(context: ComponentContext) {
+  const deployConfig = getCloudRunDeployConfig(context);
+  const allEnvVars = omit(
+    context.environment.envVars,
+    GCLOUD_DEPLOY_CREDENTIALS_KEY,
+  );
+
+  return [
+    ...collapseableSection(
+      "prepare",
+      "Prepare...",
+    )([
+      ...gcloudServiceAccountLoginCommands(context),
+      ...setGoogleProjectNumberScript(deployConfig),
+    ]),
+    ...collapseableSection(
+      "writeenvvars",
+      "Write env vars to file",
+    )(writeBashYamlToFileScript(allEnvVars, ENV_VARS_FILENAME)),
+
+    ...collapseableSection(
+      "deploy",
+      "Deploy to cloud run",
+    )([
+      ...(deployConfig.cloudSql
+        ? getDatabaseCreateScript(context, deployConfig) // we create the db, so that we can also delete it afterwards
+        : []),
+      ...getCreateScheduleScripts(context),
+      ...getJobCreateScripts(context),
+      ...getJobRunScripts(context, "preDeploy"),
+
+      ...(deployConfig.service !== false
+        ? [getServiceDeployScript(context, deployConfig.service)]
+        : []),
+      ...Object.entries(deployConfig.additionalServices ?? {}).map(
+        ([name, service]) =>
+          getServiceDeployScript(context, service, "-" + name),
+      ),
+      ...getJobRunScripts(context, "postDeploy"),
+    ]),
+    ...collapseableSection(
+      "cleanup",
+      "Cleanup",
+    )(
+      getRemoveOldRevisionsAndImagesCommand(context, "postDeploy"), // we cleanup inactive images both on deploy and stop
+    ),
+    ...getDependencyTrackUploadScript(context),
+  ];
+}

package/src/deploy/cloudRun/createJobs/getCloudRunStopScripts.ts

@@ -0,0 +1,34 @@
+import type { ComponentContext } from "../../../types/context";
+import { getDependencyTrackDeleteScript } from "../../sbom";
+
+import { getRemoveOldRevisionsAndImagesCommand } from "../cleanup";
+import { getDatabaseDeleteScript } from "../utils/database";
+import { gcloudServiceAccountLoginCommands } from "../utils/gcloudServiceAccountLoginCommands";
+import {
+  getDeleteJobsScripts,
+  getDeleteSchedulesScripts,
+  getJobRunScripts,
+} from "./cloudRunJobs";
+import { getServiceDeleteScript } from "./cloudRunServices";
+import { getCloudRunDeployConfig } from "./common";
+
+export function getCloudRunStopScripts(context: ComponentContext) {
+  const deployConfig = getCloudRunDeployConfig(context);
+  return [
+    ...gcloudServiceAccountLoginCommands(context),
+    ...getJobRunScripts(context, "preStop"),
+    ...(deployConfig.service !== false ? getServiceDeleteScript(context) : []),
+    ...Object.entries(deployConfig.additionalServices ?? {}).flatMap(([name]) =>
+      getServiceDeleteScript(context, name),
+    ),
+    ...getJobRunScripts(context, "postStop"),
+    ...getDeleteSchedulesScripts(context),
+    ...getDeleteJobsScripts(context),
+    ...(deployConfig.cloudSql && deployConfig.cloudSql.deleteDatabaseOnStop
+      ? getDatabaseDeleteScript(context, deployConfig)
+      : []),
+
+    ...getRemoveOldRevisionsAndImagesCommand(context, "onStop"), // we cleanup inactive images both on deploy and stop
+    ...getDependencyTrackDeleteScript(context),
+  ];
+}

package/src/deploy/cloudRun/createJobs/index.ts

@@ -0,0 +1,51 @@
+import { getRunnerImage } from "../../../runner";
+import type { ComponentContext } from "../../../types/context";
+import type { CatladderJob } from "../../../types/jobs";
+import { allowFailureInScripts } from "../../../utils/gitlab";
+import { createDeployementJobs } from "../../base";
+
+import { isOfDeployType } from "../../types";
+import { ENV_VARS_FILENAME } from "./constants";
+import { getCloudRunDeployScripts } from "./getCloudRunDeployScripts";
+import { getCloudRunStopScripts } from "./getCloudRunStopScripts";
+
+export const createGoogleCloudRunDeployJobs = (
+  context: ComponentContext,
+): CatladderJob[] => {
+  const deployConfig = context.deploy?.config;
+  if (!isOfDeployType(deployConfig, "google-cloudrun")) {
+    // should not happen
+    throw new Error("deploy config is wrong");
+  }
+
+  const deployScripts = getCloudRunDeployScripts(context);
+  const stopScripts = getCloudRunStopScripts(context);
+
+  return createDeployementJobs(context, {
+    deploy: {
+      variables: {
+        CLOUDSDK_CORE_DISABLE_PROMPTS: "1",
+      },
+      ...(deployConfig.debug
+        ? {
+            artifacts: {
+              paths: [
+                ENV_VARS_FILENAME, // debug
+              ],
+              when: "always",
+            },
+          }
+        : {}),
+
+      image: getRunnerImage("gcloud"),
+      script: deployScripts,
+    },
+    stop: {
+      image: getRunnerImage("gcloud"),
+      variables: {
+        CLOUDSDK_CORE_DISABLE_PROMPTS: "1",
+      },
+      script: allowFailureInScripts(stopScripts),
+    },
+  });
+};

package/src/deploy/cloudRun/createJobs/volumes.ts

@@ -0,0 +1,23 @@
+import type { DeployConfigCloudRunVolumes } from "../../types";
+import type { keyValuesArg } from "../utils/createArgsString";
+
+export const createVolumeConfig = (
+  volumes: DeployConfigCloudRunVolumes | undefined,
+  type: "service" | "job",
+): keyValuesArg[] => {
+  if (!volumes) {
+    return [];
+  }
+
+  return [
+    ...Object.entries(volumes).map(
+      ([volumeName, { type, bucket, mountPath, readonly }]) => ({
+        "add-volume": `name=${volumeName},type=${type},bucket=${bucket}${
+          readonly ? ",readonly=true" : ""
+        }`,
+        "add-volume-mount": `volume=${volumeName},mount-path=${mountPath}`,
+      }),
+    ),
+    type === "service" ? { "execution-environment": "gen2" } : {},
+  ];
+};