@catladder/pipeline 0.0.0-refactor-untangle-gitlab-api-requests-from-vorpal-57cf3cf2 → 0.0.0-reference-database-url-1befbbee

package/src/deploy/cloudRun/createJobs/execute/onDeploy.ts

@@ -3,8 +3,10 @@ import type { ComponentContext } from "../../../../types/context";
 import type {
   DeployConfigCloudRunExecuteOnDeploy,
   DeployConfigCloudRunJob,
+  DeployConfigCloudRunService,
 } from "../../../types/googleCloudRun";
 import { createArgsString } from "../../utils/createArgsString";
+import { getCloudRunServiceOrJobArgsArg } from "../../utils/getJobOrServiceArgs";
 import { getFullJobName } from "../../utils/jobName";
 import { getCloudRunJobsWithNames } from "../cloudRunJobs";
 import {
@@ -93,20 +95,8 @@ const getJobRunScriptForExecute = (
   const argString = createArgsString({
     ...commonArgs,
     wait: waitForCompletion === true ? true : undefined,
-    args: getCloudRunJobArgsArg(config.args),
+    args: getCloudRunServiceOrJobArgsArg(config.args),
   });
   const fullJobName = getFullJobName(context, jobName);
   return `${gcloudRunCmd()} jobs execute ${fullJobName.toString()} ${argString}`;
 };
-
-export const getCloudRunJobArgsArg = (
-  args:
-    | DeployConfigCloudRunExecuteOnDeploy["args"]
-    | DeployConfigCloudRunJob["args"],
-) => {
-  return args !== undefined
-    ? args.length > 0
-      ? args?.map((arg) => `"${arg}"`).join(",")
-      : ""
-    : undefined;
-};

package/src/deploy/cloudRun/createJobs/healthCheck.ts

@@ -0,0 +1,64 @@
+import type { DeployConfigCloudRunProbeShared } from "../../types";
+import {
+  defaultLivenessProbe,
+  defaultStartupProbe,
+  type DeployConfigCloudRunProbe,
+  type DeployConfigCloudRunService,
+} from "../../types";
+
+const PORT = 8080;
+
+export function healthCheckCliArgs(
+  healthCheck: DeployConfigCloudRunService["healthCheck"],
+) {
+  const shouldNotSetHealthCheck = healthCheck === undefined;
+  if (shouldNotSetHealthCheck) {
+    return undefined;
+  }
+
+  const shouldUseDefaultProbe = healthCheck === true;
+
+  if (shouldUseDefaultProbe) {
+    return {
+      "startup-probe": toArgValue(probeToKeyValuePairs(defaultStartupProbe)),
+      "liveness-probe": toArgValue(probeToKeyValuePairs(defaultLivenessProbe)),
+    };
+  }
+
+  return {
+    "startup-probe": toArgValue(probeToKeyValuePairs(healthCheck.startupProbe)),
+    "liveness-probe": healthCheck.livenessProbe
+      ? toArgValue(probeToKeyValuePairs(healthCheck.livenessProbe))
+      : "", // NOTE: empty string ("") removes liveness probe
+  };
+}
+
+function probeToKeyValuePairs(
+  probe: DeployConfigCloudRunProbe,
+): [string, unknown][] {
+  const sharedArgs = Object.entries({
+    initialDelaySeconds: probe.initialDelaySeconds,
+    timeoutSeconds: probe.timeoutSeconds,
+    periodSeconds: probe.periodSeconds,
+    failureThreshold: probe.failureThreshold,
+  } satisfies DeployConfigCloudRunProbeShared);
+
+  switch (probe.type) {
+    case "tcp":
+      return [...sharedArgs, ["tcpSocket.port", PORT]];
+    case "http1":
+      return [
+        ...sharedArgs,
+        ["httpGet.port", PORT],
+        ["httpGet.path", probe.path],
+        // NOTE: headers are not supported by "gcloud beta" at the moment
+      ];
+    default:
+      probe satisfies never;
+      return [];
+  }
+}
+
+function toArgValue(keyValues: [string, unknown][]) {
+  return keyValues.map(([key, value]) => `${key}=${value}`).join(",");
+}

package/src/deploy/cloudRun/index.ts

@@ -11,8 +11,9 @@ import { getFullDbName } from "../cloudSql/utils";
 import { createGoogleCloudRunDeployJobs } from "./createJobs";
 import { getCloudRunJobExecuteUrl } from "./utils/cloudRunExecutionUrl";
 import {
-  DATABASE_JDBC_URL,
+  DEFAULT_DB_VARIABLES_MODE,
   getDatabaseConnectionString,
+  getDatabaseJdbcUrl,
 } from "./utils/database";
 
 export const GCLOUD_DEPLOY_CREDENTIALS_KEY = "GCLOUD_DEPLOY_credentialsKey";
@@ -42,16 +43,24 @@ const getCloudSqlVariables = ({
       .map(([key, value]) => `&${key}=${value}`)
       .join("");
 
-    return {
+    const dbVars = {
       CLOUD_SQL_INSTANCE_CONNECTION_NAME:
         deployConfigRaw.cloudSql.instanceConnectionName,
       DB_NAME: DB_NAME,
       DB_USER: deployConfigRaw.cloudSql.dbUser ?? "postgres",
       DB_PASSWORD: "$" + getSecretVarName(env, componentName, "DB_PASSWORD"),
-      DATABASE_URL: `${getDatabaseConnectionString(
+    };
+    return {
+      ...dbVars,
+      DATABASE_URL: getDatabaseConnectionString(
         deployConfigRaw.cloudSql,
-      )}${additionalQueryParamsString}`,
-      DATABASE_JDBC_URL: DATABASE_JDBC_URL,
+        dbVars,
+      ).concat(additionalQueryParamsString),
+      DATABASE_JDBC_URL: getDatabaseJdbcUrl(
+        dbVars,
+        deployConfigRaw.cloudSql.dbConnectionStringVariablesMode ??
+          DEFAULT_DB_VARIABLES_MODE,
+      ),
     };
   }
   return {};

package/src/deploy/cloudRun/utils/database.ts

@@ -1,3 +1,8 @@
+import {
+  BashExpression,
+  joinBashExpressions,
+  type StringOrBashExpression,
+} from "../../../bash";
 import type { ComponentContext } from "../../../types";
 import { allowFailureInScripts, repeatOnFailure } from "../../../utils/gitlab";
 import type {
@@ -57,21 +62,101 @@ export const getDatabaseCreateScript = (
   ]);
 };
 
-export const DATABASE_JDBC_URL =
-  "jdbc:postgresql:///$DB_NAME?cloudSqlInstance=$CLOUD_SQL_INSTANCE_CONNECTION_NAME&socketFactory=com.google.cloud.sql.postgres.SocketFactory&user=$DB_USER&password=$DB_PASSWORD";
+export type DBVariables = {
+  CLOUD_SQL_INSTANCE_CONNECTION_NAME: StringOrBashExpression;
+  DB_NAME: StringOrBashExpression;
+  DB_USER: StringOrBashExpression;
+  DB_PASSWORD: StringOrBashExpression;
+};
+
+/**
+ * controls how variables in the connection string are handled
+ *
+ * - legacy: variables like $DB_USER will be kept as environment variables to be replaced at runtime (default). It is not using BashExpressions as this was not the case in the past.
+ * - embedded: variables will be replaced with their actual values in the connection string
+ *
+ * We will remove the legacy mode in the future, as it is confusing. But its unclear whether its a breaking change in some edge cases
+ */
+export type DBVariablesMode = "legacy" | "embedded";
+
+export const DEFAULT_DB_VARIABLES_MODE: DBVariablesMode = "legacy";
+
+const getVariableOrValue = (
+  key: keyof DBVariables,
+  variables: DBVariables,
+  mode: DBVariablesMode,
+): StringOrBashExpression => {
+  return mode === "legacy" ? `$${key}` : variables[key];
+};
+
+export const getDatabaseJdbcUrl = (
+  variables: DBVariables,
+  mode: DBVariablesMode,
+) => {
+  const parts = [
+    "jdbc:postgresql:///",
+    getVariableOrValue("DB_NAME", variables, mode),
+    "?cloudSqlInstance=",
+    getVariableOrValue("CLOUD_SQL_INSTANCE_CONNECTION_NAME", variables, mode),
+    "&socketFactory=com.google.cloud.sql.postgres.SocketFactory&user=",
+    getVariableOrValue("DB_USER", variables, mode),
+    "&password=",
+    getVariableOrValue("DB_PASSWORD", variables, mode),
+  ];
+
+  return joinBashExpressions(parts);
+};
+
+export const getRailsDatabaseConnectionString = (
+  variables: DBVariables,
+  mode: DBVariablesMode,
+) => {
+  const parts = [
+    "postgresql://",
+    getVariableOrValue("DB_USER", variables, mode),
+    ":",
+    getVariableOrValue("DB_PASSWORD", variables, mode),
+    "@",
+    encodeURIComponent(
+      `/cloudsql/${variables.CLOUD_SQL_INSTANCE_CONNECTION_NAME}`,
+    ),
+    "/",
+    getVariableOrValue("DB_NAME", variables, mode),
+    "?",
+  ];
+  return joinBashExpressions(parts);
+};
+
+export const getPrismaDatabaseConnectionString = (
+  variables: DBVariables,
+  mode: DBVariablesMode,
+) => {
+  const parts = [
+    "postgresql://",
+    getVariableOrValue("DB_USER", variables, mode),
+    ":",
+    getVariableOrValue("DB_PASSWORD", variables, mode),
+    "@localhost/",
+    getVariableOrValue("DB_NAME", variables, mode),
+    "?host=/cloudsql/",
+    getVariableOrValue("CLOUD_SQL_INSTANCE_CONNECTION_NAME", variables, mode),
+  ];
+  return joinBashExpressions(parts);
+};
 
 export const getDatabaseConnectionString = (
   config: DeployConfigCloudRunCloudSql,
-): string => {
+  variables: DBVariables,
+): StringOrBashExpression => {
+  const mode =
+    config.dbConnectionStringVariablesMode ?? DEFAULT_DB_VARIABLES_MODE;
   switch (config.dbConnectionStringFormat) {
     case "jdbc":
-      return DATABASE_JDBC_URL;
+      return getDatabaseJdbcUrl(variables, mode);
     case "rails":
-      return `postgresql://$DB_USER:$DB_PASSWORD@${encodeURIComponent(
-        `/cloudsql/${config.instanceConnectionName}`,
-      )}/$DB_NAME?`;
+      return getRailsDatabaseConnectionString(variables, mode);
     default:
       // prisma
-      return "postgresql://$DB_USER:$DB_PASSWORD@localhost/$DB_NAME?host=/cloudsql/$CLOUD_SQL_INSTANCE_CONNECTION_NAME";
+      return getPrismaDatabaseConnectionString(variables, mode);
   }
 };

package/src/deploy/cloudRun/utils/getJobOrServiceArgs.ts

@@ -0,0 +1,18 @@
+import type {
+  DeployConfigCloudRunExecuteOnDeploy,
+  DeployConfigCloudRunJob,
+  DeployConfigCloudRunService,
+} from "../../types";
+
+export const getCloudRunServiceOrJobArgsArg = (
+  args:
+    | DeployConfigCloudRunExecuteOnDeploy["args"]
+    | DeployConfigCloudRunJob["args"]
+    | DeployConfigCloudRunService["args"],
+) => {
+  return args !== undefined
+    ? args.length > 0
+      ? args?.map((arg) => `"${arg}"`).join(",")
+      : ""
+    : undefined;
+};

package/src/deploy/kubernetes/kubeValues.ts

@@ -1,6 +1,9 @@
 import { merge } from "lodash";
 
-import type { ComponentContext } from "../../types/context";
+import type {
+  BuildContextComponent,
+  ComponentContext,
+} from "../../types/context";
 import { mergeWithMergingArrays } from "../../utils";
 import type { DeployConfigKubernetesValues } from "../types";
 import { isOfDeployType } from "../types";
@@ -13,7 +16,7 @@ import { createKubeEnv } from "./kubeEnv";
 import { createMongodbBaseConfig } from "./mongodb";
 
 const createAppConfig = (
-  context: ComponentContext,
+  context: ComponentContext<BuildContextComponent>,
   application: DeployConfigKubernetesValues["application"],
 ): DeployConfigKubernetesValues["application"] => {
   if (application === false) {
@@ -27,7 +30,11 @@ const createAppConfig = (
   return mergeWithMergingArrays(
     {
       host: context.environment.envVars.HOSTNAME,
-      command: command ?? context.build.config.startCommand,
+      command:
+        command ??
+        (context.build.type !== "disabled"
+          ? context.build.config.startCommand
+          : undefined),
       livenessProbe: {
         httpGet: {
           path: healthRoute ?? "__health",
@@ -55,7 +62,9 @@ const removeFalsy = <T>(record?: Record<string, false | T>) => {
   );
 };
 
-export const createKubeValues = (context: ComponentContext) => {
+export const createKubeValues = (
+  context: ComponentContext<BuildContextComponent>,
+) => {
   const deployConfig = context.deploy?.config;
   if (!deployConfig) {
     return [];

package/src/deploy/sbom.ts

@@ -6,9 +6,10 @@ import {
 } from "../types/context";
 
 export const sbomDeactivated = (context: ComponentContext) =>
-  componentContextIsStandaloneBuild(context) &&
-  context.build.config.type === "custom" &&
-  context.build.config.sbom === false;
+  context.build.type === "disabled" ||
+  (componentContextIsStandaloneBuild(context) &&
+    context.build.config.type === "custom" &&
+    context.build.config.sbom === false);
 
 export const getDependencyTrackUploadScript = (
   context: ComponentContext,

package/src/deploy/types/googleCloudRun.ts

@@ -1,3 +1,4 @@
+import type { DBVariablesMode } from "../cloudRun/utils/database";
 import type { DeployConfigBase } from "./base";
 
 export type Gcloudregion =
@@ -77,6 +78,16 @@ export type DeployConfigCloudRunCloudSql = {
    */
   dbConnectionStringFormat?: "prisma" | "rails" | "jdbc";
 
+  /**
+   * controls how variables in the connection string are handled
+   *
+   * - environment: variables like $DB_USER will be kept as environment variables to be replaced at runtime (default)
+   * - embedded: variables will be replaced with their actual values in the connection string
+   *
+   * @default "environment"
+   */
+  dbConnectionStringVariablesMode?: DBVariablesMode;
+
   /**
    * add additional query params to the database connection string
    */
@@ -168,9 +179,133 @@ export type DeployConfigCloudRunService = {
    *
    */
   http2?: boolean;
+
+  /**
+   * Number of GPUs to use. Defaults to 0
+   *
+   * This is a preview feature and not all regions support it.
+   */
+  gpu?: number;
+
+  /**
+   * gpu type to use. Refer to https://cloud.google.com/run/docs/configuring/services/gpu#gcloud for defaults
+   */
+  gpuType?: string;
+
+  /**
+   * the image to use. Defaults to the image from the build.
+   *
+   * If you specify an image, you usually want to disable the build by setting build: false
+   */
+  image?: string;
+  /**
+   * args to pass to the command
+   */
+  args?: string[];
+
+  /**
+   * Configuration of the health check.
+   *
+   * When set to `true`, startup and liveness probes will be
+   * configured to `defaultStartupProbe` and `defaultLivenessProbe`.
+   *
+   * When not set, default Cloud Run's health check
+   * configuration is used (startup TCP probe and no liveness probe).
+   */
+  healthCheck?:
+    | true
+    | {
+        /**
+         * Startup probe configuration.
+         */
+        startupProbe: DeployConfigCloudRunProbe;
+
+        /**
+         * Liveness probe configuration.
+         * `false` disables the liveness probe.
+         */
+        livenessProbe: false | DeployConfigCloudRunProbe;
+      };
 } & DeployConfigCloudRunWithVolumes &
   DeployConfigCloudRunNetworkConfig;
 
+export const defaultStartupProbe: DeployConfigCloudRunProbe = {
+  type: "http1",
+  path: "/__health",
+
+  initialDelaySeconds: 5,
+  timeoutSeconds: 1,
+  periodSeconds: 30,
+  failureThreshold: 3,
+};
+
+export const defaultLivenessProbe: DeployConfigCloudRunProbe = {
+  ...defaultStartupProbe,
+
+  initialDelaySeconds: 0,
+  timeoutSeconds: 10,
+  periodSeconds: 30,
+  failureThreshold: 3,
+};
+
+export type DeployConfigCloudRunProbe =
+  | DeployConfigCloudRunHttp1Probe
+  | DeployConfigCloudRunTcpProbe;
+
+export type DeployConfigCloudRunProbeShared = {
+  /**
+   * The period in seconds at which to perform the probe.
+   * For example 2 to perform the probe every 2 seconds.
+   * Specify a value **from 1 second to 240** seconds.
+   */
+  periodSeconds: number;
+
+  /**
+   * The number of seconds to wait after the container has started before performing the first probe.
+   * Specify a value **from 0 seconds to 240** seconds.
+   */
+  initialDelaySeconds: number;
+
+  /**
+   * The number of seconds to wait until the probe times out.
+   * This value cannot exceed the value specified for `period`.
+   * Specify a value **from 1 to 240**.
+   */
+  timeoutSeconds: number;
+
+  /**
+   * The number of times to retry the probe before shutting down the container.
+   */
+  failureThreshold: number;
+};
+
+export type DeployConfigCloudRunTcpProbe = DeployConfigCloudRunProbeShared & {
+  /**
+   * Cloud Run makes a TCP connection to open the TCP Socket on the specified port.
+   * If Cloud Run is unable to establish a connection, it indicates a failure.
+   */
+  type: "tcp";
+};
+
+export type DeployConfigCloudRunHttp1Probe = DeployConfigCloudRunProbeShared & {
+  /**
+   * **This will NOT work with http2 enabled services**
+   *
+   * Cloud Run makes an HTTP GET request to the service health check endpoint (for example, /ready).
+   * Any response between 200 and 400 is a success, everything else indicates failure.
+   *
+   * If a startup probe does not succeed within the specified time (failureThreshold * period), which cannot exceed 240 seconds, the container is shut down.
+   *
+   * If the HTTP startup probe succeeds within the specified time, and you have enabled liveness probe, the HTTP liveness probe is started.
+   */
+  type: "http1";
+
+  /**
+   * the route path to check for health status of the service starting with a `/`.
+   */
+  path: string;
+};
+
 export type DeployConfigCloudRunNetworkConfig = {
   /* the vpc network, see https://cloud.google.com/sdk/gcloud/reference/run/deploy#--network  */
   network?: string;

package/src/pipeline/createJobsForComponent.ts

@@ -1,6 +1,9 @@
 import { BUILD_TYPES } from "../build";
 import { DEPLOY_TYPES } from "../deploy";
-import type { ComponentContext } from "../types/context";
+import type {
+  ComponentContext,
+  ComponentContextWithBuild,
+} from "../types/context";
 import type { CatladderJob } from "../types/jobs";
 
 const injectDefaultVarsInCustomJobs = (
@@ -24,7 +27,12 @@ const getCustomJobs = (context: ComponentContext) => {
 export const createJobsForComponentContext = (
   context: ComponentContext,
 ): CatladderJob[] => {
-  const buildJobs = BUILD_TYPES[context.build.buildType].jobs(context);
+  const buildJobs =
+    context.build.type !== "disabled"
+      ? BUILD_TYPES[context.build.buildType].jobs(
+          context as ComponentContextWithBuild,
+        )
+      : [];
   const deployJobs =
     context.componentConfig.deploy !== false
       ? DEPLOY_TYPES[context.componentConfig.deploy.type].jobs(context)

package/src/types/config.ts

@@ -83,7 +83,7 @@ export type DefaultEnvConfig = {
   /**
    * how the app is built and its runtime
    */
-  build: BuildConfig;
+  build: BuildConfig | false;
   /**
    * environment variables
    */

package/src/types/context.ts

@@ -116,6 +116,9 @@ export type BuildContextStandalone<
   buildType: C["type"];
 };
 
+export type BuildContextDisabled = BuildContextComponentBase & {
+  type: "disabled";
+};
 export type BuildContextFromWorkspace = BuildContextComponentBase & {
   config: BuildConfigFromWorkspace;
   workspaceName: string;
@@ -133,10 +136,14 @@ export type BuildContextWorkspace = BuildContextBase & {
   config: WorkspaceBuildConfig;
 };
 
-export type BuildContextComponent =
+export type BuildContextWithBuild =
   | BuildContextStandalone
   | BuildContextFromWorkspace;
 
+export type BuildContextComponent =
+  | BuildContextWithBuild
+  | BuildContextDisabled;
+
 export type BuildContext = BuildContextComponent | BuildContextWorkspace;
 
 export type DeployContext = {
@@ -172,6 +179,8 @@ export type ComponentContext<
   customJobs?: CatladderJob[];
 };
 
+export type ComponentContextWithBuild = ComponentContext<BuildContextWithBuild>;
+
 export type Context = ComponentContext | WorkspaceContext;
 
 export type CatladderJobWithContext<S = BaseStage> = CatladderJob<S> & {