@catladder/pipeline 3.9.2 → 3.10.0

package/examples/__snapshots__/rails-k8s-with-worker-dockerfile.test.ts.snap

@@ -137,7 +137,7 @@ app 🧪 test:
     - cd .temp-with-dockerfile
     - bundle config set path 'tmp/cache'
     - bundle install -j $(nproc)
-    - bundle exec rake db:setup
+    - bundle exec rake db:test:prepare
     - bundle exec rake assets:precompile assets:clean
     - bundle exec rspec
   cache: *a2

package/examples/__snapshots__/rails-k8s-with-worker.test.ts.snap

@@ -137,7 +137,7 @@ app 🧪 test:
     - cd .
     - bundle config set path 'tmp/cache'
     - bundle install -j $(nproc)
-    - bundle exec rake db:setup
+    - bundle exec rake db:test:prepare
     - bundle exec rake assets:precompile assets:clean
     - bundle exec rspec
   cache: *a2

package/examples/cloud-run-with-sql-multiple-dbs.test.ts

@@ -0,0 +1,11 @@
+import { createYamlLocalPipeline } from "./__utils__/helpers";
+import config from "./cloud-run-with-sql-multiple-dbs";
+
+/**
+ * This test is auto-generated.
+ * Modifications will be overwritten on every `yarn test` run!
+ */
+
+it("matches snapshot for cloud-run-with-sql-multiple-dbs local pipeline YAML", async () => {
+  expect(await createYamlLocalPipeline(config)).toMatchSnapshot();
+});

package/examples/cloud-run-with-sql-multiple-dbs.ts

@@ -0,0 +1,87 @@
+import type {
+  ComponentConfig,
+  Config,
+  DeployConfigCloudRunCloudSql,
+} from "../src";
+
+// both dbs currently need to run on the same instance
+const CLOUD_SQL_BASE: DeployConfigCloudRunCloudSql = {
+  type: "unmanaged",
+  instanceConnectionName: "projectId:region:instancename",
+  dbUser: "my-user",
+};
+
+const DbComponent = (name: string): ComponentConfig => ({
+  dir: "packages/" + name,
+  build: {
+    type: "node",
+  },
+  deploy: {
+    type: "google-cloudrun",
+    projectId: "google-project-id",
+    region: "europe-west6",
+    cloudSql: {
+      ...CLOUD_SQL_BASE,
+      /*
+      there is a flaw in the current implementation, 
+      that the db-url does not embed the env vars like DB_NAME, DB_USER, DB_PASSWORD, etc, 
+      This leads to the problem that if you reference the DATABASE_URL from another component,
+      it will not work properly, as it only contains var names instead of the actual values.
+      Setting this to "embedded" will embed the actual values into the db-url, which solves this problem.
+      */
+      dbConnectionStringVariablesMode: "embedded",
+    },
+    service: false,
+    jobs: {
+      migrate: {
+        command: "yarn migrate",
+      },
+    },
+    execute: {
+      migrate: {
+        type: "job",
+        job: "migrate",
+        when: "preDeploy",
+      },
+    },
+  },
+});
+const config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    // its good practise to have separated db components
+    db1: DbComponent("db1"),
+    db2: DbComponent("db2"),
+
+    api: {
+      dir: "api",
+      build: {
+        type: "node",
+        buildCommand: "yarn build:worker",
+      },
+      deploy: {
+        type: "google-cloudrun",
+        projectId: "google-project-id",
+        region: "europe-west6",
+        cloudSql: {
+          // currently one db is the default, as it injects additional env vars
+          ...CLOUD_SQL_BASE,
+          dbBaseName: "db1",
+        },
+      },
+      vars: {
+        public: {
+          // we reference the second db-url
+          DATABASE_URL_2: "${db2:DATABASE_URL}",
+        },
+      },
+    },
+  },
+} satisfies Config;
+
+export default config;
+
+export const information = {
+  title: "Cloud Run: With SQL Reuse DB",
+};

package/package.json

@@ -53,7 +53,7 @@
     }
   ],
   "license": "MIT",
-  "version": "3.9.2",
+  "version": "3.10.0",
   "scripts": {
     "build:tsc": "yarn tsc",
     "build": "yarn build:compile && yarn build:inline-variables",

package/src/build/rails/test.ts

@@ -93,7 +93,7 @@ export const createRailsTestJobs = (
             `cd ${context.build.dir}`,
             ...bundlerInstall,
             ...(ensureArrayOrNull(buildConfig.test?.command) ?? [
-              "bundle exec rake db:setup",
+              "bundle exec rake db:test:prepare",
               "bundle exec rake assets:precompile assets:clean",
               "bundle exec rspec",
             ]),

package/src/deploy/cloudRun/index.ts

@@ -11,8 +11,9 @@ import { getFullDbName } from "../cloudSql/utils";
 import { createGoogleCloudRunDeployJobs } from "./createJobs";
 import { getCloudRunJobExecuteUrl } from "./utils/cloudRunExecutionUrl";
 import {
-  DATABASE_JDBC_URL,
+  DEFAULT_DB_VARIABLES_MODE,
   getDatabaseConnectionString,
+  getDatabaseJdbcUrl,
 } from "./utils/database";
 
 export const GCLOUD_DEPLOY_CREDENTIALS_KEY = "GCLOUD_DEPLOY_credentialsKey";
@@ -42,16 +43,24 @@ const getCloudSqlVariables = ({
       .map(([key, value]) => `&${key}=${value}`)
       .join("");
 
-    return {
+    const dbVars = {
       CLOUD_SQL_INSTANCE_CONNECTION_NAME:
         deployConfigRaw.cloudSql.instanceConnectionName,
       DB_NAME: DB_NAME,
       DB_USER: deployConfigRaw.cloudSql.dbUser ?? "postgres",
       DB_PASSWORD: "$" + getSecretVarName(env, componentName, "DB_PASSWORD"),
-      DATABASE_URL: `${getDatabaseConnectionString(
+    };
+    return {
+      ...dbVars,
+      DATABASE_URL: getDatabaseConnectionString(
         deployConfigRaw.cloudSql,
-      )}${additionalQueryParamsString}`,
-      DATABASE_JDBC_URL: DATABASE_JDBC_URL,
+        dbVars,
+      ).concat(additionalQueryParamsString),
+      DATABASE_JDBC_URL: getDatabaseJdbcUrl(
+        dbVars,
+        deployConfigRaw.cloudSql.dbConnectionStringVariablesMode ??
+          DEFAULT_DB_VARIABLES_MODE,
+      ),
     };
   }
   return {};

package/src/deploy/cloudRun/utils/database.ts

@@ -1,3 +1,8 @@
+import {
+  BashExpression,
+  joinBashExpressions,
+  type StringOrBashExpression,
+} from "../../../bash";
 import type { ComponentContext } from "../../../types";
 import { allowFailureInScripts, repeatOnFailure } from "../../../utils/gitlab";
 import type {
@@ -57,21 +62,101 @@ export const getDatabaseCreateScript = (
   ]);
 };
 
-export const DATABASE_JDBC_URL =
-  "jdbc:postgresql:///$DB_NAME?cloudSqlInstance=$CLOUD_SQL_INSTANCE_CONNECTION_NAME&socketFactory=com.google.cloud.sql.postgres.SocketFactory&user=$DB_USER&password=$DB_PASSWORD";
+export type DBVariables = {
+  CLOUD_SQL_INSTANCE_CONNECTION_NAME: StringOrBashExpression;
+  DB_NAME: StringOrBashExpression;
+  DB_USER: StringOrBashExpression;
+  DB_PASSWORD: StringOrBashExpression;
+};
+
+/**
+ * controls how variables in the connection string are handled
+ *
+ * - legacy: variables like $DB_USER will be kept as environment variables to be replaced at runtime (default). It is not using BashExpressions as this was not the case in the past.
+ * - embedded: variables will be replaced with their actual values in the connection string
+ *
+ * We will remove the legacy mode in the future, as it is confusing. But its unclear whether its a breaking change in some edge cases
+ */
+export type DBVariablesMode = "legacy" | "embedded";
+
+export const DEFAULT_DB_VARIABLES_MODE: DBVariablesMode = "legacy";
+
+const getVariableOrValue = (
+  key: keyof DBVariables,
+  variables: DBVariables,
+  mode: DBVariablesMode,
+): StringOrBashExpression => {
+  return mode === "legacy" ? `$${key}` : variables[key];
+};
+
+export const getDatabaseJdbcUrl = (
+  variables: DBVariables,
+  mode: DBVariablesMode,
+) => {
+  const parts = [
+    "jdbc:postgresql:///",
+    getVariableOrValue("DB_NAME", variables, mode),
+    "?cloudSqlInstance=",
+    getVariableOrValue("CLOUD_SQL_INSTANCE_CONNECTION_NAME", variables, mode),
+    "&socketFactory=com.google.cloud.sql.postgres.SocketFactory&user=",
+    getVariableOrValue("DB_USER", variables, mode),
+    "&password=",
+    getVariableOrValue("DB_PASSWORD", variables, mode),
+  ];
+
+  return joinBashExpressions(parts);
+};
+
+export const getRailsDatabaseConnectionString = (
+  variables: DBVariables,
+  mode: DBVariablesMode,
+) => {
+  const parts = [
+    "postgresql://",
+    getVariableOrValue("DB_USER", variables, mode),
+    ":",
+    getVariableOrValue("DB_PASSWORD", variables, mode),
+    "@",
+    encodeURIComponent(
+      `/cloudsql/${variables.CLOUD_SQL_INSTANCE_CONNECTION_NAME}`,
+    ),
+    "/",
+    getVariableOrValue("DB_NAME", variables, mode),
+    "?",
+  ];
+  return joinBashExpressions(parts);
+};
+
+export const getPrismaDatabaseConnectionString = (
+  variables: DBVariables,
+  mode: DBVariablesMode,
+) => {
+  const parts = [
+    "postgresql://",
+    getVariableOrValue("DB_USER", variables, mode),
+    ":",
+    getVariableOrValue("DB_PASSWORD", variables, mode),
+    "@localhost/",
+    getVariableOrValue("DB_NAME", variables, mode),
+    "?host=/cloudsql/",
+    getVariableOrValue("CLOUD_SQL_INSTANCE_CONNECTION_NAME", variables, mode),
+  ];
+  return joinBashExpressions(parts);
+};
 
 export const getDatabaseConnectionString = (
   config: DeployConfigCloudRunCloudSql,
-): string => {
+  variables: DBVariables,
+): StringOrBashExpression => {
+  const mode =
+    config.dbConnectionStringVariablesMode ?? DEFAULT_DB_VARIABLES_MODE;
   switch (config.dbConnectionStringFormat) {
     case "jdbc":
-      return DATABASE_JDBC_URL;
+      return getDatabaseJdbcUrl(variables, mode);
     case "rails":
-      return `postgresql://$DB_USER:$DB_PASSWORD@${encodeURIComponent(
-        `/cloudsql/${config.instanceConnectionName}`,
-      )}/$DB_NAME?`;
+      return getRailsDatabaseConnectionString(variables, mode);
     default:
       // prisma
-      return "postgresql://$DB_USER:$DB_PASSWORD@localhost/$DB_NAME?host=/cloudsql/$CLOUD_SQL_INSTANCE_CONNECTION_NAME";
+      return getPrismaDatabaseConnectionString(variables, mode);
   }
 };

package/src/deploy/types/googleCloudRun.ts

@@ -1,3 +1,4 @@
+import type { DBVariablesMode } from "../cloudRun/utils/database";
 import type { DeployConfigBase } from "./base";
 
 export type Gcloudregion =
@@ -77,6 +78,16 @@ export type DeployConfigCloudRunCloudSql = {
    */
   dbConnectionStringFormat?: "prisma" | "rails" | "jdbc";
 
+  /**
+   * controls how variables in the connection string are handled
+   *
+   * - environment: variables like $DB_USER will be kept as environment variables to be replaced at runtime (default)
+   * - embedded: variables will be replaced with their actual values in the connection string
+   *
+   * @default "environment"
+   */
+  dbConnectionStringVariablesMode?: DBVariablesMode;
+
   /**
    * add additional query params to the database connection string
    */