@catladder/pipeline 4.7.0 → 4.7.1

package/src/deploy/cloudRun/utils/database.ts

@@ -5,6 +5,11 @@ import {
 } from "../../../bash";
 import type { ComponentContext } from "../../../types";
 import { allowFailureInScripts, repeatOnFailure } from "../../../utils/gitlab";
+import type { VariableValue } from "../../../variables/VariableValue";
+import {
+  VariableReference,
+  VariableValueContainingReferences,
+} from "../../../variables/VariableValueContainingReferences";
 import type {
   DeployConfigCloudRun,
   DeployConfigCloudRunCloudSql,
@@ -73,7 +78,7 @@ export type DBVariables = {
  * controls how variables in the connection string are handled
  *
  * - legacy: variables like $DB_USER will be kept as environment variables to be replaced at runtime (default). It is not using BashExpressions as this was not the case in the past.
- * - embedded: variables will be replaced with their actual values in the connection string
+ * - embedded: variables will be replaced with the component's final values in the connection string. This makes the connection string usable from other components (e.g. via ${otherComponent:DATABASE_URL}) and respects overrides of DB_USER, DB_PASSWORD, etc. in vars.public (e.g. DB_PASSWORD: "${otherComponent:DB_PASSWORD}" when reusing another component's database)
  *
  * We will remove the legacy mode in the future, as it is confusing. But its unclear whether its a breaking change in some edge cases
  */
@@ -81,82 +86,109 @@ export type DBVariablesMode = "legacy" | "embedded";
 
 export const DEFAULT_DB_VARIABLES_MODE: DBVariablesMode = "legacy";
 
+type DbUrlPart = StringOrBashExpression | VariableReference;
+
 const getVariableOrValue = (
   key: keyof DBVariables,
-  variables: DBVariables,
   mode: DBVariablesMode,
-): StringOrBashExpression => {
-  return mode === "legacy" ? `$${key}` : variables[key];
+  componentName: string,
+): DbUrlPart => {
+  // in embedded mode we reference the component's own env var, which gets
+  // resolved to its final value (including vars.public overrides) after all
+  // env vars have been merged
+  return mode === "legacy"
+    ? `$${key}`
+    : new VariableReference(componentName, key);
 };
 
+const joinDbUrlParts = (
+  parts: DbUrlPart[],
+  mode: DBVariablesMode,
+): VariableValue =>
+  mode === "legacy"
+    ? joinBashExpressions(parts as StringOrBashExpression[])
+    : new VariableValueContainingReferences(parts);
+
 export const getDatabaseJdbcUrl = (
   variables: DBVariables,
   mode: DBVariablesMode,
+  componentName: string,
 ) => {
   const parts = [
     "jdbc:postgresql:///",
-    getVariableOrValue("DB_NAME", variables, mode),
+    getVariableOrValue("DB_NAME", mode, componentName),
     "?cloudSqlInstance=",
-    getVariableOrValue("CLOUD_SQL_INSTANCE_CONNECTION_NAME", variables, mode),
+    getVariableOrValue(
+      "CLOUD_SQL_INSTANCE_CONNECTION_NAME",
+      mode,
+      componentName,
+    ),
     "&socketFactory=com.google.cloud.sql.postgres.SocketFactory&user=",
-    getVariableOrValue("DB_USER", variables, mode),
+    getVariableOrValue("DB_USER", mode, componentName),
     "&password=",
-    getVariableOrValue("DB_PASSWORD", variables, mode),
+    getVariableOrValue("DB_PASSWORD", mode, componentName),
   ];
 
-  return joinBashExpressions(parts);
+  return joinDbUrlParts(parts, mode);
 };
 
 export const getRailsDatabaseConnectionString = (
   variables: DBVariables,
   mode: DBVariablesMode,
+  componentName: string,
 ) => {
   const parts = [
     "postgresql://",
-    getVariableOrValue("DB_USER", variables, mode),
+    getVariableOrValue("DB_USER", mode, componentName),
     ":",
-    getVariableOrValue("DB_PASSWORD", variables, mode),
+    getVariableOrValue("DB_PASSWORD", mode, componentName),
     "@",
     encodeURIComponent(
       `/cloudsql/${variables.CLOUD_SQL_INSTANCE_CONNECTION_NAME}`,
     ),
     "/",
-    getVariableOrValue("DB_NAME", variables, mode),
+    getVariableOrValue("DB_NAME", mode, componentName),
     "?",
   ];
-  return joinBashExpressions(parts);
+  return joinDbUrlParts(parts, mode);
 };
 
 export const getPrismaDatabaseConnectionString = (
   variables: DBVariables,
   mode: DBVariablesMode,
+  componentName: string,
 ) => {
   const parts = [
     "postgresql://",
-    getVariableOrValue("DB_USER", variables, mode),
+    getVariableOrValue("DB_USER", mode, componentName),
     ":",
-    getVariableOrValue("DB_PASSWORD", variables, mode),
+    getVariableOrValue("DB_PASSWORD", mode, componentName),
     "@localhost/",
-    getVariableOrValue("DB_NAME", variables, mode),
+    getVariableOrValue("DB_NAME", mode, componentName),
     "?host=/cloudsql/",
-    getVariableOrValue("CLOUD_SQL_INSTANCE_CONNECTION_NAME", variables, mode),
+    getVariableOrValue(
+      "CLOUD_SQL_INSTANCE_CONNECTION_NAME",
+      mode,
+      componentName,
+    ),
   ];
-  return joinBashExpressions(parts);
+  return joinDbUrlParts(parts, mode);
 };
 
 export const getDatabaseConnectionString = (
   config: DeployConfigCloudRunCloudSql,
   variables: DBVariables,
-): StringOrBashExpression => {
+  componentName: string,
+): VariableValue => {
   const mode =
     config.dbConnectionStringVariablesMode ?? DEFAULT_DB_VARIABLES_MODE;
   switch (config.dbConnectionStringFormat) {
     case "jdbc":
-      return getDatabaseJdbcUrl(variables, mode);
+      return getDatabaseJdbcUrl(variables, mode, componentName);
     case "rails":
-      return getRailsDatabaseConnectionString(variables, mode);
+      return getRailsDatabaseConnectionString(variables, mode, componentName);
     default:
       // prisma
-      return getPrismaDatabaseConnectionString(variables, mode);
+      return getPrismaDatabaseConnectionString(variables, mode, componentName);
   }
 };

package/src/deploy/index.ts

@@ -1,9 +1,9 @@
 import type { BuildConfig, SecretEnvVar } from "..";
-import type { BashExpression } from "../bash/BashExpression";
 import type { ComponentContext } from "../types/context";
 import type { EnvironmentContext } from "../types/environmentContext";
 import type { CatladderJob } from "../types/jobs";
 import type { PartialDeep } from "../types/utils";
+import type { VariableValue } from "../variables/VariableValue";
 import { GCLOUD_RUN_DEPLOY_TYPE } from "./cloudRun";
 import { CUSTOM_DEPLOY_TYPE } from "./custom";
 import { DOCKER_TAG_DEPLOY_TYPE } from "./dockerTag";
@@ -26,7 +26,7 @@ export type DeployTypeDefinition<D extends DeployConfig> = {
   ) => SecretEnvVar[];
   getAdditionalEnvVars: (
     envContext: EnvironmentContext<BuildConfig, D>,
-  ) => Record<string, string | BashExpression | undefined | null>;
+  ) => Record<string, VariableValue | undefined | null>;
   /**
    * script lines the deploy type contributes to the start of the verify job,
    * e.g. to authenticate against a non-public service

package/src/variables/VariableValueContainingReferences.ts

@@ -43,6 +43,17 @@ export class VariableValueContainingReferences {
     );
   }
 
+  /**
+   * concats values to this one and returns a new VariableValueContainingReferences.
+   * mirrors String.prototype.concat / BashExpression.concat so a VariableValue
+   * can be extended regardless of its concrete type
+   */
+  public concat(
+    ...values: Array<VariableValuePart | VariableValueContainingReferences>
+  ): VariableValueContainingReferences {
+    return new VariableValueContainingReferences([...this.parts, ...values]);
+  }
+
   public toString(
     options: EscapeOptions = {
       quotes: false,

package/src/variables/__tests__/resolveAllReferences.test.ts

@@ -1,5 +1,7 @@
 import { describe, it, expect } from "vitest";
+import { getBashVariable } from "../../bash/BashExpression";
 import {
+  VariableReference,
   VariableValueContainingReferences,
   createVariableValueContainingReferencesFromString,
 } from "../VariableValueContainingReferences";
@@ -68,6 +70,53 @@ describe("replaceAllReferences", () => {
     });
   });
 
+  it("passes through plain strings and bash expressions and resolves self references against the map fetched for the own component", async () => {
+    const DB_PASSWORD_SECRET = getBashVariable("CL_dev_api_DB_PASSWORD");
+
+    // a merged env var map as produced by getEnvironmentVariables: mostly
+    // plain values, plus a framework-generated value containing self
+    // references (like the embedded database connection string)
+    const values = {
+      DB_USER: "my-user",
+      DB_PASSWORD: DB_PASSWORD_SECRET,
+      DATABASE_URL: new VariableValueContainingReferences([
+        "postgresql://",
+        new VariableReference("worker", "DB_USER"),
+        ":",
+        new VariableReference("worker", "DB_PASSWORD"),
+        "@localhost/db",
+      ]),
+      EMPTY: null,
+    };
+
+    const getEnvVars = async (componentName: string) => {
+      expect(componentName).toBe("worker");
+      return {
+        DB_USER: "my-user",
+        // the fetched map contains the override, like a vars.public override
+        // referencing another component's password
+        DB_PASSWORD: DB_PASSWORD_SECRET,
+      };
+    };
+
+    const result = await resolveAllReferences(values, getEnvVars);
+    expect(result.DB_USER).toBe("my-user");
+    expect(result.DB_PASSWORD).toBe(DB_PASSWORD_SECRET);
+    expect(result.EMPTY).toBe(null);
+    expect(result.DATABASE_URL).toEqual(
+      new VariableValueContainingReferences([
+        "postgresql://",
+        "my-user",
+        ":",
+        DB_PASSWORD_SECRET,
+        "@localhost/db",
+      ]),
+    );
+    expect(result.DATABASE_URL.toString()).toBe(
+      "postgresql://my-user:$CL_dev_api_DB_PASSWORD@localhost/db",
+    );
+  });
+
   it("detects infinte loop", async () => {
     const values = {
       myVar: createVariableValueContainingReferencesFromString(

package/src/variables/resolveAllReferences.ts

@@ -1,31 +1,35 @@
 import type { VariableValue } from "./VariableValue";
-import type { VariableValueContainingReferences } from "./VariableValueContainingReferences";
-import { VariableReference } from "./VariableValueContainingReferences";
+import {
+  VariableReference,
+  VariableValueContainingReferences,
+} from "./VariableValueContainingReferences";
 import { resolveAllReferencesOnce as resolveAllReferencesOnce } from "./resolveAllReferencesOnce";
 
-export const resolveAllReferences = async (
-  values: Record<string, VariableValueContainingReferences>,
+const hasUnresolvedReferences = (value: VariableValue | null | undefined) =>
+  value instanceof VariableValueContainingReferences &&
+  value.parts.some((part) => part instanceof VariableReference);
+
+export const resolveAllReferences = async <
+  T extends Record<string, VariableValue | null | undefined>,
+>(
+  values: T,
   getEnvVars: (
     componentName: string,
   ) => Promise<Record<string, VariableValue | null | undefined>>,
-) => {
+): Promise<T> => {
   // replace until there aren't any references left
   let result = values;
 
   let i = 0;
 
-  while (
-    Object.values(result).some((value) =>
-      value.parts.some((part) => part instanceof VariableReference),
-    )
-  ) {
+  while (Object.values(result).some(hasUnresolvedReferences)) {
     const replaced = await resolveAllReferencesOnce(result, getEnvVars);
 
     result = replaced;
     i++;
     if (i > 1000) {
       const unresolved = Object.entries(result).filter(([key, value]) =>
-        value.parts.some((part) => part instanceof VariableReference),
+        hasUnresolvedReferences(value),
       );
 
       throw new Error(
@@ -33,9 +37,9 @@ export const resolveAllReferences = async (
           unresolved
             .map(
               ([key, value]) =>
-                `${key} (last reference: ${value.parts.find(
-                  (part) => part instanceof VariableReference,
-                )})`,
+                `${key} (last reference: ${(
+                  value as VariableValueContainingReferences
+                ).parts.find((part) => part instanceof VariableReference)})`,
             )
             .join(", "),
       );

package/src/variables/resolveAllReferencesOnce.ts

@@ -1,19 +1,24 @@
 import type { VariableValue } from "./VariableValue";
-import type { VariableValueContainingReferences } from "./VariableValueContainingReferences";
-import { VariableReference } from "./VariableValueContainingReferences";
+import {
+  VariableReference,
+  VariableValueContainingReferences,
+} from "./VariableValueContainingReferences";
 import { resolveReferencesOnce } from "./resolveReferencesOnce";
 
-export const resolveAllReferencesOnce = async (
-  values: Record<string, VariableValueContainingReferences>,
+export const resolveAllReferencesOnce = async <
+  T extends Record<string, VariableValue | null | undefined>,
+>(
+  values: T,
   getEnvVars: (
     componentName: string,
   ) => Promise<Record<string, VariableValue | undefined | null>>,
-) => {
-  const allReferences = Object.values(values).flatMap(
-    (value) =>
-      value?.parts.filter(
-        (part) => part instanceof VariableReference,
-      ) as VariableReference[],
+): Promise<T> => {
+  const allReferences = Object.values(values).flatMap((value) =>
+    value instanceof VariableValueContainingReferences
+      ? (value.parts.filter(
+          (part) => part instanceof VariableReference,
+        ) as VariableReference[])
+      : [],
   );
 
   const allComponentsUnique = Array.from(
@@ -34,11 +39,11 @@ export const resolveAllReferencesOnce = async (
   return Object.fromEntries(
     Object.entries(values).map(([key, value]) => [
       key,
-      value !== null && value !== undefined
+      value instanceof VariableValueContainingReferences
         ? resolveReferencesOnce(value, ({ componentName, variableName }) => {
             return allEnvVarsInComponents[componentName][variableName];
           })
         : value,
     ]),
-  );
+  ) as T;
 };