@catladder/pipeline 4.6.4 → 4.7.1

package/src/deploy/cloudRun/utils/database.ts

@@ -5,6 +5,11 @@ import {
 } from "../../../bash";
 import type { ComponentContext } from "../../../types";
 import { allowFailureInScripts, repeatOnFailure } from "../../../utils/gitlab";
+import type { VariableValue } from "../../../variables/VariableValue";
+import {
+  VariableReference,
+  VariableValueContainingReferences,
+} from "../../../variables/VariableValueContainingReferences";
 import type {
   DeployConfigCloudRun,
   DeployConfigCloudRunCloudSql,
@@ -73,7 +78,7 @@ export type DBVariables = {
  * controls how variables in the connection string are handled
  *
  * - legacy: variables like $DB_USER will be kept as environment variables to be replaced at runtime (default). It is not using BashExpressions as this was not the case in the past.
- * - embedded: variables will be replaced with their actual values in the connection string
+ * - embedded: variables will be replaced with the component's final values in the connection string. This makes the connection string usable from other components (e.g. via ${otherComponent:DATABASE_URL}) and respects overrides of DB_USER, DB_PASSWORD, etc. in vars.public (e.g. DB_PASSWORD: "${otherComponent:DB_PASSWORD}" when reusing another component's database)
  *
  * We will remove the legacy mode in the future, as it is confusing. But its unclear whether its a breaking change in some edge cases
  */
@@ -81,82 +86,109 @@ export type DBVariablesMode = "legacy" | "embedded";
 
 export const DEFAULT_DB_VARIABLES_MODE: DBVariablesMode = "legacy";
 
+type DbUrlPart = StringOrBashExpression | VariableReference;
+
 const getVariableOrValue = (
   key: keyof DBVariables,
-  variables: DBVariables,
   mode: DBVariablesMode,
-): StringOrBashExpression => {
-  return mode === "legacy" ? `$${key}` : variables[key];
+  componentName: string,
+): DbUrlPart => {
+  // in embedded mode we reference the component's own env var, which gets
+  // resolved to its final value (including vars.public overrides) after all
+  // env vars have been merged
+  return mode === "legacy"
+    ? `$${key}`
+    : new VariableReference(componentName, key);
 };
 
+const joinDbUrlParts = (
+  parts: DbUrlPart[],
+  mode: DBVariablesMode,
+): VariableValue =>
+  mode === "legacy"
+    ? joinBashExpressions(parts as StringOrBashExpression[])
+    : new VariableValueContainingReferences(parts);
+
 export const getDatabaseJdbcUrl = (
   variables: DBVariables,
   mode: DBVariablesMode,
+  componentName: string,
 ) => {
   const parts = [
     "jdbc:postgresql:///",
-    getVariableOrValue("DB_NAME", variables, mode),
+    getVariableOrValue("DB_NAME", mode, componentName),
     "?cloudSqlInstance=",
-    getVariableOrValue("CLOUD_SQL_INSTANCE_CONNECTION_NAME", variables, mode),
+    getVariableOrValue(
+      "CLOUD_SQL_INSTANCE_CONNECTION_NAME",
+      mode,
+      componentName,
+    ),
     "&socketFactory=com.google.cloud.sql.postgres.SocketFactory&user=",
-    getVariableOrValue("DB_USER", variables, mode),
+    getVariableOrValue("DB_USER", mode, componentName),
     "&password=",
-    getVariableOrValue("DB_PASSWORD", variables, mode),
+    getVariableOrValue("DB_PASSWORD", mode, componentName),
   ];
 
-  return joinBashExpressions(parts);
+  return joinDbUrlParts(parts, mode);
 };
 
 export const getRailsDatabaseConnectionString = (
   variables: DBVariables,
   mode: DBVariablesMode,
+  componentName: string,
 ) => {
   const parts = [
     "postgresql://",
-    getVariableOrValue("DB_USER", variables, mode),
+    getVariableOrValue("DB_USER", mode, componentName),
     ":",
-    getVariableOrValue("DB_PASSWORD", variables, mode),
+    getVariableOrValue("DB_PASSWORD", mode, componentName),
     "@",
     encodeURIComponent(
       `/cloudsql/${variables.CLOUD_SQL_INSTANCE_CONNECTION_NAME}`,
     ),
     "/",
-    getVariableOrValue("DB_NAME", variables, mode),
+    getVariableOrValue("DB_NAME", mode, componentName),
     "?",
   ];
-  return joinBashExpressions(parts);
+  return joinDbUrlParts(parts, mode);
 };
 
 export const getPrismaDatabaseConnectionString = (
   variables: DBVariables,
   mode: DBVariablesMode,
+  componentName: string,
 ) => {
   const parts = [
     "postgresql://",
-    getVariableOrValue("DB_USER", variables, mode),
+    getVariableOrValue("DB_USER", mode, componentName),
     ":",
-    getVariableOrValue("DB_PASSWORD", variables, mode),
+    getVariableOrValue("DB_PASSWORD", mode, componentName),
     "@localhost/",
-    getVariableOrValue("DB_NAME", variables, mode),
+    getVariableOrValue("DB_NAME", mode, componentName),
     "?host=/cloudsql/",
-    getVariableOrValue("CLOUD_SQL_INSTANCE_CONNECTION_NAME", variables, mode),
+    getVariableOrValue(
+      "CLOUD_SQL_INSTANCE_CONNECTION_NAME",
+      mode,
+      componentName,
+    ),
   ];
-  return joinBashExpressions(parts);
+  return joinDbUrlParts(parts, mode);
 };
 
 export const getDatabaseConnectionString = (
   config: DeployConfigCloudRunCloudSql,
   variables: DBVariables,
-): StringOrBashExpression => {
+  componentName: string,
+): VariableValue => {
   const mode =
     config.dbConnectionStringVariablesMode ?? DEFAULT_DB_VARIABLES_MODE;
   switch (config.dbConnectionStringFormat) {
     case "jdbc":
-      return getDatabaseJdbcUrl(variables, mode);
+      return getDatabaseJdbcUrl(variables, mode, componentName);
     case "rails":
-      return getRailsDatabaseConnectionString(variables, mode);
+      return getRailsDatabaseConnectionString(variables, mode, componentName);
     default:
       // prisma
-      return getPrismaDatabaseConnectionString(variables, mode);
+      return getPrismaDatabaseConnectionString(variables, mode, componentName);
   }
 };

package/src/deploy/index.ts

@@ -1,9 +1,9 @@
 import type { BuildConfig, SecretEnvVar } from "..";
-import type { BashExpression } from "../bash/BashExpression";
 import type { ComponentContext } from "../types/context";
 import type { EnvironmentContext } from "../types/environmentContext";
 import type { CatladderJob } from "../types/jobs";
 import type { PartialDeep } from "../types/utils";
+import type { VariableValue } from "../variables/VariableValue";
 import { GCLOUD_RUN_DEPLOY_TYPE } from "./cloudRun";
 import { CUSTOM_DEPLOY_TYPE } from "./custom";
 import { DOCKER_TAG_DEPLOY_TYPE } from "./dockerTag";
@@ -26,7 +26,12 @@ export type DeployTypeDefinition<D extends DeployConfig> = {
   ) => SecretEnvVar[];
   getAdditionalEnvVars: (
     envContext: EnvironmentContext<BuildConfig, D>,
-  ) => Record<string, string | BashExpression | undefined | null>;
+  ) => Record<string, VariableValue | undefined | null>;
+  /**
+   * script lines the deploy type contributes to the start of the verify job,
+   * e.g. to authenticate against a non-public service
+   */
+  verifyJobSetupScript?: (context: ComponentContext) => string[];
 };
 export type DeployTypes = {
   [T in DeployConfigType]: DeployTypeDefinition<DeployConfigGeneric<T>>;

package/src/index.ts

@@ -6,6 +6,7 @@ export * from "./config";
 export * from "./context";
 export * from "./build";
 export * from "./deploy";
+export * from "./verify";
 export * from "./utils/writeFiles";
 export * from "./utils/measureTime";
 export * from "./variables/VariableValue";

package/src/pipeline/createJobsForComponent.ts

@@ -5,6 +5,7 @@ import type {
   ComponentContextWithBuild,
 } from "../types/context";
 import type { CatladderJob } from "../types/jobs";
+import { createVerifyJobs } from "../verify/createVerifyJobs";
 
 const injectDefaultVarsInCustomJobs = (
   context: ComponentContext,
@@ -27,7 +28,7 @@ const getCustomJobs = (context: ComponentContext) => {
 export const createJobsForComponentContext = async (
   context: ComponentContext,
 ): Promise<CatladderJob[]> => {
-  const [buildJobs, deployJobs] = await Promise.all([
+  const [buildJobs, deployJobs, verifyJobs] = await Promise.all([
     context.build.type !== "disabled"
       ? BUILD_TYPES[context.build.buildType].jobs(
           context as ComponentContextWithBuild,
@@ -36,9 +37,10 @@ export const createJobsForComponentContext = async (
     context.componentConfig.deploy !== false
       ? DEPLOY_TYPES[context.componentConfig.deploy.type].jobs(context)
       : [],
+    createVerifyJobs(context),
   ]);
 
   const customJobs = getCustomJobs(context);
 
-  return [...buildJobs, ...deployJobs, ...customJobs];
+  return [...buildJobs, ...deployJobs, ...verifyJobs, ...customJobs];
 };

package/src/types/config.ts

@@ -8,6 +8,7 @@ import type { PipelineType, WorkspaceBuildConfig } from "..";
 import type { AgentConfig } from "./agent";
 import type { Hooks } from "./hooks";
 import type { ReleaseConfig } from "./release";
+import type { VerifyConfig } from "../verify/types";
 
 export const ALL_PIPELINE_TRIGGERS = [
   "mainBranch",
@@ -91,6 +92,14 @@ export type DefaultEnvConfig = {
    * environment variables
    */
   vars?: EnvVars;
+
+  /**
+   * post-deploy verification: runs a job in the `verify` stage after the deploy,
+   * e.g. an e2e test suite against the freshly deployed environment.
+   *
+   * Runs in all deployed envs by default, disable per env with `verify: false`
+   */
+  verify?: VerifyConfig | false;
 };
 
 export type DevLocalEnvConfig = {

package/src/variables/VariableValueContainingReferences.ts

@@ -43,6 +43,17 @@ export class VariableValueContainingReferences {
     );
   }
 
+  /**
+   * concats values to this one and returns a new VariableValueContainingReferences.
+   * mirrors String.prototype.concat / BashExpression.concat so a VariableValue
+   * can be extended regardless of its concrete type
+   */
+  public concat(
+    ...values: Array<VariableValuePart | VariableValueContainingReferences>
+  ): VariableValueContainingReferences {
+    return new VariableValueContainingReferences([...this.parts, ...values]);
+  }
+
   public toString(
     options: EscapeOptions = {
       quotes: false,

package/src/variables/__tests__/resolveAllReferences.test.ts

@@ -1,5 +1,7 @@
 import { describe, it, expect } from "vitest";
+import { getBashVariable } from "../../bash/BashExpression";
 import {
+  VariableReference,
   VariableValueContainingReferences,
   createVariableValueContainingReferencesFromString,
 } from "../VariableValueContainingReferences";
@@ -68,6 +70,53 @@ describe("replaceAllReferences", () => {
     });
   });
 
+  it("passes through plain strings and bash expressions and resolves self references against the map fetched for the own component", async () => {
+    const DB_PASSWORD_SECRET = getBashVariable("CL_dev_api_DB_PASSWORD");
+
+    // a merged env var map as produced by getEnvironmentVariables: mostly
+    // plain values, plus a framework-generated value containing self
+    // references (like the embedded database connection string)
+    const values = {
+      DB_USER: "my-user",
+      DB_PASSWORD: DB_PASSWORD_SECRET,
+      DATABASE_URL: new VariableValueContainingReferences([
+        "postgresql://",
+        new VariableReference("worker", "DB_USER"),
+        ":",
+        new VariableReference("worker", "DB_PASSWORD"),
+        "@localhost/db",
+      ]),
+      EMPTY: null,
+    };
+
+    const getEnvVars = async (componentName: string) => {
+      expect(componentName).toBe("worker");
+      return {
+        DB_USER: "my-user",
+        // the fetched map contains the override, like a vars.public override
+        // referencing another component's password
+        DB_PASSWORD: DB_PASSWORD_SECRET,
+      };
+    };
+
+    const result = await resolveAllReferences(values, getEnvVars);
+    expect(result.DB_USER).toBe("my-user");
+    expect(result.DB_PASSWORD).toBe(DB_PASSWORD_SECRET);
+    expect(result.EMPTY).toBe(null);
+    expect(result.DATABASE_URL).toEqual(
+      new VariableValueContainingReferences([
+        "postgresql://",
+        "my-user",
+        ":",
+        DB_PASSWORD_SECRET,
+        "@localhost/db",
+      ]),
+    );
+    expect(result.DATABASE_URL.toString()).toBe(
+      "postgresql://my-user:$CL_dev_api_DB_PASSWORD@localhost/db",
+    );
+  });
+
   it("detects infinte loop", async () => {
     const values = {
       myVar: createVariableValueContainingReferencesFromString(

package/src/variables/resolveAllReferences.ts

@@ -1,31 +1,35 @@
 import type { VariableValue } from "./VariableValue";
-import type { VariableValueContainingReferences } from "./VariableValueContainingReferences";
-import { VariableReference } from "./VariableValueContainingReferences";
+import {
+  VariableReference,
+  VariableValueContainingReferences,
+} from "./VariableValueContainingReferences";
 import { resolveAllReferencesOnce as resolveAllReferencesOnce } from "./resolveAllReferencesOnce";
 
-export const resolveAllReferences = async (
-  values: Record<string, VariableValueContainingReferences>,
+const hasUnresolvedReferences = (value: VariableValue | null | undefined) =>
+  value instanceof VariableValueContainingReferences &&
+  value.parts.some((part) => part instanceof VariableReference);
+
+export const resolveAllReferences = async <
+  T extends Record<string, VariableValue | null | undefined>,
+>(
+  values: T,
   getEnvVars: (
     componentName: string,
   ) => Promise<Record<string, VariableValue | null | undefined>>,
-) => {
+): Promise<T> => {
   // replace until there aren't any references left
   let result = values;
 
   let i = 0;
 
-  while (
-    Object.values(result).some((value) =>
-      value.parts.some((part) => part instanceof VariableReference),
-    )
-  ) {
+  while (Object.values(result).some(hasUnresolvedReferences)) {
     const replaced = await resolveAllReferencesOnce(result, getEnvVars);
 
     result = replaced;
     i++;
     if (i > 1000) {
       const unresolved = Object.entries(result).filter(([key, value]) =>
-        value.parts.some((part) => part instanceof VariableReference),
+        hasUnresolvedReferences(value),
       );
 
       throw new Error(
@@ -33,9 +37,9 @@ export const resolveAllReferences = async (
           unresolved
             .map(
               ([key, value]) =>
-                `${key} (last reference: ${value.parts.find(
-                  (part) => part instanceof VariableReference,
-                )})`,
+                `${key} (last reference: ${(
+                  value as VariableValueContainingReferences
+                ).parts.find((part) => part instanceof VariableReference)})`,
             )
             .join(", "),
       );

package/src/variables/resolveAllReferencesOnce.ts

@@ -1,19 +1,24 @@
 import type { VariableValue } from "./VariableValue";
-import type { VariableValueContainingReferences } from "./VariableValueContainingReferences";
-import { VariableReference } from "./VariableValueContainingReferences";
+import {
+  VariableReference,
+  VariableValueContainingReferences,
+} from "./VariableValueContainingReferences";
 import { resolveReferencesOnce } from "./resolveReferencesOnce";
 
-export const resolveAllReferencesOnce = async (
-  values: Record<string, VariableValueContainingReferences>,
+export const resolveAllReferencesOnce = async <
+  T extends Record<string, VariableValue | null | undefined>,
+>(
+  values: T,
   getEnvVars: (
     componentName: string,
   ) => Promise<Record<string, VariableValue | undefined | null>>,
-) => {
-  const allReferences = Object.values(values).flatMap(
-    (value) =>
-      value?.parts.filter(
-        (part) => part instanceof VariableReference,
-      ) as VariableReference[],
+): Promise<T> => {
+  const allReferences = Object.values(values).flatMap((value) =>
+    value instanceof VariableValueContainingReferences
+      ? (value.parts.filter(
+          (part) => part instanceof VariableReference,
+        ) as VariableReference[])
+      : [],
   );
 
   const allComponentsUnique = Array.from(
@@ -34,11 +39,11 @@ export const resolveAllReferencesOnce = async (
   return Object.fromEntries(
     Object.entries(values).map(([key, value]) => [
       key,
-      value !== null && value !== undefined
+      value instanceof VariableValueContainingReferences
         ? resolveReferencesOnce(value, ({ componentName, variableName }) => {
             return allEnvVarsInComponents[componentName][variableName];
           })
         : value,
     ]),
-  );
+  ) as T;
 };

package/src/verify/createVerifyJobs.ts

@@ -0,0 +1,81 @@
+import { createArtifactsConfig } from "../build/base/createArtifactsConfig";
+import { createJobCacheFromCacheConfigs } from "../build/cache/createJobCache";
+import { getNodeCache } from "../build/node/cache";
+import { NODE_RUNNER_BUILD_VARIABLES } from "../build/node/constants";
+import { ensureNodeVersion, getYarnInstall } from "../build/node/yarn";
+import { DEPLOY_TYPES } from "../deploy";
+import { DEPLOY_JOB_NAME } from "../deploy/base/deploy";
+import { getRunnerImage } from "../runner";
+import type { ComponentContext } from "../types/context";
+import type { CatladderJob } from "../types/jobs";
+import { ensureArray } from "../utils";
+
+export const VERIFY_JOB_NAME = "🔍 verify";
+
+export const createVerifyJobs = async (
+  context: ComponentContext,
+): Promise<CatladderJob[]> => {
+  const verifyConfig = context.componentConfig.verify;
+  if (!verifyConfig) {
+    return [];
+  }
+
+  const deployConfig = context.deploy?.config;
+  if (!deployConfig) {
+    // nothing is deployed, so there is nothing to verify
+    return [];
+  }
+
+  const isNodeBuild =
+    context.build.type !== "disabled" && context.build.buildType === "node";
+
+  const [yarnInstall, nodeCache] = isNodeBuild
+    ? await Promise.all([
+        getYarnInstall(context),
+        getNodeCache(context, "pull"),
+      ])
+    : [null, null];
+
+  const setupScript =
+    DEPLOY_TYPES[deployConfig.type].verifyJobSetupScript?.(context) ?? [];
+
+  return [
+    {
+      name: VERIFY_JOB_NAME,
+      stage: "verify",
+      envMode: "stagePerEnv",
+      // wait for this component's own deploy
+      needsStages: [{ stage: "deploy", artifacts: false }],
+      // ...and for the deploys of the components listed in waitFor
+      needs: (verifyConfig.waitFor ?? []).map((componentName) => ({
+        job: DEPLOY_JOB_NAME,
+        componentName,
+        artifacts: false,
+      })),
+      image: verifyConfig.jobImage ?? getRunnerImage("jobs-testing-chrome"),
+      cache: nodeCache
+        ? createJobCacheFromCacheConfigs(context, nodeCache)
+        : undefined,
+      variables: {
+        ...context.environment.envVars,
+      },
+      runnerVariables: {
+        ...NODE_RUNNER_BUILD_VARIABLES,
+        ...(verifyConfig.runnerVariables ?? {}),
+      },
+      script: [
+        ...setupScript,
+        ...(yarnInstall ? ensureNodeVersion(context) : []),
+        `cd ${context.build.dir}`,
+        ...(yarnInstall ?? []),
+        ...ensureArray(verifyConfig.command),
+      ],
+      allow_failure: verifyConfig.allowFailure,
+      ...createArtifactsConfig(
+        context.build.dir,
+        verifyConfig.artifactsReports,
+        verifyConfig.artifacts,
+      ),
+    },
+  ];
+};

package/src/verify/index.ts

@@ -0,0 +1,2 @@
+export * from "./types";
+export * from "./createVerifyJobs";

package/src/verify/types.ts

@@ -0,0 +1,18 @@
+import type { TestJobCustom } from "../build/types";
+
+/**
+ * post-deploy verification job configuration
+ */
+export type VerifyConfig = TestJobCustom & {
+  /**
+   * command(s) to run against the deployed environment (e.g. `yarn e2e`).
+   * Runs in the component's directory.
+   */
+  command: string | string[];
+
+  /**
+   * names of other components whose deploy must finish before this verify job runs
+   * (in addition to this component's own deploy, which is always awaited)
+   */
+  waitFor?: string[];
+};