@catladder/pipeline 1.89.4 → 1.91.0

package/examples/__snapshots__/rails-k8s-with-worker.ts.snap

@@ -89,8 +89,11 @@ application:
     enabled: true
     command: 'launcher bundle exec rake jobs:work'
 cloudsql:
-  proxyCredentials: '$CL_dev_app_cloudsqlProxyCredentials'
   enabled: true
+  dbUser: 'postgres'
+  instanceConnectionName: 'some-project-id:europe-west6:pan-test-app-dev'
+  proxyCredentials: '$CL_dev_app_cloudsqlProxyCredentials'
+  fullDbName: 'app'
   projectId: 'some-project-id'
 jobs:
   db-migrate:
@@ -297,8 +300,11 @@ application:
     enabled: true
     command: 'launcher bundle exec rake jobs:work'
 cloudsql:
-  proxyCredentials: '$CL_dev_app_cloudsqlProxyCredentials'
   enabled: true
+  dbUser: 'postgres'
+  instanceConnectionName: 'some-project-id:europe-west6:pan-test-app-dev'
+  proxyCredentials: '$CL_dev_app_cloudsqlProxyCredentials'
+  fullDbName: 'app'
   projectId: 'some-project-id'
 jobs:
   db-migrate:
@@ -403,8 +409,11 @@ application:
     enabled: true
     command: 'launcher bundle exec rake jobs:work'
 cloudsql:
-  proxyCredentials: '$CL_dev_app_cloudsqlProxyCredentials'
   enabled: true
+  dbUser: 'postgres'
+  instanceConnectionName: 'some-project-id:europe-west6:pan-test-app-dev'
+  proxyCredentials: '$CL_dev_app_cloudsqlProxyCredentials'
+  fullDbName: 'app'
   projectId: 'some-project-id'
 jobs:
   db-migrate:
@@ -630,8 +639,11 @@ application:
     enabled: true
     command: 'launcher bundle exec rake jobs:work'
 cloudsql:
-  proxyCredentials: '$CL_review_app_cloudsqlProxyCredentials'
   enabled: true
+  dbUser: 'postgres'
+  instanceConnectionName: 'some-project-id:europe-west6:pan-test-app-review'
+  proxyCredentials: '$CL_review_app_cloudsqlProxyCredentials'
+  fullDbName: 'mr1234-app'
   projectId: 'some-project-id'
 jobs:
   db-migrate:
@@ -841,8 +853,11 @@ application:
     enabled: true
     command: 'launcher bundle exec rake jobs:work'
 cloudsql:
-  proxyCredentials: '$CL_review_app_cloudsqlProxyCredentials'
   enabled: true
+  dbUser: 'postgres'
+  instanceConnectionName: 'some-project-id:europe-west6:pan-test-app-review'
+  proxyCredentials: '$CL_review_app_cloudsqlProxyCredentials'
+  fullDbName: 'mr1234-app'
   projectId: 'some-project-id'
 jobs:
   db-migrate:
@@ -950,8 +965,11 @@ application:
     enabled: true
     command: 'launcher bundle exec rake jobs:work'
 cloudsql:
-  proxyCredentials: '$CL_review_app_cloudsqlProxyCredentials'
   enabled: true
+  dbUser: 'postgres'
+  instanceConnectionName: 'some-project-id:europe-west6:pan-test-app-review'
+  proxyCredentials: '$CL_review_app_cloudsqlProxyCredentials'
+  fullDbName: 'mr1234-app'
   projectId: 'some-project-id'
 jobs:
   db-migrate:
@@ -1180,8 +1198,11 @@ application:
     enabled: true
     command: 'launcher bundle exec rake jobs:work'
 cloudsql:
-  proxyCredentials: '$CL_prod_app_cloudsqlProxyCredentials'
   enabled: true
+  dbUser: 'postgres'
+  instanceConnectionName: 'some-project-id:europe-west6:pan-test-app-prod'
+  proxyCredentials: '$CL_prod_app_cloudsqlProxyCredentials'
+  fullDbName: 'app'
   projectId: 'some-project-id'
 jobs:
   db-migrate:
@@ -1281,8 +1302,11 @@ application:
     enabled: true
     command: 'launcher bundle exec rake jobs:work'
 cloudsql:
-  proxyCredentials: '$CL_stage_app_cloudsqlProxyCredentials'
   enabled: true
+  dbUser: 'postgres'
+  instanceConnectionName: 'some-project-id:europe-west6:pan-test-app-stage'
+  proxyCredentials: '$CL_stage_app_cloudsqlProxyCredentials'
+  fullDbName: 'app'
   projectId: 'some-project-id'
 jobs:
   db-migrate:
@@ -1495,8 +1519,11 @@ application:
     enabled: true
     command: 'launcher bundle exec rake jobs:work'
 cloudsql:
-  proxyCredentials: '$CL_prod_app_cloudsqlProxyCredentials'
   enabled: true
+  dbUser: 'postgres'
+  instanceConnectionName: 'some-project-id:europe-west6:pan-test-app-prod'
+  proxyCredentials: '$CL_prod_app_cloudsqlProxyCredentials'
+  fullDbName: 'app'
   projectId: 'some-project-id'
 jobs:
   db-migrate:
@@ -1609,8 +1636,11 @@ application:
     enabled: true
     command: 'launcher bundle exec rake jobs:work'
 cloudsql:
-  proxyCredentials: '$CL_stage_app_cloudsqlProxyCredentials'
   enabled: true
+  dbUser: 'postgres'
+  instanceConnectionName: 'some-project-id:europe-west6:pan-test-app-stage'
+  proxyCredentials: '$CL_stage_app_cloudsqlProxyCredentials'
+  fullDbName: 'app'
   projectId: 'some-project-id'
 jobs:
   db-migrate:
@@ -1715,8 +1745,11 @@ application:
     enabled: true
     command: 'launcher bundle exec rake jobs:work'
 cloudsql:
-  proxyCredentials: '$CL_prod_app_cloudsqlProxyCredentials'
   enabled: true
+  dbUser: 'postgres'
+  instanceConnectionName: 'some-project-id:europe-west6:pan-test-app-prod'
+  proxyCredentials: '$CL_prod_app_cloudsqlProxyCredentials'
+  fullDbName: 'app'
   projectId: 'some-project-id'
 jobs:
   db-migrate:
@@ -1821,8 +1854,11 @@ application:
     enabled: true
     command: 'launcher bundle exec rake jobs:work'
 cloudsql:
-  proxyCredentials: '$CL_stage_app_cloudsqlProxyCredentials'
   enabled: true
+  dbUser: 'postgres'
+  instanceConnectionName: 'some-project-id:europe-west6:pan-test-app-stage'
+  proxyCredentials: '$CL_stage_app_cloudsqlProxyCredentials'
+  fullDbName: 'app'
   projectId: 'some-project-id'
 jobs:
   db-migrate:

package/examples/custom-deploy.ts

@@ -0,0 +1,25 @@
+import type { Config } from "../src";
+import { createAllPipelines } from "./__utils__/helpers";
+
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    www: {
+      dir: "www",
+      build: {
+        type: "node",
+      },
+      deploy: {
+        type: "custom",
+        requiresDocker: true,
+        script: ["echo 'would deploy'"],
+        stopScript: ["echo 'would stop'"],
+      },
+    },
+  },
+};
+
+it("matches snapshot", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});

package/examples/kubernetes-with-cloud-sql.ts

@@ -0,0 +1,35 @@
+import type { Config } from "../src";
+import { createAllPipelines } from "./__utils__/helpers";
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    api: {
+      dir: "api",
+      build: {
+        type: "node",
+      },
+      deploy: {
+        type: "kubernetes",
+        cluster: {
+          name: "some-cluster-name",
+          region: "europe-west6",
+          projectId: "some-project-id",
+          type: "gcloud",
+          domainCanonical: "panter.cloud",
+        },
+        values: {
+          cloudsql: {
+            enabled: true,
+            type: "unmanaged",
+            instanceConnectionName: "myproject:europe-west6:instance-name",
+          },
+        },
+      },
+    },
+  },
+};
+
+it("matches snapshot", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@catladder/pipeline",
-  "version": "1.89.4",
+  "version": "1.91.0",
   "scripts": {
     "build:tsc": "yarn tsc",
     "build": "yarn build:compile && yarn build:inline-variables && yarn build:bundle",

package/src/context/getEnvironmentVariables.ts

@@ -9,6 +9,11 @@ import {
   translateLegacyFromComponents,
 } from "./resolveReferences";
 
+export type SecretEnvVar = {
+  key: string;
+  // hidden env vars are not shown in config-secrets
+  hidden?: boolean;
+};
 export const getEnvironmentVariables = async (
   config: Config,
   componentName: string,
@@ -17,7 +22,7 @@ export const getEnvironmentVariables = async (
   alreadyVisited: Record<string, Record<string, boolean>> = {} // to prevent endless loop
 ): Promise<{
   envVars: Record<string, string>;
-  secretEnvVarKeys: string[];
+  secretEnvVarKeys: SecretEnvVar[];
   host: string;
   url: string;
 }> => {
@@ -81,12 +86,12 @@ export const getEnvironmentVariables = async (
       )
     : [];
 
-  const secretEnvVarKeys = [
-    ...(envConfigRaw.vars?.secret ?? []),
+  const secretEnvVarKeys: SecretEnvVar[] = [
+    ...(envConfigRaw.vars?.secret ?? []).map((key) => ({ key })),
     ...additionalSecretKeys,
   ];
   const secretEnvVars = Object.fromEntries(
-    secretEnvVarKeys.map((key) => [
+    secretEnvVarKeys.map(({ key }) => [
       key,
       `$${getSecretVarName(env, componentName, key)}`,
     ])

package/src/deploy/cloudRun/deployJob.ts

@@ -1,5 +1,5 @@
 import { dump } from "js-yaml";
-import { isNil, merge, omit } from "lodash";
+import { merge, omit } from "lodash";
 import { GCLOUD_DEPLOY_CREDENTIALS_KEY } from ".";
 import { getDockerJobBaseProps, gitlabDockerLogin } from "../../build/docker";
 import { getLabels } from "../../context/getLabels";

package/src/deploy/cloudRun/index.ts

@@ -1,7 +1,8 @@
 import type { DeployTypeDefinition } from "..";
 import { getSecretVarName } from "../../context";
-import { sanitizeForBashVariable } from "../../utils/gitlab";
 import type { EnvironmentContext } from "../../types/environmentContext";
+import { sanitizeForBashVariable } from "../../utils/gitlab";
+import { getFullDbName } from "../cloudSql/utils";
 import { createGoogleCloudRunDeployJobs } from "./deployJob";
 import { getCloudRunJobName } from "./utils/jobName";
 
@@ -18,14 +19,12 @@ const getCloudSqlVariables = ({
   fullConfig,
 }: EnvironmentContext<any, "google-cloudrun">) => {
   if (deployConfigRaw && deployConfigRaw.cloudSql) {
-    const DB_NAME = [
-      deployConfigRaw.cloudSql.dbNamePrefix ??
-        `${fullConfig.customerName}-${fullConfig.appName}`,
+    const DB_NAME = getFullDbName(
+      deployConfigRaw.cloudSql,
+      fullConfig,
       environmentSlugPrefix,
-      deployConfigRaw.cloudSql.dbBaseName ?? componentName,
-    ]
-      .filter(Boolean)
-      .join("-");
+      componentName
+    );
 
     const additionalQueryParamsString = Object.entries(
       deployConfigRaw.cloudSql.dbAdditionalQueryParams ?? {}
@@ -58,9 +57,16 @@ export const GCLOUD_RUN_DEPLOY_TYPE: DeployTypeDefinition<"google-cloudrun"> = {
     return {};
   },
   additionalSecretKeys: (ctx) => [
-    GCLOUD_DEPLOY_CREDENTIALS_KEY,
+    {
+      key: GCLOUD_DEPLOY_CREDENTIALS_KEY,
+      hidden: true,
+    },
+    {
+      key: GCLOUD_RUN_CANONICAL_HOST_SUFFIX,
+      hidden: true,
+    },
     ...(ctx.deployConfigRaw && ctx.deployConfigRaw.cloudSql
-      ? ["DB_PASSWORD"]
+      ? [{ key: "DB_PASSWORD" }]
       : []),
   ],
   getAdditionalEnvVars: (ctx) => {

package/src/deploy/cloudSql/utils.ts

@@ -0,0 +1,23 @@
+import type { Config } from "../..";
+import type {
+  DeployConfigCloudRunCloudSql,
+  DeployConfigKubernetesValuesCloudSQLUnmanaged,
+} from "../types";
+
+export const getFullDbName = (
+  cloudSqlConfig:
+    | DeployConfigCloudRunCloudSql
+    | DeployConfigKubernetesValuesCloudSQLUnmanaged,
+  fullConfig: Config<never>,
+  environmentSlugPrefix: string,
+  componentName: string
+) => {
+  return [
+    cloudSqlConfig.dbNamePrefix ??
+      `${fullConfig.customerName}-${fullConfig.appName}`,
+    environmentSlugPrefix,
+    cloudSqlConfig.dbBaseName ?? componentName,
+  ]
+    .filter(Boolean)
+    .join("-");
+};

package/src/deploy/custom/deployJob.ts

@@ -4,7 +4,7 @@ import { getYarnInstall } from "../../build/node/yarn";
 import { getRunnerImage } from "../../runner";
 import type { Context } from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
-import { getBaseDeploymentJob } from "../base";
+import { getBaseDeploymentJob, getBaseDeploymentStopJob } from "../base";
 import { isOfDeployType } from "../types";
 
 export const createCustomDeployJobs = (context: Context): CatladderJob[] => {
@@ -29,5 +29,13 @@ export const createCustomDeployJobs = (context: Context): CatladderJob[] => {
         ...deployConfig.script,
       ],
     }),
+    ...(deployConfig.stopScript
+      ? [
+          merge({}, getBaseDeploymentStopJob(context), {
+            image: deployConfig.jobImage ?? getRunnerImage("jobs-default"),
+            script: deployConfig.stopScript,
+          }),
+        ]
+      : []),
   ];
 };

package/src/deploy/index.ts

@@ -1,3 +1,4 @@
+import type { SecretEnvVar } from "..";
 import type { Context } from "../types/context";
 import type { EnvironmentContext } from "../types/environmentContext";
 import type { CatladderJob } from "../types/jobs";
@@ -17,7 +18,9 @@ export type DeployTypeDefinition<T extends DeployConfigType> = {
   defaults: (
     envContext: EnvironmentContext<any, T>
   ) => PartialDeep<DeployConfigGeneric<T>>;
-  additionalSecretKeys: (envContext: EnvironmentContext<any, T>) => string[];
+  additionalSecretKeys: (
+    envContext: EnvironmentContext<any, T>
+  ) => SecretEnvVar[];
   getAdditionalEnvVars: (
     envContext: EnvironmentContext<any, T>
   ) => Record<string, string | undefined | null>;

package/src/deploy/kubernetes/additionalSecretKeys.ts

@@ -19,5 +19,5 @@ export const additionalKubernetesSecretKeys = ({
     keys.push("cloudsqlProxyCredentials");
   }
 
-  return keys;
+  return keys.map((key) => ({ key }));
 };

package/src/deploy/kubernetes/cloudSql/index.ts

@@ -1,5 +1,6 @@
 import { getSecretVarNameForContext } from "../../../context";
 import type { Context } from "../../../types";
+import { getFullDbName } from "../../cloudSql/utils";
 import { isOfDeployType } from "../../types";
 
 export const hasKubernetesCloudSQL = (context: Context) => {
@@ -9,20 +10,63 @@ export const hasKubernetesCloudSQL = (context: Context) => {
   return false;
 };
 
-export const getKubernetesCloudSQLConfig = (context: Context) => {
-  if (isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
-    return context.componentConfig.deploy.values?.cloudsql;
-  }
-  throw new Error("cannot get cloud sql config");
+type CloudSqlValues = {
+  enabled: boolean;
+  proxyCredentials: string;
+
+  instanceConnectionName: string;
+  fullDbName: string;
+  dbUser: string;
 };
+export const createKubernetesCloudsqlBaseValues = (
+  context: Context
+): {
+  cloudsql: CloudSqlValues;
+} => {
+  if (!isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
+    throw new Error("cannot get cloud sql config");
+  }
+
+  const config = context.componentConfig.deploy.values?.cloudsql;
 
-export const createCloudsqlBaseConfig = (context: Context) => {
-  return {
-    cloudsql: {
-      proxyCredentials: `$${getSecretVarNameForContext(
-        context,
-        "cloudsqlProxyCredentials"
-      )}`,
-    },
-  };
+  if (!config) {
+    throw new Error("cannot get cloud sql config");
+  }
+
+  const proxyCredentials = `$${getSecretVarNameForContext(
+    context,
+    "cloudsqlProxyCredentials"
+  )}`;
+  if (config?.type !== "unmanaged") {
+    const instanceConnectionName = `${config.projectId ?? "skynet-164509"}:${
+      config.region ?? "europe-west6"
+    }:${config.instanceId ?? context.environment.envVars.KUBE_NAMESPACE}`;
+
+    return {
+      cloudsql: {
+        enabled: config.enabled,
+        dbUser: "postgres",
+        instanceConnectionName,
+        proxyCredentials,
+        fullDbName: context.environment.envVars.KUBE_APP_NAME,
+      },
+    };
+  } else if (config.type === "unmanaged") {
+    return {
+      cloudsql: {
+        enabled: config.enabled,
+        dbUser: config.dbUser ?? "postgres",
+        instanceConnectionName: config.instanceConnectionName,
+        proxyCredentials,
+        fullDbName: getFullDbName(
+          config,
+          context.fullConfig,
+          context.environment.slugPrefix,
+          context.componentName
+        ),
+      },
+    };
+  } else {
+    throw new Error("unknonw type");
+  }
 };

package/src/deploy/kubernetes/kubeValues.ts

@@ -4,7 +4,10 @@ import type { Context } from "../../types/context";
 import { mergeWithMergingArrays } from "../../utils";
 import type { DeployConfigKubernetesValues } from "../types";
 import { isOfDeployType } from "../types";
-import { hasKubernetesCloudSQL, createCloudsqlBaseConfig } from "./cloudSql";
+import {
+  hasKubernetesCloudSQL,
+  createKubernetesCloudsqlBaseValues,
+} from "./cloudSql";
 
 import { createKubeEnv } from "./kubeEnv";
 import { createMongodbBaseConfig } from "./mongodb";
@@ -77,7 +80,9 @@ export const createKubeValues = (context: Context) => {
       env,
       application: createAppConfig(context, application),
     },
-    hasKubernetesCloudSQL(context) ? createCloudsqlBaseConfig(context) : {},
+    hasKubernetesCloudSQL(context)
+      ? createKubernetesCloudsqlBaseValues(context)
+      : {},
     deployConfig.values?.mongodb?.enabled
       ? createMongodbBaseConfig(context)
       : {}

package/src/deploy/types/kubernetes.ts

@@ -128,16 +128,69 @@ export type DeployConfigMongodbReplicaset = {
 export type DeployConfigMongodb = DeployConfigMongodbBase &
   (DeployConfigMongodbStandalone | DeployConfigMongodbReplicaset);
 
+/**
+ * @deprecated use config with instanceConnectionName
+ */
+type DeployConfigKubernetesValuesCloudSQLLegacy = {
+  enabled: boolean;
+  /**
+   * @deprecated use config with instanceConnectionName
+   */
+  instanceId?: string;
+  /**
+   * @deprecated use config with instanceConnectionName
+   */
+  projectId?: string;
+  /**
+   * @deprecated use config with instanceConnectionName
+   */
+  region?: string;
+  /**
+   * @deprecated use type "unmanaged".
+   *
+   * Be aware that "unmanaged" is not fully backward compatible with "legacy".
+   *
+   *
+   */
+  type?: "legacy";
+};
+
+export type DeployConfigKubernetesValuesCloudSQLUnmanaged = {
+  enabled: boolean;
+  instanceConnectionName: string;
+  /**
+   * the recommended type for the config.
+   *
+   * this mode is similar to cloudrun and declares fully-qualified db names including env.
+   *
+   * This makes it easier to share db instances across envs and dbs
+   */
+  type: "unmanaged";
+  /**
+   * the database username, defaults to "postgres"
+   */
+  dbUser?: string;
+
+  /**
+   * the prefix of the database, the full db name is this plus the environment slug prefix plus the componentName
+   *
+   * defaults to customerName-appName
+   */
+  dbNamePrefix?: string | false;
+
+  /**
+   * the base name of the db, defaults to the componentName
+   */
+  dbBaseName?: string;
+};
+export type DeployConfigKubernetesValuesCloudSQL =
+  | DeployConfigKubernetesValuesCloudSQLLegacy
+  | DeployConfigKubernetesValuesCloudSQLUnmanaged;
 export type DeployConfigKubernetesValues = AllowUnknownProps<{
   /**
    * enable cloudsql db. Currently you have to manually set it up
    */
-  cloudsql?: {
-    enabled: boolean;
-    instanceId?: string;
-    projectId?: string;
-    region?: string;
-  };
+  cloudsql?: DeployConfigKubernetesValuesCloudSQL;
   /**
    * enable mongodb. The mongodb is deployed using a helm chart.
    * See https://github.com/bitnami/charts/tree/master/bitnami/mongodb

package/src/types/context.ts

@@ -1,3 +1,4 @@
+import type { SecretEnvVar } from "../context";
 import type {
   PipelineTrigger,
   ComponentConfig,
@@ -25,7 +26,7 @@ export type Environment = {
    */
   envVars: Record<string, string>;
   envType: EnvType;
-  secretEnvVarKeys: string[];
+  secretEnvVarKeys: SecretEnvVar[];
 };
 
 export type CommitInfo = {