@malloy-publisher/server 0.0.217 → 0.0.219

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.217",
+   "version": "0.0.219",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"

package/src/dto/connection.dto.ts

@@ -1,10 +1,12 @@
 import { Type } from "class-transformer";
 import {
+   IsDefined,
    IsEnum,
    IsArray,
    IsNumber,
    IsOptional,
    IsString,
+   ValidateIf,
    ValidateNested,
 } from "class-validator";
 import "reflect-metadata";
@@ -199,6 +201,42 @@ export class DuckdbConnectionDto {
    attachedDatabases?: AttachedDatabase[];
 }
 
+export class SshProxyConfigDto {
+   @IsString()
+   host!: string;
+
+   @IsOptional()
+   @IsNumber()
+   port?: number;
+
+   @IsString()
+   username!: string;
+
+   @IsString()
+   privateKey!: string;
+
+   @IsOptional()
+   @IsString()
+   privateKeyPass?: string;
+
+   @IsOptional()
+   @IsString()
+   hostKey?: string;
+}
+
+export class ConnectionProxyDto {
+   @IsEnum(["ssh"])
+   type!: "ssh";
+
+   // ssh config is required for the ssh proxy type (the only type today); keep
+   // the conditional so a future proxy type isn't forced to supply `ssh`.
+   @ValidateIf((o) => o.type === "ssh")
+   @IsDefined()
+   @ValidateNested()
+   @Type(() => SshProxyConfigDto)
+   ssh?: SshProxyConfigDto;
+}
+
 export class ConnectionDto implements ApiConnection {
    @IsOptional()
    @IsString()
@@ -256,4 +294,9 @@ export class ConnectionDto implements ApiConnection {
    @ValidateNested()
    @Type(() => DuckdbConnectionDto)
    duckdbConnection?: DuckdbConnectionDto;
+
+   @IsOptional()
+   @ValidateNested()
+   @Type(() => ConnectionProxyDto)
+   proxy?: ConnectionProxyDto;
 }

package/src/service/connection.spec.ts

@@ -1,10 +1,12 @@
 import { DuckDBConnection } from "@malloydata/db-duckdb";
 import { afterEach, beforeEach, describe, expect, it } from "bun:test";
 import fs from "fs/promises";
+import os from "os";
 import path from "path";
 import sinon from "sinon";
 import { components } from "../api";
 import {
+   buildProxiedSslQuery,
    createEnvironmentConnections,
    testConnectionConfig,
 } from "./connection";
@@ -1846,3 +1848,71 @@ describe("connection integration tests", () => {
       );
    });
 });
+
+describe("buildProxiedSslQuery", () => {
+   const ORIG_CA = process.env.NODE_EXTRA_CA_CERTS;
+   afterEach(() => {
+      if (ORIG_CA === undefined) delete process.env.NODE_EXTRA_CA_CERTS;
+      else process.env.NODE_EXTRA_CA_CERTS = ORIG_CA;
+   });
+
+   it("defaults to no-verify when sslmode is unset (force-SSL target isn't rejected for plaintext)", () => {
+      expect(buildProxiedSslQuery("conn")).toBe("?sslmode=no-verify");
+   });
+
+   it("returns no-verify for explicit no-verify", () => {
+      expect(buildProxiedSslQuery("conn", "no-verify")).toBe(
+         "?sslmode=no-verify",
+      );
+   });
+
+   it("emits an explicit sslmode=disable, never an empty query", () => {
+      // Regression: an empty query lets pg fall back to the deployment's PGSSLMODE
+      // env, which would verify and fail the 127.0.0.1 hostname check.
+      expect(buildProxiedSslQuery("conn", "disable")).toBe("?sslmode=disable");
+   });
+
+   it("builds a chain-verifying, hostname-skipped query for verify-ca", () => {
+      process.env.NODE_EXTRA_CA_CERTS = "/etc/ssl/certs/rds bundle.pem";
+      const q = buildProxiedSslQuery("conn", "verify-ca");
+      expect(q).toContain("uselibpqcompat=true");
+      expect(q).toContain("sslmode=verify-ca");
+      expect(q).toContain(
+         `sslrootcert=${encodeURIComponent("/etc/ssl/certs/rds bundle.pem")}`,
+      );
+   });
+
+   it("throws for verify-ca when no CA bundle is available", () => {
+      delete process.env.NODE_EXTRA_CA_CERTS;
+      expect(() => buildProxiedSslQuery("conn", "verify-ca")).toThrow(
+         /no trusted CA bundle/i,
+      );
+   });
+
+   it("throws for an unsupported sslmode", () => {
+      expect(() => buildProxiedSslQuery("conn", "require")).toThrow(
+         /unsupported sslmode/i,
+      );
+   });
+
+   it("verify-ca connectionString parses to an ssl config (pg-connection-string version guard)", async () => {
+      // uselibpqcompat=true (pg-connection-string >= 2.7) is what turns verify-ca
+      // into "verify chain, skip hostname". If pg is ever downgraded below that
+      // floor, sslmode would be ignored — this guard fails loudly instead.
+      const caPath = path.join(os.tmpdir(), `proxy-ca-${process.pid}.pem`);
+      await fs.writeFile(
+         caPath,
+         "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n",
+      );
+      process.env.NODE_EXTRA_CA_CERTS = caPath;
+      try {
+         const q = buildProxiedSslQuery("conn", "verify-ca");
+         const { parse } = await import("pg-connection-string");
+         const parsed = parse(`postgresql://127.0.0.1:5432/db${q}`);
+         expect(parsed.ssl).toBeTruthy();
+         expect(typeof parsed.ssl).toBe("object");
+      } finally {
+         await fs.rm(caPath, { force: true });
+      }
+   });
+});

package/src/service/connection.ts

@@ -6,7 +6,10 @@ import "@malloydata/db-duckdb/native";
 import "@malloydata/db-mysql";
 import type { MySQLConnection } from "@malloydata/db-mysql";
 import "@malloydata/db-postgres";
-import type { PostgresConnection } from "@malloydata/db-postgres";
+import {
+   PooledPostgresConnection,
+   type PostgresConnection,
+} from "@malloydata/db-postgres";
 // Registers the "publisher" connection type (proxies SQL to a remote Publisher
 // dataplane). No live-class branch is needed in lookupConnection — the default
 // path resolves it through the registry like any other registered type.
@@ -41,6 +44,7 @@ import {
    normalizeSnowflakePrivateKey,
 } from "./connection_config";
 import { CloudStorageCredentials } from "./gcs_s3_utils";
+import { openProxy, type ProxyEndpoint } from "./proxy";
 
 type AttachedDatabase = components["schemas"]["AttachedDatabase"];
 type ApiConnection = components["schemas"]["Connection"];
@@ -921,6 +925,75 @@ function buildSnowflakePrivateKeyConnection(
    });
 }
 
+// TLS mode for a proxied postgres connection, as `pg` connectionString query
+// params. @malloydata/db-postgres exposes no ssl config, but it forwards a
+// connectionString straight to pg — which parses sslmode. The tunnel terminates
+// at 127.0.0.1, so pg's hostname check can't apply: `verify-ca` validates the
+// cert chain against the trusted CA bundle (NODE_EXTRA_CA_CERTS, e.g. the baked
+// Amazon RDS roots) WITHOUT the hostname; `no-verify` encrypts without checking;
+// `disable` uses no TLS. Full verify-full through a tunnel needs a servername
+// override (a raw ssl object) — awaits an upstream passthrough (malloydata/malloy#2960).
+export function buildProxiedSslQuery(name: string, sslmode?: string): string {
+   const caBundle = process.env.NODE_EXTRA_CA_CERTS;
+   // Default: encrypt (no-verify) so a force-SSL target (the common RDS case)
+   // isn't rejected for plaintext. Operators opt up to verify-ca when the
+   // target's CA is in the trust bundle, or down to disable for non-TLS DBs.
+   const mode = sslmode ?? "no-verify";
+   switch (mode) {
+      case "disable":
+         // Explicit: with no sslmode in the connectionString, pg falls back to
+         // the PGSSLMODE env (set on the non-proxied path), which would verify
+         // and fail the 127.0.0.1 hostname check. Force plaintext explicitly.
+         return "?sslmode=disable";
+      case "no-verify":
+         return "?sslmode=no-verify";
+      case "verify-ca":
+         // Env-set backstop; validateConnectionShape does the readable-file check
+         // at config load (so a bad bundle fails fast instead of at connect time).
+         if (!caBundle) {
+            throw new Error(
+               `Connection proxy on '${name}' uses sslmode 'verify-ca' but no trusted CA bundle is available (NODE_EXTRA_CA_CERTS is unset). Add the CA bundle to the image or use sslmode 'no-verify'.`,
+            );
+         }
+         return `?uselibpqcompat=true&sslmode=verify-ca&sslrootcert=${encodeURIComponent(caBundle)}`;
+      default:
+         throw new Error(
+            `Connection proxy on '${name}' has unsupported sslmode '${mode}' (expected disable | no-verify | verify-ca).`,
+         );
+   }
+}
+
+function buildProxiedPostgresConnection(
+   metadata: EnvironmentConnectionMetadata,
+   endpoint: ProxyEndpoint,
+): PooledPostgresConnection {
+   const name = metadata.apiConnection.name!;
+   const pg = metadata.apiConnection.postgresConnection;
+   if (!pg) {
+      throw new Error(
+         `Proxied connection '${name}' has type 'postgres' but no postgresConnection config.`,
+      );
+   }
+   // Build a connectionString to the LOCAL tunnel endpoint carrying the TLS mode.
+   // This is NOT the tenant's connectionString (rejected in validateConnectionShape
+   // because it can't be tunneled) — we construct it ourselves and it targets
+   // 127.0.0.1:<localport>, so there's no ambiguity about where it connects.
+   const enc = encodeURIComponent;
+   const auth = pg.userName
+      ? `${enc(pg.userName)}${pg.password ? `:${enc(pg.password)}` : ""}@`
+      : "";
+   const db = pg.databaseName ? `/${enc(pg.databaseName)}` : "";
+   const ssl = buildProxiedSslQuery(name, pg.sslmode);
+   const connectionString = `postgresql://${auth}${endpoint.host}:${endpoint.port}${db}${ssl}`;
+   return new PooledPostgresConnection({
+      name,
+      connectionString,
+      // Pool sizing mirrors buildSnowflakePrivateKeyConnection.
+      poolMin: 1,
+      poolMax: 20,
+   });
+}
+
 function buildDuckLakeConnection(
    metadata: EnvironmentConnectionMetadata,
    entry: CoreConnectionEntry,
@@ -987,6 +1060,11 @@ export function buildEnvironmentMalloyConfig(
    const duckLakeCache = new Map<string, Promise<DuckLakeConnection>>();
    const snowflakeJwtCache = new Map<string, Promise<SnowflakeConnection>>();
    const azureDuckDBCache = new Map<string, Promise<AzureDuckDBConnection>>();
+   const proxyConnectionCache = new Map<
+      string,
+      Promise<PooledPostgresConnection>
+   >();
+   const proxyEndpoints = new Map<string, ProxyEndpoint>();
    const attachPromises = new WeakMap<Connection, Promise<void>>();
 
    const malloyConfig = new MalloyConfig(assembled.pojo, {
@@ -1044,6 +1122,64 @@ export function buildEnvironmentMalloyConfig(
                return connection;
             }
 
+            if (metadata?.proxy) {
+               let connectionPromise = proxyConnectionCache.get(name!);
+               if (!connectionPromise) {
+                  const connType = metadata.apiConnection.type;
+                  if (connType !== "postgres") {
+                     throw new Error(
+                        `SSH proxy is not yet supported for connection type '${connType}'. ` +
+                           `Only 'postgres' is implemented.`,
+                     );
+                  }
+                  // validateConnectionShape already enforces explicit host/port
+                  // at config load (the connectionString form can't be tunneled);
+                  // this is a defense-in-depth guard for any path that reaches
+                  // lookup without that validation.
+                  const pgConfig = metadata.apiConnection.postgresConnection;
+                  if (!pgConfig?.host || !pgConfig?.port) {
+                     throw new Error(
+                        `SSH proxy for connection '${name}' requires explicit host and ` +
+                           `port on the postgres connection; the connectionString form ` +
+                           `is not supported with a proxy.`,
+                     );
+                  }
+                  connectionPromise = (async () => {
+                     const endpoint = await openProxy(metadata.proxy!, {
+                        host: pgConfig.host!,
+                        port: pgConfig.port!,
+                     });
+                     try {
+                        const connection = buildProxiedPostgresConnection(
+                           metadata,
+                           endpoint,
+                        );
+                        // Register only after the build succeeds: a throw between
+                        // openProxy and here would otherwise leave an orphaned
+                        // tunnel in proxyEndpoints that a retry can't reach (the
+                        // outer .catch evicts the cache but not the endpoint).
+                        proxyEndpoints.set(name!, endpoint);
+                        return connection;
+                     } catch (err) {
+                        await endpoint.close().catch(() => {});
+                        throw err;
+                     }
+                  })();
+                  // Drop a rejected build from the cache so a later lookup can
+                  // retry, rather than replaying a transient SSH failure (bastion
+                  // restart, network blip) until the environment is rebuilt.
+                  connectionPromise.catch(() => {
+                     if (
+                        proxyConnectionCache.get(name!) === connectionPromise
+                     ) {
+                        proxyConnectionCache.delete(name!);
+                     }
+                  });
+                  proxyConnectionCache.set(name!, connectionPromise);
+               }
+               return connectionPromise;
+            }
+
             if (metadata?.hasSnowflakePrivateKey) {
                let connectionPromise = snowflakeJwtCache.get(name!);
                if (!connectionPromise) {
@@ -1086,6 +1222,7 @@ export function buildEnvironmentMalloyConfig(
             ...duckLakeCache.values(),
             ...snowflakeJwtCache.values(),
             ...azureDuckDBCache.values(),
+            ...proxyConnectionCache.values(),
          ];
          const closeResults = await Promise.allSettled([
             malloyConfig.shutdown("close"),
@@ -1094,11 +1231,21 @@ export function buildEnvironmentMalloyConfig(
                await connection.close();
             }),
          ]);
+         // Close proxy tunnels only AFTER the build promises above have settled:
+         // openProxy may still be in flight during a teardown race, and the
+         // endpoint is registered inside that promise — capturing the map
+         // earlier would miss it and leak the tunnel. Closing after the DB
+         // connections also keeps the tunnel alive while they drain.
+         const endpointResults = await Promise.allSettled(
+            [...proxyEndpoints.values()].map((ep) => ep.close()),
+         );
          duckLakeCache.clear();
          snowflakeJwtCache.clear();
          azureDuckDBCache.clear();
+         proxyConnectionCache.clear();
+         proxyEndpoints.clear();
 
-         const failures = closeResults.filter(
+         const failures = [...closeResults, ...endpointResults].filter(
             (result): result is PromiseRejectedResult =>
                result.status === "rejected",
          );

package/src/service/connection_config.spec.ts

@@ -349,3 +349,257 @@ describe("assembleEnvironmentConnections — publisher", () => {
       });
    });
 });
+
+describe("SSH proxy validation", () => {
+   const validSshProxy: ApiConnection = {
+      name: "pg-via-bastion",
+      type: "postgres",
+      postgresConnection: {
+         host: "127.0.0.1",
+         port: 5432,
+         databaseName: "mydb",
+         userName: "user",
+         password: "pass",
+      },
+      proxy: {
+         type: "ssh",
+         ssh: {
+            host: "bastion.example.com",
+            username: "ec2-user",
+            privateKey:
+               "-----BEGIN OPENSSH PRIVATE KEY-----\nfake\n-----END OPENSSH PRIVATE KEY-----",
+         },
+      },
+   };
+
+   it("allows a postgres+ssh-proxy connection with no env flag required", () => {
+      // The SSH proxy is a normal connection capability — deliberately NOT gated
+      // by PUBLISHER_ALLOW_PROXY_CONNECTIONS (that flag is for the `publisher`
+      // HTTP multi-hop type). Prove it assembles with the flag unset.
+      const priorFlag = process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS;
+      delete process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS;
+      try {
+         const { pojo } = assembleEnvironmentConnections([validSshProxy]);
+         expect(pojo.connections["pg-via-bastion"].is).toBe("postgres");
+      } finally {
+         if (priorFlag === undefined) {
+            delete process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS;
+         } else {
+            process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS = priorFlag;
+         }
+      }
+   });
+
+   it("rejects a non-postgres (bigquery) connection with a proxy", () => {
+      const conn: ApiConnection = {
+         name: "bq-via-bastion",
+         type: "bigquery",
+         bigqueryConnection: {
+            serviceAccountKeyJson: JSON.stringify({
+               type: "service_account",
+               project_id: "proj",
+               private_key: "key",
+               client_email: "sa@proj.iam.gserviceaccount.com",
+            }),
+         },
+         proxy: {
+            type: "ssh",
+            ssh: {
+               host: "bastion.example.com",
+               username: "ec2-user",
+               privateKey: "key",
+            },
+         },
+      };
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "Connection proxy is not supported for type 'bigquery' (only 'postgres' today)",
+      );
+   });
+
+   it("rejects an ssh proxy with no ssh config object", () => {
+      const conn: ApiConnection = {
+         name: "pg-no-ssh-config",
+         type: "postgres",
+         postgresConnection: {
+            host: "127.0.0.1",
+            databaseName: "mydb",
+         },
+         proxy: {
+            type: "ssh",
+         },
+      };
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "Connection proxy on 'pg-no-ssh-config' has type 'ssh' but no 'ssh' config object",
+      );
+   });
+
+   it("rejects a proxied postgres connection that also carries a connectionString", () => {
+      const conn: ApiConnection = {
+         ...validSshProxy,
+         name: "pg-with-connstring",
+         postgresConnection: {
+            connectionString: "postgresql://real-db.example.com:5432/mydb",
+            host: "127.0.0.1",
+            port: 5432,
+            databaseName: "mydb",
+         },
+      };
+      // The tunnel forwards to host/port; a connectionString would be silently
+      // ignored and could point at a different database, so it's rejected.
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "does not support the connectionString form",
+      );
+   });
+
+   it("rejects a proxied postgres connection missing explicit host/port", () => {
+      const conn: ApiConnection = {
+         ...validSshProxy,
+         name: "pg-no-host-port",
+         postgresConnection: {
+            databaseName: "mydb",
+            userName: "user",
+         },
+      };
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "requires explicit host and port",
+      );
+   });
+
+   it("accepts a multi-line hostKey and sslmode=no-verify", () => {
+      const conn: ApiConnection = {
+         ...validSshProxy,
+         postgresConnection: {
+            ...validSshProxy.postgresConnection!,
+            sslmode: "no-verify",
+         },
+         proxy: {
+            type: "ssh",
+            ssh: {
+               ...validSshProxy.proxy!.ssh!,
+               hostKey:
+                  "bastion ssh-ed25519 AAAAdecoy\nbastion ssh-rsa AAAAreal",
+            },
+         },
+      };
+      expect(() => assembleEnvironmentConnections([conn])).not.toThrow();
+   });
+
+   it("rejects a hostKey that parses to zero keys (comment/blank only) — fail-closed at config load", () => {
+      const conn: ApiConnection = {
+         ...validSshProxy,
+         proxy: {
+            type: "ssh",
+            ssh: {
+               ...validSshProxy.proxy!.ssh!,
+               hostKey: "# ssh-keyscan bastion timed out\n   \n",
+            },
+         },
+      };
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "no usable host-key line",
+      );
+   });
+
+   it("rejects a whitespace-only hostKey (must not silently connect unpinned)", () => {
+      const conn: ApiConnection = {
+         ...validSshProxy,
+         proxy: {
+            type: "ssh",
+            ssh: {
+               ...validSshProxy.proxy!.ssh!,
+               hostKey: "   ",
+            },
+         },
+      };
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "no usable host-key line",
+      );
+   });
+
+   it("treats an empty-string hostKey as unpinned (omission-equivalent)", () => {
+      const conn: ApiConnection = {
+         ...validSshProxy,
+         proxy: {
+            type: "ssh",
+            ssh: { ...validSshProxy.proxy!.ssh!, hostKey: "" },
+         },
+      };
+      expect(() => assembleEnvironmentConnections([conn])).not.toThrow();
+   });
+
+   it("rejects an unsupported sslmode", () => {
+      const conn = {
+         ...validSshProxy,
+         postgresConnection: {
+            ...validSshProxy.postgresConnection!,
+            sslmode: "require",
+         },
+      } as unknown as ApiConnection;
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "unsupported sslmode 'require'",
+      );
+   });
+
+   it("rejects an empty-string sslmode at config load (not deferred to connect)", () => {
+      const conn = {
+         ...validSshProxy,
+         postgresConnection: {
+            ...validSshProxy.postgresConnection!,
+            sslmode: "",
+         },
+      } as unknown as ApiConnection;
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "unsupported sslmode",
+      );
+   });
+
+   it("rejects sslmode=verify-ca when no CA bundle is available", () => {
+      const prior = process.env.NODE_EXTRA_CA_CERTS;
+      delete process.env.NODE_EXTRA_CA_CERTS;
+      try {
+         const conn: ApiConnection = {
+            ...validSshProxy,
+            postgresConnection: {
+               ...validSshProxy.postgresConnection!,
+               sslmode: "verify-ca",
+            },
+         };
+         expect(() => assembleEnvironmentConnections([conn])).toThrow(
+            "verify-ca",
+         );
+      } finally {
+         if (prior !== undefined) process.env.NODE_EXTRA_CA_CERTS = prior;
+      }
+   });
+
+   it("rejects sslmode set on a non-proxied connection (silent-ignore footgun)", () => {
+      const conn: ApiConnection = {
+         name: "pg-direct",
+         type: "postgres",
+         postgresConnection: {
+            host: "db.example.com",
+            port: 5432,
+            databaseName: "mydb",
+            userName: "user",
+            password: "pass",
+            sslmode: "verify-ca",
+         },
+      };
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "only supported for proxied connections",
+      );
+   });
+
+   it("rejects a proxied database name with URI-reserved characters", () => {
+      const conn: ApiConnection = {
+         ...validSshProxy,
+         postgresConnection: {
+            ...validSshProxy.postgresConnection!,
+            databaseName: "sales@prod",
+         },
+      };
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "URI-reserved characters",
+      );
+   });
+});