@malloy-publisher/server 0.0.217 → 0.0.219

package/src/service/proxy.spec.ts

@@ -0,0 +1,414 @@
+/**
+ * Unit tests for the SSH proxy layer.
+ *
+ * These tests are hermetic: they stand up an in-process ssh2 server that
+ * forwards to a plain TCP echo server, call openProxy(), connect a client
+ * to the returned local endpoint, and assert that bytes round-trip through
+ * the tunnel.
+ *
+ * No external network access is required.
+ */
+
+import net from "net";
+import { generateKeyPairSync } from "crypto";
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import {
+   Server as SshServer,
+   utils as sshUtils,
+   type Connection as SshServerConnection,
+} from "ssh2";
+import { openProxy } from "./proxy";
+
+// ── Key material generated once for the test suite ────────────────────────────
+
+// ssh2's server requires PEM private keys it can parse. These are ephemeral,
+// test-only keys; RSA-2048 (not 1024) so static analysis doesn't flag them.
+const hostKeys = generateKeyPairSync("rsa", { modulusLength: 2048 });
+const hostPrivatePem = hostKeys.privateKey
+   .export({ type: "pkcs1", format: "pem" })
+   .toString();
+
+const clientKeys = generateKeyPairSync("rsa", { modulusLength: 2048 });
+const clientPrivatePem = clientKeys.privateKey
+   .export({ type: "pkcs1", format: "pem" })
+   .toString();
+// Helpers ─────────────────────────────────────────────────────────────────────
+
+function startEchoServer(): Promise<{
+   port: number;
+   close: () => Promise<void>;
+}> {
+   return new Promise((resolve, reject) => {
+      // Track accepted sockets so close() can force them down. server.close()
+      // alone only stops accepting and then waits for open connections to end;
+      // a lingering forwarded socket would otherwise hang teardown (and the
+      // afterEach hook) until the test timeout, especially under --serial CI.
+      const sockets = new Set<net.Socket>();
+      const server = net.createServer((socket) => {
+         sockets.add(socket);
+         socket.on("close", () => sockets.delete(socket));
+         socket.pipe(socket);
+      });
+      server.on("error", reject);
+      server.listen(0, "127.0.0.1", () => {
+         const addr = server.address() as net.AddressInfo;
+         resolve({
+            port: addr.port,
+            close: () =>
+               new Promise((res, rej) => {
+                  for (const s of sockets) s.destroy();
+                  server.close((err) => (err ? rej(err) : res()));
+               }),
+         });
+      });
+   });
+}
+
+/**
+ * Start a minimal in-process SSH server that accepts the test client key
+ * and fulfils tcpip-forward (forwardOut) requests.
+ */
+function startSshServer(opts: {
+   rejectAuth?: boolean;
+   rejectForward?: boolean;
+}): Promise<{
+   port: number;
+   hostKeyBase64: string;
+   close: () => Promise<void>;
+}> {
+   return new Promise((resolve, reject) => {
+      // Track live client connections and forwarded destination sockets so
+      // close() can force them down; otherwise a lingering forwarded socket
+      // keeps sshd.close() (and the afterEach hook) hanging until the test
+      // timeout, which is what made the tunnel tests flake under --serial CI.
+      const clients = new Set<SshServerConnection>();
+      const destSockets = new Set<net.Socket>();
+      const sshd = new SshServer(
+         { hostKeys: [hostPrivatePem] },
+         (client: SshServerConnection) => {
+            clients.add(client);
+            client.on("close", () => clients.delete(client));
+            client.on("authentication", (ctx) => {
+               if (opts.rejectAuth) {
+                  ctx.reject();
+                  return;
+               }
+               // Accept publickey auth for our test client key.
+               if (ctx.method === "publickey" && !opts.rejectAuth) {
+                  ctx.accept();
+               } else {
+                  ctx.reject();
+               }
+            });
+
+            client.on("ready", () => {
+               client.on("tcpip", (accept, _reject, info) => {
+                  if (opts.rejectForward) {
+                     _reject();
+                     return;
+                  }
+                  const channel = accept();
+                  // Connect to the real destination and pipe.
+                  const dest = net.createConnection(
+                     { host: info.destIP, port: info.destPort },
+                     () => {
+                        channel.pipe(dest).pipe(channel);
+                        channel.on("error", () => dest.destroy());
+                        dest.on("error", () => channel.destroy());
+                     },
+                  );
+                  destSockets.add(dest);
+                  dest.on("close", () => destSockets.delete(dest));
+                  dest.on("error", () => channel.destroy());
+               });
+            });
+
+            client.on("error", () => {
+               /* swallow per-client errors in tests */
+            });
+         },
+      );
+
+      sshd.on("error", reject);
+      sshd.listen(0, "127.0.0.1", () => {
+         const addr = sshd.address() as net.AddressInfo;
+
+         // Capture the server's host key fingerprint (base64 of the raw DER) by
+         // parsing the pem via the ssh2 util so the format matches what
+         // hostVerifier receives.
+         const parsed = sshUtils.parseKey(hostPrivatePem);
+         const hostKeyBase64 = (parsed as { getPublicSSH(): Buffer })
+            .getPublicSSH()
+            .toString("base64");
+
+         resolve({
+            port: addr.port,
+            hostKeyBase64,
+            close: () =>
+               new Promise((res) => {
+                  for (const s of destSockets) s.destroy();
+                  for (const c of clients) c.end();
+                  sshd.close(() => res());
+               }),
+         });
+      });
+   });
+}
+
+function connectAndSend(port: number, message: string): Promise<string> {
+   return new Promise((resolve, reject) => {
+      const socket = net.createConnection({ host: "127.0.0.1", port }, () => {
+         socket.write(message);
+      });
+      const chunks: Buffer[] = [];
+      socket.on("data", (chunk) => {
+         chunks.push(chunk);
+         if (Buffer.concat(chunks).toString().includes(message)) {
+            socket.destroy();
+            resolve(Buffer.concat(chunks).toString());
+         }
+      });
+      socket.on("error", reject);
+   });
+}
+
+// Best-effort, time-bounded teardown. Destroying every ssh2/socket handle is
+// reliable locally but a lingering graceful close can still stall on some CI
+// platforms (Bun + ssh2 on Linux/Windows), which would hang the afterEach hook
+// until the 100s test timeout. The assertions have already run by teardown, so
+// cap each close and move on rather than block.
+function closeQuietly(close: () => Promise<void>, ms = 3000): Promise<void> {
+   return Promise.race([
+      close().catch(() => {}),
+      new Promise<void>((resolve) => {
+         const timer = setTimeout(resolve, ms);
+         (timer as unknown as { unref?: () => void }).unref?.();
+      }),
+   ]);
+}
+
+// ── Tests ─────────────────────────────────────────────────────────────────────
+
+describe("openProxy — SSH tunnel", () => {
+   let echoServer: { port: number; close: () => Promise<void> };
+   let sshServer: {
+      port: number;
+      hostKeyBase64: string;
+      close: () => Promise<void>;
+   };
+
+   beforeEach(async () => {
+      echoServer = await startEchoServer();
+      sshServer = await startSshServer({});
+   });
+
+   afterEach(async () => {
+      await closeQuietly(() => echoServer.close());
+      await closeQuietly(() => sshServer.close());
+   });
+
+   it("round-trips bytes through the tunnel", async () => {
+      const ep = await openProxy(
+         {
+            type: "ssh",
+            ssh: {
+               host: "127.0.0.1",
+               port: sshServer.port,
+               username: "testuser",
+               privateKey: clientPrivatePem,
+               hostKey: sshServer.hostKeyBase64,
+            },
+         },
+         { host: "127.0.0.1", port: echoServer.port },
+      );
+
+      try {
+         const reply = await connectAndSend(ep.port, "hello-proxy");
+         expect(reply).toContain("hello-proxy");
+      } finally {
+         await closeQuietly(() => ep.close());
+      }
+   });
+
+   it("accepts a hostKey given as a full known_hosts line", async () => {
+      // Users typically paste `ssh-keyscan` output, e.g.
+      // `bastion.example.com ssh-rsa AAAA…` — the verifier must extract the blob.
+      const knownHostsLine = `bastion.example.com ssh-rsa ${sshServer.hostKeyBase64}`;
+      const ep = await openProxy(
+         {
+            type: "ssh",
+            ssh: {
+               host: "127.0.0.1",
+               port: sshServer.port,
+               username: "testuser",
+               privateKey: clientPrivatePem,
+               hostKey: knownHostsLine,
+            },
+         },
+         { host: "127.0.0.1", port: echoServer.port },
+      );
+
+      try {
+         const reply = await connectAndSend(ep.port, "known-hosts-line");
+         expect(reply).toContain("known-hosts-line");
+      } finally {
+         await closeQuietly(() => ep.close());
+      }
+   });
+
+   it("accepts a hashed (|1|) known_hosts line — the hash only obscures the hostname", async () => {
+      // `ssh-keyscan -H` output hashes the hostname (`|1|salt|hash`) but leaves
+      // the key blob in the clear; the verifier extracts the blob token and never
+      // matches hostnames, so hashed lines work the same as unhashed ones.
+      const hashedLine = `|1|dNQdBg9dg8u7Vw8vq3B0uZ3example=|kZ2exampleHashValue0000000= ssh-rsa ${sshServer.hostKeyBase64}`;
+      const ep = await openProxy(
+         {
+            type: "ssh",
+            ssh: {
+               host: "127.0.0.1",
+               port: sshServer.port,
+               username: "testuser",
+               privateKey: clientPrivatePem,
+               hostKey: hashedLine,
+            },
+         },
+         { host: "127.0.0.1", port: echoServer.port },
+      );
+
+      try {
+         const reply = await connectAndSend(ep.port, "hashed-line");
+         expect(reply).toContain("hashed-line");
+      } finally {
+         await closeQuietly(() => ep.close());
+      }
+   });
+
+   it("rejects when hostKey does not match", async () => {
+      const wrongKey = Buffer.alloc(32, 0xff).toString("base64");
+
+      await expect(
+         openProxy(
+            {
+               type: "ssh",
+               ssh: {
+                  host: "127.0.0.1",
+                  port: sshServer.port,
+                  username: "testuser",
+                  privateKey: clientPrivatePem,
+                  hostKey: wrongKey,
+               },
+            },
+            { host: "127.0.0.1", port: echoServer.port },
+         ),
+      ).rejects.toThrow(/host-key verification failed/i);
+   });
+
+   it("connects when hostKey is absent — unpinned self-service default", async () => {
+      // No hostKey → connect without host-key verification (the SSH transport is
+      // still encrypted). Matches mainstream BI tools' SSH-tunnel default.
+      const ep = await openProxy(
+         {
+            type: "ssh",
+            ssh: {
+               host: "127.0.0.1",
+               port: sshServer.port,
+               username: "testuser",
+               privateKey: clientPrivatePem,
+            },
+         },
+         { host: "127.0.0.1", port: echoServer.port },
+      );
+      try {
+         const reply = await connectAndSend(ep.port, "unpinned");
+         expect(reply).toContain("unpinned");
+      } finally {
+         await closeQuietly(() => ep.close());
+      }
+   });
+
+   it("accepts a multi-line hostKey listing several keys — matches any (LB bastion)", async () => {
+      // A load-balanced bastion presents a different host key per backend, so the
+      // pin lists every backend's key and any match is accepted.
+      const decoy = Buffer.alloc(32, 0xff).toString("base64");
+      const multiKey = [
+         `bastion.example.com ssh-ed25519 ${decoy}`,
+         `bastion.example.com ssh-rsa ${sshServer.hostKeyBase64}`,
+      ].join("\n");
+      const ep = await openProxy(
+         {
+            type: "ssh",
+            ssh: {
+               host: "127.0.0.1",
+               port: sshServer.port,
+               username: "testuser",
+               privateKey: clientPrivatePem,
+               hostKey: multiKey,
+            },
+         },
+         { host: "127.0.0.1", port: echoServer.port },
+      );
+      try {
+         const reply = await connectAndSend(ep.port, "multi-key");
+         expect(reply).toContain("multi-key");
+      } finally {
+         await closeQuietly(() => ep.close());
+      }
+   });
+
+   it("fails closed when hostKey is set but parses to zero keys (comment/blank only)", async () => {
+      // A set-but-degenerate pin must NOT fall through to the unpinned branch —
+      // the security boundary is gated on hostKey being configured, not on it
+      // parsing to >=1 key. (Config load rejects this earlier; this guards the
+      // boundary itself.)
+      const commentOnly = "# ssh-keyscan bastion.example.com timed out\n   \n";
+      await expect(
+         openProxy(
+            {
+               type: "ssh",
+               ssh: {
+                  host: "127.0.0.1",
+                  port: sshServer.port,
+                  username: "testuser",
+                  privateKey: clientPrivatePem,
+                  hostKey: commentOnly,
+               },
+            },
+            { host: "127.0.0.1", port: echoServer.port },
+         ),
+      ).rejects.toThrow(/host-key verification failed/i);
+   });
+
+   it("rejects a multi-line hostKey when none of the listed keys match", async () => {
+      const decoy1 = Buffer.alloc(32, 0xff).toString("base64");
+      const decoy2 = Buffer.alloc(32, 0xaa).toString("base64");
+      const multiKey = [
+         `bastion.example.com ssh-ed25519 ${decoy1}`,
+         `bastion.example.com ssh-rsa ${decoy2}`,
+      ].join("\n");
+      await expect(
+         openProxy(
+            {
+               type: "ssh",
+               ssh: {
+                  host: "127.0.0.1",
+                  port: sshServer.port,
+                  username: "testuser",
+                  privateKey: clientPrivatePem,
+                  hostKey: multiKey,
+               },
+            },
+            { host: "127.0.0.1", port: echoServer.port },
+         ),
+      ).rejects.toThrow(/host-key verification failed/i);
+   });
+
+   it("rejects for unsupported proxy type", async () => {
+      await expect(
+         openProxy(
+            // @ts-expect-error — intentionally passing an unsupported type
+            { type: "unsupported" },
+            { host: "127.0.0.1", port: echoServer.port },
+         ),
+      ).rejects.toThrow(/not supported/i);
+   });
+});

package/src/service/proxy.ts

@@ -0,0 +1,248 @@
+/**
+ * Generic TCP proxy layer for publisher database connections.
+ *
+ * Currently supports SSH bastion tunnels (type "ssh").  Future connection
+ * types (mysql, trino, …) only need a new `openProxy` branch — the caller
+ * interface is unchanged.
+ *
+ * A connection proxy is a normal connection capability (authorized by whoever
+ * configures the connection); it is intentionally NOT behind an env-flag gate,
+ * and is kept separate from the `publisher` HTTP multi-hop type's
+ * PUBLISHER_ALLOW_PROXY_CONNECTIONS gate. When host-key pinning is configured it
+ * is fail-closed (below).
+ *
+ * # Host-key policy
+ *
+ * `ssh.hostKey` is optional. When set, it pins the bastion's host key(s) and the
+ * tunnel is fail-closed on mismatch; it may list multiple known_hosts lines (a
+ * load-balanced/HA bastion presents a different key per backend), and any listed
+ * key is accepted. When absent, the tunnel connects without host-key
+ * verification — the self-service default, matching mainstream BI tools' SSH
+ * tunnels. The SSH transport is still encrypted; unpinned, a MITM on the
+ * publisher→bastion hop is possible, mitigated by the customer allowlisting our
+ * egress on the bastion.
+ */
+
+import net from "net";
+import { Client as SshClient } from "ssh2";
+import type { ConnectConfig, SyncHostVerifier } from "ssh2";
+import { components } from "../api";
+import { logger } from "../logger";
+
+type ConnectionProxy = components["schemas"]["ConnectionProxy"];
+
+export interface ProxyEndpoint {
+   host: string;
+   port: number;
+   close(): Promise<void>;
+}
+
+const SSH_CONNECT_TIMEOUT_MS = 15_000;
+const SSH_KEEPALIVE_INTERVAL_MS = 15_000;
+
+/**
+ * Parse a stored hostKey into the set of base64 wire-format key blobs to accept.
+ * The value may hold one or more OpenSSH known_hosts entries (one per line) — a
+ * load-balanced bastion presents a different host key per backend, so pinning
+ * such a bastion means listing every backend's key. Blank lines and `#` comments
+ * are skipped. Each entry may be a full known_hosts line (`[markers] host keytype
+ * AAAA…`), a `keytype AAAA…` pair, or a bare base64 blob; SSH key blobs
+ * base64-encode to a string starting with "AAAA" (the length prefix of the
+ * key-type name), so we pick that token, else the last whitespace-delimited
+ * token. Compared against what ssh2 hands us (`key.toString("base64")`).
+ */
+export function parseHostKeys(raw: string): Set<string> {
+   const keys = new Set<string>();
+   for (const line of raw.split(/\r?\n/)) {
+      const trimmed = line.trim();
+      if (!trimmed || trimmed.startsWith("#")) continue;
+      const tokens = trimmed.split(/\s+/);
+      const blob = tokens.find((t) => /^AAAA[A-Za-z0-9+/]+={0,2}$/.test(t));
+      const key = blob ?? tokens[tokens.length - 1];
+      if (key) keys.add(key);
+   }
+   return keys;
+}
+
+/**
+ * Open a proxy tunnel described by `proxy` that ultimately delivers traffic to
+ * `target`.  Returns a local `127.0.0.1:port` endpoint the DB driver should
+ * connect to in place of the real host/port.
+ */
+export async function openProxy(
+   proxy: ConnectionProxy,
+   target: { host: string; port: number },
+): Promise<ProxyEndpoint> {
+   if (proxy.type === "ssh") {
+      if (!proxy.ssh) {
+         throw new Error(
+            "ConnectionProxy type is 'ssh' but the 'ssh' config object is missing.",
+         );
+      }
+      return openSshProxy(proxy.ssh, target);
+   }
+   throw new Error(
+      `Proxy type '${proxy.type}' is not supported yet.  Only 'ssh' is implemented.`,
+   );
+}
+
+function openSshProxy(
+   ssh: components["schemas"]["SshProxyConfig"],
+   target: { host: string; port: number },
+): Promise<ProxyEndpoint> {
+   return new Promise((resolve, reject) => {
+      const client = new SshClient();
+      let localPort = 0;
+      let server: net.Server | undefined;
+      let settled = false;
+      // Track accepted local sockets so close() can force them down. Relying on
+      // server.close() alone waits for in-flight sockets to drain, which can
+      // hang teardown indefinitely if a forwarded socket lingers.
+      const sockets = new Set<net.Socket>();
+
+      function fail(err: Error): void {
+         if (settled) return;
+         settled = true;
+         server?.close();
+         client.end();
+         reject(err);
+      }
+
+      const connectConfig: ConnectConfig = {
+         host: ssh.host,
+         port: ssh.port ?? 22,
+         username: ssh.username,
+         privateKey: ssh.privateKey,
+         ...(ssh.privateKeyPass ? { passphrase: ssh.privateKeyPass } : {}),
+         readyTimeout: SSH_CONNECT_TIMEOUT_MS,
+         // Send SSH-level keepalives so an idle tunnel isn't silently reaped by
+         // Cloud NAT / bastion ClientAlive / stateful-firewall idle timeouts —
+         // which would otherwise only surface as a confusing pg error on the
+         // next query. 3 missed (~45s) before ssh2 considers the link dead.
+         keepaliveInterval: SSH_KEEPALIVE_INTERVAL_MS,
+         keepaliveCountMax: 3,
+         hostVerifier: ((key: Buffer): boolean => {
+            // `key` is the raw Buffer of the host's public key.
+            const presented = key.toString("base64");
+            // "Pinned" is gated on hostKey being a non-empty value, not on it
+            // parsing to ≥1 key: a set-but-degenerate hostKey (only whitespace,
+            // blanks, or comments) must fail closed, never fall through to the
+            // unpinned branch. Config load (validateConnectionShape) already
+            // rejects that case; this is the security-boundary backstop for any
+            // path that reaches here directly. ("" is the unpinned signal.)
+            if (ssh.hostKey) {
+               // Fail-closed, accepting any listed key — an LB/HA bastion presents
+               // a different host key per backend.
+               const pinned = parseHostKeys(ssh.hostKey);
+               if (pinned.has(presented)) {
+                  return true;
+               }
+               fail(
+                  new Error(
+                     `SSH host-key verification failed for ${ssh.host}: the presented key ` +
+                        `(${presented.slice(0, 24)}…) is not among the ${pinned.size} pinned ` +
+                        `host key(s).`,
+                  ),
+               );
+               return false;
+            }
+            // Unpinned (no hostKey): connect without host-key verification — the
+            // self-service default (see file header).
+            logger.warn(
+               `Connecting to SSH bastion ${ssh.host} without host-key verification (no hostKey pinned).`,
+            );
+            return true;
+         }) as SyncHostVerifier,
+      };
+
+      client.on("error", (err) => fail(err));
+
+      client.on("ready", () => {
+         server = net.createServer((socket) => {
+            sockets.add(socket);
+            socket.on("close", () => sockets.delete(socket));
+            // Buffer incoming data immediately so we don't lose bytes that
+            // arrive before the forwardOut channel is open (relevant in Bun
+            // where resume() doesn't re-emit already-buffered data).
+            const earlyData: Buffer[] = [];
+            socket.on("data", (chunk: Buffer) => earlyData.push(chunk));
+
+            client.forwardOut(
+               "127.0.0.1",
+               localPort,
+               target.host,
+               target.port,
+               (err, channel) => {
+                  if (err) {
+                     socket.destroy(err);
+                     return;
+                  }
+                  // The local socket may have closed while forwardOut was still
+                  // opening the channel; if so, tear the channel down instead of
+                  // attaching it to a dead socket (which would leak the remote
+                  // connection). The rest of this callback is synchronous, so no
+                  // close can slip in between this check and the listeners below.
+                  if (socket.destroyed) {
+                     channel.destroy();
+                     return;
+                  }
+                  // Flush buffered data, then switch to live forwarding.
+                  for (const chunk of earlyData) {
+                     channel.write(chunk);
+                  }
+                  earlyData.length = 0;
+                  socket.removeAllListeners("data");
+                  // pipe() honors backpressure (pauses the source when the
+                  // destination's buffer is full) — unlike a raw on('data') =>
+                  // write() which would buffer unbounded on a slow peer.
+                  socket.pipe(channel);
+                  channel.pipe(socket);
+                  socket.on("error", () => channel.destroy());
+                  channel.on("error", () => socket.destroy());
+                  // Propagate teardown both ways so neither side is left
+                  // half-open when the other closes (e.g. pool eviction or the
+                  // DB dropping the connection).
+                  socket.on("close", () => channel.destroy());
+                  channel.on("close", () => socket.destroy());
+               },
+            );
+         });
+
+         server.listen(0, "127.0.0.1", () => {
+            const addr = server!.address();
+            if (!addr || typeof addr === "string") {
+               fail(
+                  new Error(
+                     "Failed to obtain local listen port for SSH proxy.",
+                  ),
+               );
+               return;
+            }
+            localPort = addr.port;
+
+            if (settled) return;
+            settled = true;
+
+            resolve({
+               host: "127.0.0.1",
+               port: localPort,
+               close(): Promise<void> {
+                  return new Promise((res) => {
+                     // Force-destroy live forwarded sockets first so server.close()
+                     // doesn't wait on them (which would hang teardown).
+                     for (const s of sockets) s.destroy();
+                     server!.close(() => {
+                        client.end();
+                        res();
+                     });
+                  });
+               },
+            });
+         });
+
+         server.on("error", (err) => fail(err));
+      });
+
+      client.connect(connectConfig);
+   });
+}