@malloy-publisher/server 0.0.217 → 0.0.218

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.217",
+   "version": "0.0.218",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"

package/src/dto/connection.dto.ts

@@ -1,10 +1,12 @@
 import { Type } from "class-transformer";
 import {
+   IsDefined,
    IsEnum,
    IsArray,
    IsNumber,
    IsOptional,
    IsString,
+   ValidateIf,
    ValidateNested,
 } from "class-validator";
 import "reflect-metadata";
@@ -199,6 +201,42 @@ export class DuckdbConnectionDto {
    attachedDatabases?: AttachedDatabase[];
 }
 
+export class SshProxyConfigDto {
+   @IsString()
+   host!: string;
+
+   @IsOptional()
+   @IsNumber()
+   port?: number;
+
+   @IsString()
+   username!: string;
+
+   @IsString()
+   privateKey!: string;
+
+   @IsOptional()
+   @IsString()
+   privateKeyPass?: string;
+
+   @IsOptional()
+   @IsString()
+   hostKey?: string;
+}
+
+export class ConnectionProxyDto {
+   @IsEnum(["ssh"])
+   type!: "ssh";
+
+   // ssh config is required for the ssh proxy type (the only type today); keep
+   // the conditional so a future proxy type isn't forced to supply `ssh`.
+   @ValidateIf((o) => o.type === "ssh")
+   @IsDefined()
+   @ValidateNested()
+   @Type(() => SshProxyConfigDto)
+   ssh?: SshProxyConfigDto;
+}
+
 export class ConnectionDto implements ApiConnection {
    @IsOptional()
    @IsString()
@@ -256,4 +294,9 @@ export class ConnectionDto implements ApiConnection {
    @ValidateNested()
    @Type(() => DuckdbConnectionDto)
    duckdbConnection?: DuckdbConnectionDto;
+
+   @IsOptional()
+   @ValidateNested()
+   @Type(() => ConnectionProxyDto)
+   proxy?: ConnectionProxyDto;
 }

package/src/service/connection.ts

@@ -6,7 +6,10 @@ import "@malloydata/db-duckdb/native";
 import "@malloydata/db-mysql";
 import type { MySQLConnection } from "@malloydata/db-mysql";
 import "@malloydata/db-postgres";
-import type { PostgresConnection } from "@malloydata/db-postgres";
+import {
+   PooledPostgresConnection,
+   type PostgresConnection,
+} from "@malloydata/db-postgres";
 // Registers the "publisher" connection type (proxies SQL to a remote Publisher
 // dataplane). No live-class branch is needed in lookupConnection — the default
 // path resolves it through the registry like any other registered type.
@@ -41,6 +44,7 @@ import {
    normalizeSnowflakePrivateKey,
 } from "./connection_config";
 import { CloudStorageCredentials } from "./gcs_s3_utils";
+import { openProxy, type ProxyEndpoint } from "./proxy";
 
 type AttachedDatabase = components["schemas"]["AttachedDatabase"];
 type ApiConnection = components["schemas"]["Connection"];
@@ -921,6 +925,30 @@ function buildSnowflakePrivateKeyConnection(
    });
 }
 
+function buildProxiedPostgresConnection(
+   metadata: EnvironmentConnectionMetadata,
+   endpoint: ProxyEndpoint,
+): PooledPostgresConnection {
+   const name = metadata.apiConnection.name!;
+   const pg = metadata.apiConnection.postgresConnection;
+   if (!pg) {
+      throw new Error(
+         `Proxied connection '${name}' has type 'postgres' but no postgresConnection config.`,
+      );
+   }
+   return new PooledPostgresConnection({
+      name,
+      host: endpoint.host,
+      port: endpoint.port,
+      username: pg.userName,
+      password: pg.password,
+      databaseName: pg.databaseName,
+      // Pool sizing mirrors buildSnowflakePrivateKeyConnection.
+      poolMin: 1,
+      poolMax: 20,
+   });
+}
+
 function buildDuckLakeConnection(
    metadata: EnvironmentConnectionMetadata,
    entry: CoreConnectionEntry,
@@ -987,6 +1015,11 @@ export function buildEnvironmentMalloyConfig(
    const duckLakeCache = new Map<string, Promise<DuckLakeConnection>>();
    const snowflakeJwtCache = new Map<string, Promise<SnowflakeConnection>>();
    const azureDuckDBCache = new Map<string, Promise<AzureDuckDBConnection>>();
+   const proxyConnectionCache = new Map<
+      string,
+      Promise<PooledPostgresConnection>
+   >();
+   const proxyEndpoints = new Map<string, ProxyEndpoint>();
    const attachPromises = new WeakMap<Connection, Promise<void>>();
 
    const malloyConfig = new MalloyConfig(assembled.pojo, {
@@ -1044,6 +1077,64 @@ export function buildEnvironmentMalloyConfig(
                return connection;
             }
 
+            if (metadata?.proxy) {
+               let connectionPromise = proxyConnectionCache.get(name!);
+               if (!connectionPromise) {
+                  const connType = metadata.apiConnection.type;
+                  if (connType !== "postgres") {
+                     throw new Error(
+                        `SSH proxy is not yet supported for connection type '${connType}'. ` +
+                           `Only 'postgres' is implemented.`,
+                     );
+                  }
+                  // validateConnectionShape already enforces explicit host/port
+                  // at config load (the connectionString form can't be tunneled);
+                  // this is a defense-in-depth guard for any path that reaches
+                  // lookup without that validation.
+                  const pgConfig = metadata.apiConnection.postgresConnection;
+                  if (!pgConfig?.host || !pgConfig?.port) {
+                     throw new Error(
+                        `SSH proxy for connection '${name}' requires explicit host and ` +
+                           `port on the postgres connection; the connectionString form ` +
+                           `is not supported with a proxy.`,
+                     );
+                  }
+                  connectionPromise = (async () => {
+                     const endpoint = await openProxy(metadata.proxy!, {
+                        host: pgConfig.host!,
+                        port: pgConfig.port!,
+                     });
+                     try {
+                        const connection = buildProxiedPostgresConnection(
+                           metadata,
+                           endpoint,
+                        );
+                        // Register only after the build succeeds: a throw between
+                        // openProxy and here would otherwise leave an orphaned
+                        // tunnel in proxyEndpoints that a retry can't reach (the
+                        // outer .catch evicts the cache but not the endpoint).
+                        proxyEndpoints.set(name!, endpoint);
+                        return connection;
+                     } catch (err) {
+                        await endpoint.close().catch(() => {});
+                        throw err;
+                     }
+                  })();
+                  // Drop a rejected build from the cache so a later lookup can
+                  // retry, rather than replaying a transient SSH failure (bastion
+                  // restart, network blip) until the environment is rebuilt.
+                  connectionPromise.catch(() => {
+                     if (
+                        proxyConnectionCache.get(name!) === connectionPromise
+                     ) {
+                        proxyConnectionCache.delete(name!);
+                     }
+                  });
+                  proxyConnectionCache.set(name!, connectionPromise);
+               }
+               return connectionPromise;
+            }
+
             if (metadata?.hasSnowflakePrivateKey) {
                let connectionPromise = snowflakeJwtCache.get(name!);
                if (!connectionPromise) {
@@ -1086,6 +1177,7 @@ export function buildEnvironmentMalloyConfig(
             ...duckLakeCache.values(),
             ...snowflakeJwtCache.values(),
             ...azureDuckDBCache.values(),
+            ...proxyConnectionCache.values(),
          ];
          const closeResults = await Promise.allSettled([
             malloyConfig.shutdown("close"),
@@ -1094,11 +1186,21 @@ export function buildEnvironmentMalloyConfig(
                await connection.close();
             }),
          ]);
+         // Close proxy tunnels only AFTER the build promises above have settled:
+         // openProxy may still be in flight during a teardown race, and the
+         // endpoint is registered inside that promise — capturing the map
+         // earlier would miss it and leak the tunnel. Closing after the DB
+         // connections also keeps the tunnel alive while they drain.
+         const endpointResults = await Promise.allSettled(
+            [...proxyEndpoints.values()].map((ep) => ep.close()),
+         );
          duckLakeCache.clear();
          snowflakeJwtCache.clear();
          azureDuckDBCache.clear();
+         proxyConnectionCache.clear();
+         proxyEndpoints.clear();
 
-         const failures = closeResults.filter(
+         const failures = [...closeResults, ...endpointResults].filter(
             (result): result is PromiseRejectedResult =>
                result.status === "rejected",
          );

package/src/service/connection_config.spec.ts

@@ -349,3 +349,119 @@ describe("assembleEnvironmentConnections — publisher", () => {
       });
    });
 });
+
+describe("SSH proxy validation", () => {
+   const validSshProxy: ApiConnection = {
+      name: "pg-via-bastion",
+      type: "postgres",
+      postgresConnection: {
+         host: "127.0.0.1",
+         port: 5432,
+         databaseName: "mydb",
+         userName: "user",
+         password: "pass",
+      },
+      proxy: {
+         type: "ssh",
+         ssh: {
+            host: "bastion.example.com",
+            username: "ec2-user",
+            privateKey:
+               "-----BEGIN OPENSSH PRIVATE KEY-----\nfake\n-----END OPENSSH PRIVATE KEY-----",
+         },
+      },
+   };
+
+   it("allows a postgres+ssh-proxy connection with no env flag required", () => {
+      // The SSH proxy is a normal connection capability — deliberately NOT gated
+      // by PUBLISHER_ALLOW_PROXY_CONNECTIONS (that flag is for the `publisher`
+      // HTTP multi-hop type). Prove it assembles with the flag unset.
+      const priorFlag = process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS;
+      delete process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS;
+      try {
+         const { pojo } = assembleEnvironmentConnections([validSshProxy]);
+         expect(pojo.connections["pg-via-bastion"].is).toBe("postgres");
+      } finally {
+         if (priorFlag === undefined) {
+            delete process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS;
+         } else {
+            process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS = priorFlag;
+         }
+      }
+   });
+
+   it("rejects a non-postgres (bigquery) connection with a proxy", () => {
+      const conn: ApiConnection = {
+         name: "bq-via-bastion",
+         type: "bigquery",
+         bigqueryConnection: {
+            serviceAccountKeyJson: JSON.stringify({
+               type: "service_account",
+               project_id: "proj",
+               private_key: "key",
+               client_email: "sa@proj.iam.gserviceaccount.com",
+            }),
+         },
+         proxy: {
+            type: "ssh",
+            ssh: {
+               host: "bastion.example.com",
+               username: "ec2-user",
+               privateKey: "key",
+            },
+         },
+      };
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "Connection proxy is not supported for type 'bigquery' (only 'postgres' today)",
+      );
+   });
+
+   it("rejects an ssh proxy with no ssh config object", () => {
+      const conn: ApiConnection = {
+         name: "pg-no-ssh-config",
+         type: "postgres",
+         postgresConnection: {
+            host: "127.0.0.1",
+            databaseName: "mydb",
+         },
+         proxy: {
+            type: "ssh",
+         },
+      };
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "Connection proxy on 'pg-no-ssh-config' has type 'ssh' but no 'ssh' config object",
+      );
+   });
+
+   it("rejects a proxied postgres connection that also carries a connectionString", () => {
+      const conn: ApiConnection = {
+         ...validSshProxy,
+         name: "pg-with-connstring",
+         postgresConnection: {
+            connectionString: "postgresql://real-db.example.com:5432/mydb",
+            host: "127.0.0.1",
+            port: 5432,
+            databaseName: "mydb",
+         },
+      };
+      // The tunnel forwards to host/port; a connectionString would be silently
+      // ignored and could point at a different database, so it's rejected.
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "does not support the connectionString form",
+      );
+   });
+
+   it("rejects a proxied postgres connection missing explicit host/port", () => {
+      const conn: ApiConnection = {
+         ...validSshProxy,
+         name: "pg-no-host-port",
+         postgresConnection: {
+            databaseName: "mydb",
+            userName: "user",
+         },
+      };
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "requires explicit host and port",
+      );
+   });
+});

package/src/service/connection_config.ts

@@ -22,6 +22,7 @@ export type EnvironmentConnectionMetadata = {
    isDuckLake: boolean;
    databasePath?: string;
    workingDirectory: string;
+   proxy?: ApiConnection["proxy"];
 };
 
 export type AssembledEnvironmentConnections = {
@@ -258,6 +259,52 @@ function buildDuckdbEntry(
 }
 
 function validateConnectionShape(connection: ApiConnection): void {
+   if (connection.proxy) {
+      // A connection proxy makes THIS server open an outbound SSH tunnel to a
+      // tenant-configured bastion. It's a normal connection capability,
+      // authorized by whoever configures the connection — deliberately NOT gated
+      // by an env flag, and kept separate from the `publisher` HTTP multi-hop
+      // type's PUBLISHER_ALLOW_PROXY_CONNECTIONS gate below (that flag is about
+      // publisher-to-publisher proxying, a distinct operator decision). Host-key
+      // pinning is fail-closed at connect time (see openProxy).
+      if (connection.proxy.type !== "ssh") {
+         throw new Error(
+            `Connection '${connection.name}' has an unsupported proxy type '${connection.proxy.type}'. Only 'ssh' is supported.`,
+         );
+      }
+      if (connection.type !== "postgres") {
+         throw new Error(
+            `Connection proxy is not supported for type '${connection.type}' (only 'postgres' today).`,
+         );
+      }
+      if (!connection.proxy.ssh) {
+         throw new Error(
+            `Connection proxy on '${connection.name}' has type 'ssh' but no 'ssh' config object.`,
+         );
+      }
+      // The tunnel forwards to an explicit host:port; the connectionString form
+      // can't be rewritten to the local endpoint. Reject it outright when a
+      // proxy is set — normal postgres gives connectionString precedence over
+      // host/port, so a config carrying BOTH would silently tunnel to
+      // host/port and ignore the connectionString, connecting to a different
+      // database than the operator configured. Require discrete host/port.
+      if (connection.postgresConnection?.connectionString) {
+         throw new Error(
+            `Connection proxy on '${connection.name}' does not support the connectionString form; ` +
+               `provide discrete host and port instead (the tunnel forwards to an explicit endpoint).`,
+         );
+      }
+      if (
+         !connection.postgresConnection?.host ||
+         !connection.postgresConnection?.port
+      ) {
+         throw new Error(
+            `Connection proxy on '${connection.name}' requires explicit host and port on the ` +
+               `postgres connection; the connectionString form is not supported with a proxy.`,
+         );
+      }
+   }
+
    switch (connection.type) {
       case "postgres":
       case "mysql":
@@ -445,6 +492,7 @@ export function assembleEnvironmentConnections(
          isDuckLake,
          databasePath,
          workingDirectory: environmentPath,
+         proxy: connection.proxy,
       });
 
       switch (connection.type) {

package/src/service/model.ts

@@ -115,6 +115,18 @@ function quoteMalloyIdentifier(name: string | undefined): string {
    return "`" + (name ?? "").replace(/\\/g, "\\\\").replace(/`/g, "\\`") + "`";
 }
 
+/**
+ * A non-fatal render-tag finding from {@link Model.validateRenderTags}: an
+ * error-severity issue that affects only how a field renders, never whether the
+ * model compiles or a query runs. `target` is the query or view it sits on
+ * (e.g. `by_carrier` or `flights -> by_carrier`).
+ */
+export interface RenderTagWarning {
+   target: string;
+   message: string;
+   severity: "error" | "warn";
+}
+
 export class Model {
    private packageName: string;
    private modelPath: string;
@@ -952,12 +964,13 @@ export class Model {
     * it does not fail the package load. Such a tag still renders as
     * "[object Object]" at query time, so the warning is the operator-facing
     * signal. Lower-severity findings are left for the query-time `renderLogs`
-    * surface.
+    * surface. The findings are returned so the owning Package can surface them
+    * as non-fatal `warnings` on its response.
     */
-   public async validateRenderTags(): Promise<void> {
+   public async validateRenderTags(): Promise<RenderTagWarning[]> {
       const mm = this.modelMaterializer;
       if (!mm) {
-         return;
+         return [];
       }
       // Dynamic import (like execute_query_tool.ts): the renderer is heavy and
       // mutates DOM globals on load, so only pull it in when there's a model to
@@ -1003,6 +1016,7 @@ export class Model {
          }
       }
 
+      const findings: RenderTagWarning[] = [];
       for (const target of targets) {
          let result: Malloy.Result;
          try {
@@ -1024,8 +1038,16 @@ export class Model {
                   .map((e) => e.message)
                   .join("; ")}`,
             );
+            for (const e of errors) {
+               findings.push({
+                  target: target.label,
+                  message: e.message,
+                  severity: "error",
+               });
+            }
          }
       }
+      return findings;
    }
 
    public async getModel(): Promise<ApiCompiledModel> {

package/src/service/package.ts

@@ -40,6 +40,7 @@ type ApiDatabase = components["schemas"]["Database"];
 type ApiModel = components["schemas"]["Model"];
 type ApiNotebook = components["schemas"]["Notebook"];
 export type ApiPackage = components["schemas"]["Package"];
+type ApiPackageWarning = NonNullable<ApiPackage["warnings"]>[number];
 type ApiColumn = components["schemas"]["Column"];
 type ApiTableDescription = components["schemas"]["TableDescription"];
 // A thunk lets callers pass a live reference to the *current* environment
@@ -77,6 +78,12 @@ export class Package {
    // no persist source. Surfaced read-only on getPackageMetadata() so a caller
    // can derive build instructions without a separate plan round-trip.
    private buildPlan: BuildPlan | null = null;
+   // Non-fatal render-tag findings aggregated across the package's models (each
+   // tagged with its model path), surfaced read-only on
+   // getPackageMetadata().warnings. Refreshed on load and reload. A bad render
+   // tag does not fail the load (see Model.validateRenderTags); this is the
+   // response-level signal that a tag is misconfigured.
+   private renderTagWarnings: ApiPackageWarning[] = [];
    private static meter = publisherMeter();
    private static packageLoadHistogram = this.meter.createHistogram(
       "malloy_package_load_duration",
@@ -350,6 +357,7 @@ export class Package {
       // placeholders instead; that branch goes through a different
       // hydration path.)
       const models = new Map<string, Model>();
+      const renderTagWarnings: ApiPackageWarning[] = [];
       for (const sm of outcome.models) {
          if (sm.compilationError) {
             const err = Model.deserializeCompilationError(sm.compilationError);
@@ -371,7 +379,10 @@ export class Package {
          // Validate renderer tags on the main thread (the renderer is too heavy
          // to load inside the pure-CPU package-load worker). A misconfigured tag
          // is logged as a warning naming the target; it does not fail the load.
-         await model.validateRenderTags();
+         // The findings also ride the package response as non-fatal `warnings`.
+         for (const w of await model.validateRenderTags()) {
+            renderTagWarnings.push({ model: sm.modelPath, ...w });
+         }
          // Reject unquoted `#@ persist name=` annotations the same way: an
          // unquoted name is dropped from the build plan, so the source would
          // publish but never materialize. Scan the raw `.malloy` source (the
@@ -406,6 +417,7 @@ export class Package {
          models,
          malloyConfig,
       );
+      pkg.renderTagWarnings = renderTagWarnings;
 
       // Compute the persist build plan off the live (unbound) models, before the
       // caller binds any configured manifest, so the surfaced plan reflects the
@@ -484,6 +496,9 @@ export class Package {
       if (warnings.length > 0) {
          metadata.exploresWarnings = warnings;
       }
+      if (this.renderTagWarnings.length > 0) {
+         metadata.warnings = [...this.renderTagWarnings];
+      }
       return metadata;
    }
 
@@ -704,6 +719,7 @@ export class Package {
       }
 
       const nextModels = new Map<string, Model>();
+      const renderTagWarnings: ApiPackageWarning[] = [];
       for (const sm of outcome.models) {
          if (sm.compilationError) {
             const err = Model.deserializeCompilationError(sm.compilationError);
@@ -735,7 +751,9 @@ export class Package {
             // unexpected internal failure is recorded as this model's
             // compilationError rather than aborting the whole reload.
             try {
-               await model.validateRenderTags();
+               for (const w of await model.validateRenderTags()) {
+                  renderTagWarnings.push({ model: sm.modelPath, ...w });
+               }
                nextModels.set(sm.modelPath, model);
             } catch (renderErr) {
                const err =
@@ -760,6 +778,7 @@ export class Package {
          }
       }
       this.models = nextModels;
+      this.renderTagWarnings = renderTagWarnings;
       // A reload re-reads publisher.json in the worker; pick up any change to
       // the explore set and query-boundary mode so listModels()/the gate
       // reflect edited explores without a full Package.create.