@malloy-publisher/server 0.0.217 → 0.0.219

package/src/service/connection_config.ts

@@ -1,10 +1,18 @@
 import { createPrivateKey } from "crypto";
+import { existsSync } from "fs";
 import path from "path";
 import { components } from "../api";
+import { parseHostKeys } from "./proxy";
 
 type ApiConnection = components["schemas"]["Connection"];
 type AttachedDatabase = components["schemas"]["AttachedDatabase"];
 
+// TLS modes accepted on a proxied postgres connection. Canonical here (rather
+// than in connection.ts, which imports this module) so both the config-load
+// validator and the connect-time builder derive from one list. Mirrors the
+// `sslmode` enum in api-doc.yaml.
+export const PROXIED_SSLMODES = ["disable", "no-verify", "verify-ca"] as const;
+
 export type CoreConnectionEntry = {
    is: string;
    [key: string]: unknown;
@@ -22,6 +30,7 @@ export type EnvironmentConnectionMetadata = {
    isDuckLake: boolean;
    databasePath?: string;
    workingDirectory: string;
+   proxy?: ApiConnection["proxy"];
 };
 
 export type AssembledEnvironmentConnections = {
@@ -258,6 +267,123 @@ function buildDuckdbEntry(
 }
 
 function validateConnectionShape(connection: ApiConnection): void {
+   if (connection.proxy) {
+      // A connection proxy makes THIS server open an outbound SSH tunnel to a
+      // tenant-configured bastion. It's a normal connection capability,
+      // authorized by whoever configures the connection — deliberately NOT gated
+      // by an env flag, and kept separate from the `publisher` HTTP multi-hop
+      // type's PUBLISHER_ALLOW_PROXY_CONNECTIONS gate below (that flag is about
+      // publisher-to-publisher proxying, a distinct operator decision). Optional
+      // host-key pinning is fail-closed at connect time when configured (see
+      // openProxy); the proxy-specific fields are validated up front below so a
+      // permanent misconfig fails at config load, not by repeatedly dialing the
+      // tenant's bastion at query time.
+      if (connection.proxy.type !== "ssh") {
+         throw new Error(
+            `Connection '${connection.name}' has an unsupported proxy type '${connection.proxy.type}'. Only 'ssh' is supported.`,
+         );
+      }
+      if (connection.type !== "postgres") {
+         throw new Error(
+            `Connection proxy is not supported for type '${connection.type}' (only 'postgres' today).`,
+         );
+      }
+      if (!connection.proxy.ssh) {
+         throw new Error(
+            `Connection proxy on '${connection.name}' has type 'ssh' but no 'ssh' config object.`,
+         );
+      }
+      // The tunnel forwards to an explicit host:port; the connectionString form
+      // can't be rewritten to the local endpoint. Reject it outright when a
+      // proxy is set — normal postgres gives connectionString precedence over
+      // host/port, so a config carrying BOTH would silently tunnel to
+      // host/port and ignore the connectionString, connecting to a different
+      // database than the operator configured. Require discrete host/port.
+      if (connection.postgresConnection?.connectionString) {
+         throw new Error(
+            `Connection proxy on '${connection.name}' does not support the connectionString form; ` +
+               `provide discrete host and port instead (the tunnel forwards to an explicit endpoint).`,
+         );
+      }
+      if (
+         !connection.postgresConnection?.host ||
+         !connection.postgresConnection?.port
+      ) {
+         throw new Error(
+            `Connection proxy on '${connection.name}' requires explicit host and port on the ` +
+               `postgres connection; the connectionString form is not supported with a proxy.`,
+         );
+      }
+
+      // hostKey is optional (omitted or empty string => connect unpinned), but a
+      // non-empty hostKey that parses to zero keys — only blank lines, whitespace,
+      // or `#` comments, e.g. a paste that grabbed just ssh-keyscan's
+      // `# host:port ...` header — is a misconfigured pin, not a licence to
+      // connect unverified. Reject it here so the operator gets a config error
+      // instead of a silently unpinned tunnel. (Truthiness, not trim(): "" is the
+      // unpinned signal; "   " is a non-empty value that must yield a key.)
+      const hostKey = connection.proxy.ssh?.hostKey;
+      if (hostKey && parseHostKeys(hostKey).size === 0) {
+         throw new Error(
+            `Connection proxy on '${connection.name}' has a hostKey with no usable host-key line ` +
+               `(only blanks/comments). Provide an OpenSSH known_hosts line or base64 blob, or omit ` +
+               `hostKey to connect unpinned.`,
+         );
+      }
+
+      // Validate the proxied TLS mode up front. The tunnel is dialed lazily on
+      // first lookup and a failed build is retried on every subsequent query, so
+      // a permanent sslmode misconfig left to throw at connect time would re-dial
+      // the tenant's bastion indefinitely. Fail at config load instead.
+      // `!= null` (not truthiness) so a present-but-empty sslmode ("") is caught
+      // here as unsupported rather than slipping through to fail at tunnel-build;
+      // null/undefined mean unset (server applies the default).
+      const sslmode = connection.postgresConnection?.sslmode;
+      if (sslmode != null) {
+         if (!(PROXIED_SSLMODES as readonly string[]).includes(sslmode)) {
+            throw new Error(
+               `Connection proxy on '${connection.name}' has unsupported sslmode '${sslmode}' ` +
+                  `(expected ${PROXIED_SSLMODES.join(" | ")}).`,
+            );
+         }
+         if (sslmode === "verify-ca") {
+            const caBundle = process.env.NODE_EXTRA_CA_CERTS;
+            if (!caBundle || !existsSync(caBundle)) {
+               throw new Error(
+                  `Connection proxy on '${connection.name}' uses sslmode 'verify-ca' but no readable ` +
+                     `CA bundle is available (NODE_EXTRA_CA_CERTS is unset or points at a missing file). ` +
+                     `Add the CA bundle to the image or use sslmode 'no-verify'.`,
+               );
+            }
+         }
+      }
+
+      // The proxied path builds a connectionString to the local tunnel endpoint;
+      // pg decodes the database path with decodeURI, which leaves URI-reserved
+      // characters percent-encoded — so a db name containing them would resolve
+      // to the wrong database. Reject it clearly rather than failing later with a
+      // confusing "database does not exist". (user/password use decodeURIComponent
+      // on parse and round-trip fine.)
+      const dbName = connection.postgresConnection?.databaseName;
+      if (dbName && decodeURI(encodeURIComponent(dbName)) !== dbName) {
+         throw new Error(
+            `Connection proxy on '${connection.name}' has a database name with characters that can't ` +
+               `be carried over a proxied connection (${JSON.stringify(dbName)}). Use a database name ` +
+               `without URI-reserved characters (; , / ? : @ & = + $ #).`,
+         );
+      }
+   }
+
+   // sslmode is only honored on the proxied path (the direct path builds TLS from
+   // the deployment PGSSLMODE). Reject it on a non-proxied connection so a tenant
+   // who sets it doesn't silently get a different TLS posture than they asked for.
+   if (!connection.proxy && connection.postgresConnection?.sslmode) {
+      throw new Error(
+         `Connection '${connection.name}' sets postgresConnection.sslmode but has no proxy; sslmode is ` +
+            `only supported for proxied connections (direct connections use the deployment PGSSLMODE).`,
+      );
+   }
+
    switch (connection.type) {
       case "postgres":
       case "mysql":
@@ -445,6 +571,7 @@ export function assembleEnvironmentConnections(
          isDuckLake,
          databasePath,
          workingDirectory: environmentPath,
+         proxy: connection.proxy,
       });
 
       switch (connection.type) {

package/src/service/model.ts

@@ -115,6 +115,18 @@ function quoteMalloyIdentifier(name: string | undefined): string {
    return "`" + (name ?? "").replace(/\\/g, "\\\\").replace(/`/g, "\\`") + "`";
 }
 
+/**
+ * A non-fatal render-tag finding from {@link Model.validateRenderTags}: an
+ * error-severity issue that affects only how a field renders, never whether the
+ * model compiles or a query runs. `target` is the query or view it sits on
+ * (e.g. `by_carrier` or `flights -> by_carrier`).
+ */
+export interface RenderTagWarning {
+   target: string;
+   message: string;
+   severity: "error" | "warn";
+}
+
 export class Model {
    private packageName: string;
    private modelPath: string;
@@ -952,12 +964,13 @@ export class Model {
     * it does not fail the package load. Such a tag still renders as
     * "[object Object]" at query time, so the warning is the operator-facing
     * signal. Lower-severity findings are left for the query-time `renderLogs`
-    * surface.
+    * surface. The findings are returned so the owning Package can surface them
+    * as non-fatal `warnings` on its response.
     */
-   public async validateRenderTags(): Promise<void> {
+   public async validateRenderTags(): Promise<RenderTagWarning[]> {
       const mm = this.modelMaterializer;
       if (!mm) {
-         return;
+         return [];
       }
       // Dynamic import (like execute_query_tool.ts): the renderer is heavy and
       // mutates DOM globals on load, so only pull it in when there's a model to
@@ -1003,6 +1016,7 @@ export class Model {
          }
       }
 
+      const findings: RenderTagWarning[] = [];
       for (const target of targets) {
          let result: Malloy.Result;
          try {
@@ -1024,8 +1038,16 @@ export class Model {
                   .map((e) => e.message)
                   .join("; ")}`,
             );
+            for (const e of errors) {
+               findings.push({
+                  target: target.label,
+                  message: e.message,
+                  severity: "error",
+               });
+            }
          }
       }
+      return findings;
    }
 
    public async getModel(): Promise<ApiCompiledModel> {

package/src/service/package.ts

@@ -40,6 +40,7 @@ type ApiDatabase = components["schemas"]["Database"];
 type ApiModel = components["schemas"]["Model"];
 type ApiNotebook = components["schemas"]["Notebook"];
 export type ApiPackage = components["schemas"]["Package"];
+type ApiPackageWarning = NonNullable<ApiPackage["warnings"]>[number];
 type ApiColumn = components["schemas"]["Column"];
 type ApiTableDescription = components["schemas"]["TableDescription"];
 // A thunk lets callers pass a live reference to the *current* environment
@@ -77,6 +78,12 @@ export class Package {
    // no persist source. Surfaced read-only on getPackageMetadata() so a caller
    // can derive build instructions without a separate plan round-trip.
    private buildPlan: BuildPlan | null = null;
+   // Non-fatal render-tag findings aggregated across the package's models (each
+   // tagged with its model path), surfaced read-only on
+   // getPackageMetadata().warnings. Refreshed on load and reload. A bad render
+   // tag does not fail the load (see Model.validateRenderTags); this is the
+   // response-level signal that a tag is misconfigured.
+   private renderTagWarnings: ApiPackageWarning[] = [];
    private static meter = publisherMeter();
    private static packageLoadHistogram = this.meter.createHistogram(
       "malloy_package_load_duration",
@@ -350,6 +357,7 @@ export class Package {
       // placeholders instead; that branch goes through a different
       // hydration path.)
       const models = new Map<string, Model>();
+      const renderTagWarnings: ApiPackageWarning[] = [];
       for (const sm of outcome.models) {
          if (sm.compilationError) {
             const err = Model.deserializeCompilationError(sm.compilationError);
@@ -371,7 +379,10 @@ export class Package {
          // Validate renderer tags on the main thread (the renderer is too heavy
          // to load inside the pure-CPU package-load worker). A misconfigured tag
          // is logged as a warning naming the target; it does not fail the load.
-         await model.validateRenderTags();
+         // The findings also ride the package response as non-fatal `warnings`.
+         for (const w of await model.validateRenderTags()) {
+            renderTagWarnings.push({ model: sm.modelPath, ...w });
+         }
          // Reject unquoted `#@ persist name=` annotations the same way: an
          // unquoted name is dropped from the build plan, so the source would
          // publish but never materialize. Scan the raw `.malloy` source (the
@@ -406,6 +417,7 @@ export class Package {
          models,
          malloyConfig,
       );
+      pkg.renderTagWarnings = renderTagWarnings;
 
       // Compute the persist build plan off the live (unbound) models, before the
       // caller binds any configured manifest, so the surfaced plan reflects the
@@ -484,6 +496,9 @@ export class Package {
       if (warnings.length > 0) {
          metadata.exploresWarnings = warnings;
       }
+      if (this.renderTagWarnings.length > 0) {
+         metadata.warnings = [...this.renderTagWarnings];
+      }
       return metadata;
    }
 
@@ -704,6 +719,7 @@ export class Package {
       }
 
       const nextModels = new Map<string, Model>();
+      const renderTagWarnings: ApiPackageWarning[] = [];
       for (const sm of outcome.models) {
          if (sm.compilationError) {
             const err = Model.deserializeCompilationError(sm.compilationError);
@@ -735,7 +751,9 @@ export class Package {
             // unexpected internal failure is recorded as this model's
             // compilationError rather than aborting the whole reload.
             try {
-               await model.validateRenderTags();
+               for (const w of await model.validateRenderTags()) {
+                  renderTagWarnings.push({ model: sm.modelPath, ...w });
+               }
                nextModels.set(sm.modelPath, model);
             } catch (renderErr) {
                const err =
@@ -760,6 +778,7 @@ export class Package {
          }
       }
       this.models = nextModels;
+      this.renderTagWarnings = renderTagWarnings;
       // A reload re-reads publisher.json in the worker; pick up any change to
       // the explore set and query-boundary mode so listModels()/the gate
       // reflect edited explores without a full Package.create.

package/src/service/package_worker_path.spec.ts

@@ -316,6 +316,17 @@ source: gated is duckdb.sql("select 1 as id")`,
                   m.includes("nums -> card"),
             ),
          ).toBe(true);
+         // The same finding rides the package response (non-fatal) so the
+         // control plane / UI can surface it without scraping logs.
+         const responseWarnings = pkg.getPackageMetadata().warnings ?? [];
+         expect(
+            responseWarnings.some(
+               (w) =>
+                  w.model === "bad_render.malloy" &&
+                  w.target === "nums -> card" &&
+                  w.severity === "error",
+            ),
+         ).toBe(true);
       } finally {
          warnSpy.mockRestore();
          await duckdb.close();
@@ -337,6 +348,8 @@ source: gated is duckdb.sql("select 1 as id")`,
       try {
          const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
          expect(pkg.getModelPaths()).toEqual(["good_render.malloy"]);
+         // A clean package omits the warnings field entirely.
+         expect(pkg.getPackageMetadata().warnings).toBeUndefined();
       } finally {
          await duckdb.close();
       }
@@ -449,6 +462,14 @@ source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
          expect(
             warnings.some((m) => m.includes("Invalid renderer configuration")),
          ).toBe(true);
+         // The notebook finding also rides the package response (not just logs).
+         const responseWarnings = pkg.getPackageMetadata().warnings ?? [];
+         expect(
+            responseWarnings.some(
+               (w) =>
+                  w.model === "bad_render.malloynb" && w.severity === "error",
+            ),
+         ).toBe(true);
       } finally {
          warnSpy.mockRestore();
          await duckdb.close();
@@ -499,12 +520,94 @@ source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
          expect(
             warnings.some((m) => m.includes("Invalid renderer configuration")),
          ).toBe(true);
+         // Reload refreshes the response-level warnings too.
+         const responseWarnings = pkg.getPackageMetadata().warnings ?? [];
+         expect(responseWarnings.some((w) => w.model === "m.malloy")).toBe(
+            true,
+         );
       } finally {
          warnSpy.mockRestore();
          await duckdb.close();
       }
    });
 
+   it("clears render-tag warnings on reload when a bad tag is fixed (bad -> clean)", async () => {
+      writeManifest();
+      // Start BAD so the package loads with a warning.
+      fs.writeFileSync(
+         path.join(tempDir, "m.malloy"),
+         `source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
+  measure: total is a.sum()
+  # big_value { sparkline=trend }
+  view: card is {
+    aggregate: total
+    nest:
+      # line_chart { size=spark }
+      trend is { group_by: a; aggregate: total }
+  }
+}`,
+      );
+
+      const { malloyConfig, duckdb } = await makeMalloyConfig();
+      try {
+         const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
+         expect(
+            (pkg.getPackageMetadata().warnings ?? []).some(
+               (w) => w.model === "m.malloy",
+            ),
+         ).toBe(true);
+
+         // Fix the tag, then reload: renderTagWarnings is rebuilt from scratch,
+         // so the stale finding must disappear (field omitted when empty).
+         fs.writeFileSync(
+            path.join(tempDir, "m.malloy"),
+            `source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
+  measure: total is a.sum()
+  # bar_chart
+  view: chart is { group_by: a; aggregate: total }
+}`,
+         );
+         await pkg.reloadAllModels({});
+
+         expect(pkg.getPackageMetadata().warnings).toBeUndefined();
+      } finally {
+         await duckdb.close();
+      }
+   });
+
+   it("aggregates render-tag warnings from multiple models, each tagged with its own path", async () => {
+      writeManifest();
+      const badModel = (view: string) =>
+         `source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
+  measure: total is a.sum()
+  # big_value { sparkline=trend }
+  view: ${view} is {
+    aggregate: total
+    nest:
+      # line_chart { size=spark }
+      trend is { group_by: a; aggregate: total }
+  }
+}`;
+      fs.writeFileSync(path.join(tempDir, "a.malloy"), badModel("card_a"));
+      fs.writeFileSync(path.join(tempDir, "b.malloy"), badModel("card_b"));
+
+      const { malloyConfig, duckdb } = await makeMalloyConfig();
+      try {
+         const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
+         const responseWarnings = pkg.getPackageMetadata().warnings ?? [];
+         const models = new Set(responseWarnings.map((w) => w.model));
+         expect(models.has("a.malloy")).toBe(true);
+         expect(models.has("b.malloy")).toBe(true);
+         expect(models.size).toBe(2);
+         // The message field is carried through, not just model/target/severity.
+         expect(
+            responseWarnings.every((w) => (w.message ?? "").length > 0),
+         ).toBe(true);
+      } finally {
+         await duckdb.close();
+      }
+   });
+
    // NB: kept last in this describe — swapping the singleton for a
    // pre-shutdown pool also tears down the shared `pool` (the swap
    // implementation shuts down the outgoing singleton). Subsequent