@malloy-publisher/server 0.0.216 → 0.0.218

package/src/service/package.ts

@@ -40,6 +40,7 @@ type ApiDatabase = components["schemas"]["Database"];
 type ApiModel = components["schemas"]["Model"];
 type ApiNotebook = components["schemas"]["Notebook"];
 export type ApiPackage = components["schemas"]["Package"];
+type ApiPackageWarning = NonNullable<ApiPackage["warnings"]>[number];
 type ApiColumn = components["schemas"]["Column"];
 type ApiTableDescription = components["schemas"]["TableDescription"];
 // A thunk lets callers pass a live reference to the *current* environment
@@ -77,6 +78,12 @@ export class Package {
    // no persist source. Surfaced read-only on getPackageMetadata() so a caller
    // can derive build instructions without a separate plan round-trip.
    private buildPlan: BuildPlan | null = null;
+   // Non-fatal render-tag findings aggregated across the package's models (each
+   // tagged with its model path), surfaced read-only on
+   // getPackageMetadata().warnings. Refreshed on load and reload. A bad render
+   // tag does not fail the load (see Model.validateRenderTags); this is the
+   // response-level signal that a tag is misconfigured.
+   private renderTagWarnings: ApiPackageWarning[] = [];
    private static meter = publisherMeter();
    private static packageLoadHistogram = this.meter.createHistogram(
       "malloy_package_load_duration",
@@ -350,6 +357,7 @@ export class Package {
       // placeholders instead; that branch goes through a different
       // hydration path.)
       const models = new Map<string, Model>();
+      const renderTagWarnings: ApiPackageWarning[] = [];
       for (const sm of outcome.models) {
          if (sm.compilationError) {
             const err = Model.deserializeCompilationError(sm.compilationError);
@@ -371,7 +379,10 @@ export class Package {
          // Validate renderer tags on the main thread (the renderer is too heavy
          // to load inside the pure-CPU package-load worker). A misconfigured tag
          // is logged as a warning naming the target; it does not fail the load.
-         await model.validateRenderTags();
+         // The findings also ride the package response as non-fatal `warnings`.
+         for (const w of await model.validateRenderTags()) {
+            renderTagWarnings.push({ model: sm.modelPath, ...w });
+         }
          // Reject unquoted `#@ persist name=` annotations the same way: an
          // unquoted name is dropped from the build plan, so the source would
          // publish but never materialize. Scan the raw `.malloy` source (the
@@ -406,6 +417,7 @@ export class Package {
          models,
          malloyConfig,
       );
+      pkg.renderTagWarnings = renderTagWarnings;
 
       // Compute the persist build plan off the live (unbound) models, before the
       // caller binds any configured manifest, so the surfaced plan reflects the
@@ -484,6 +496,9 @@ export class Package {
       if (warnings.length > 0) {
          metadata.exploresWarnings = warnings;
       }
+      if (this.renderTagWarnings.length > 0) {
+         metadata.warnings = [...this.renderTagWarnings];
+      }
       return metadata;
    }
 
@@ -704,6 +719,7 @@ export class Package {
       }
 
       const nextModels = new Map<string, Model>();
+      const renderTagWarnings: ApiPackageWarning[] = [];
       for (const sm of outcome.models) {
          if (sm.compilationError) {
             const err = Model.deserializeCompilationError(sm.compilationError);
@@ -735,7 +751,9 @@ export class Package {
             // unexpected internal failure is recorded as this model's
             // compilationError rather than aborting the whole reload.
             try {
-               await model.validateRenderTags();
+               for (const w of await model.validateRenderTags()) {
+                  renderTagWarnings.push({ model: sm.modelPath, ...w });
+               }
                nextModels.set(sm.modelPath, model);
             } catch (renderErr) {
                const err =
@@ -760,6 +778,7 @@ export class Package {
          }
       }
       this.models = nextModels;
+      this.renderTagWarnings = renderTagWarnings;
       // A reload re-reads publisher.json in the worker; pick up any change to
       // the explore set and query-boundary mode so listModels()/the gate
       // reflect edited explores without a full Package.create.

package/src/service/package_worker_path.spec.ts

@@ -316,6 +316,17 @@ source: gated is duckdb.sql("select 1 as id")`,
                   m.includes("nums -> card"),
             ),
          ).toBe(true);
+         // The same finding rides the package response (non-fatal) so the
+         // control plane / UI can surface it without scraping logs.
+         const responseWarnings = pkg.getPackageMetadata().warnings ?? [];
+         expect(
+            responseWarnings.some(
+               (w) =>
+                  w.model === "bad_render.malloy" &&
+                  w.target === "nums -> card" &&
+                  w.severity === "error",
+            ),
+         ).toBe(true);
       } finally {
          warnSpy.mockRestore();
          await duckdb.close();
@@ -337,6 +348,8 @@ source: gated is duckdb.sql("select 1 as id")`,
       try {
          const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
          expect(pkg.getModelPaths()).toEqual(["good_render.malloy"]);
+         // A clean package omits the warnings field entirely.
+         expect(pkg.getPackageMetadata().warnings).toBeUndefined();
       } finally {
          await duckdb.close();
       }
@@ -449,6 +462,14 @@ source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
          expect(
             warnings.some((m) => m.includes("Invalid renderer configuration")),
          ).toBe(true);
+         // The notebook finding also rides the package response (not just logs).
+         const responseWarnings = pkg.getPackageMetadata().warnings ?? [];
+         expect(
+            responseWarnings.some(
+               (w) =>
+                  w.model === "bad_render.malloynb" && w.severity === "error",
+            ),
+         ).toBe(true);
       } finally {
          warnSpy.mockRestore();
          await duckdb.close();
@@ -499,12 +520,94 @@ source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
          expect(
             warnings.some((m) => m.includes("Invalid renderer configuration")),
          ).toBe(true);
+         // Reload refreshes the response-level warnings too.
+         const responseWarnings = pkg.getPackageMetadata().warnings ?? [];
+         expect(responseWarnings.some((w) => w.model === "m.malloy")).toBe(
+            true,
+         );
       } finally {
          warnSpy.mockRestore();
          await duckdb.close();
       }
    });
 
+   it("clears render-tag warnings on reload when a bad tag is fixed (bad -> clean)", async () => {
+      writeManifest();
+      // Start BAD so the package loads with a warning.
+      fs.writeFileSync(
+         path.join(tempDir, "m.malloy"),
+         `source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
+  measure: total is a.sum()
+  # big_value { sparkline=trend }
+  view: card is {
+    aggregate: total
+    nest:
+      # line_chart { size=spark }
+      trend is { group_by: a; aggregate: total }
+  }
+}`,
+      );
+
+      const { malloyConfig, duckdb } = await makeMalloyConfig();
+      try {
+         const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
+         expect(
+            (pkg.getPackageMetadata().warnings ?? []).some(
+               (w) => w.model === "m.malloy",
+            ),
+         ).toBe(true);
+
+         // Fix the tag, then reload: renderTagWarnings is rebuilt from scratch,
+         // so the stale finding must disappear (field omitted when empty).
+         fs.writeFileSync(
+            path.join(tempDir, "m.malloy"),
+            `source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
+  measure: total is a.sum()
+  # bar_chart
+  view: chart is { group_by: a; aggregate: total }
+}`,
+         );
+         await pkg.reloadAllModels({});
+
+         expect(pkg.getPackageMetadata().warnings).toBeUndefined();
+      } finally {
+         await duckdb.close();
+      }
+   });
+
+   it("aggregates render-tag warnings from multiple models, each tagged with its own path", async () => {
+      writeManifest();
+      const badModel = (view: string) =>
+         `source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
+  measure: total is a.sum()
+  # big_value { sparkline=trend }
+  view: ${view} is {
+    aggregate: total
+    nest:
+      # line_chart { size=spark }
+      trend is { group_by: a; aggregate: total }
+  }
+}`;
+      fs.writeFileSync(path.join(tempDir, "a.malloy"), badModel("card_a"));
+      fs.writeFileSync(path.join(tempDir, "b.malloy"), badModel("card_b"));
+
+      const { malloyConfig, duckdb } = await makeMalloyConfig();
+      try {
+         const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
+         const responseWarnings = pkg.getPackageMetadata().warnings ?? [];
+         const models = new Set(responseWarnings.map((w) => w.model));
+         expect(models.has("a.malloy")).toBe(true);
+         expect(models.has("b.malloy")).toBe(true);
+         expect(models.size).toBe(2);
+         // The message field is carried through, not just model/target/severity.
+         expect(
+            responseWarnings.every((w) => (w.message ?? "").length > 0),
+         ).toBe(true);
+      } finally {
+         await duckdb.close();
+      }
+   });
+
    // NB: kept last in this describe — swapping the singleton for a
    // pre-shutdown pool also tears down the shared `pool` (the swap
    // implementation shuts down the outgoing singleton). Subsequent

package/src/service/proxy.spec.ts

@@ -0,0 +1,367 @@
+/**
+ * Unit tests for the SSH proxy layer.
+ *
+ * These tests are hermetic: they stand up an in-process ssh2 server that
+ * forwards to a plain TCP echo server, call openProxy(), connect a client
+ * to the returned local endpoint, and assert that bytes round-trip through
+ * the tunnel.
+ *
+ * No external network access is required.
+ */
+
+import net from "net";
+import { generateKeyPairSync } from "crypto";
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import {
+   Server as SshServer,
+   utils as sshUtils,
+   type Connection as SshServerConnection,
+} from "ssh2";
+import { openProxy } from "./proxy";
+
+// ── Key material generated once for the test suite ────────────────────────────
+
+// ssh2's server requires PEM private keys it can parse. These are ephemeral,
+// test-only keys; RSA-2048 (not 1024) so static analysis doesn't flag them.
+const hostKeys = generateKeyPairSync("rsa", { modulusLength: 2048 });
+const hostPrivatePem = hostKeys.privateKey
+   .export({ type: "pkcs1", format: "pem" })
+   .toString();
+
+const clientKeys = generateKeyPairSync("rsa", { modulusLength: 2048 });
+const clientPrivatePem = clientKeys.privateKey
+   .export({ type: "pkcs1", format: "pem" })
+   .toString();
+// Helpers ─────────────────────────────────────────────────────────────────────
+
+function startEchoServer(): Promise<{
+   port: number;
+   close: () => Promise<void>;
+}> {
+   return new Promise((resolve, reject) => {
+      // Track accepted sockets so close() can force them down. server.close()
+      // alone only stops accepting and then waits for open connections to end;
+      // a lingering forwarded socket would otherwise hang teardown (and the
+      // afterEach hook) until the test timeout, especially under --serial CI.
+      const sockets = new Set<net.Socket>();
+      const server = net.createServer((socket) => {
+         sockets.add(socket);
+         socket.on("close", () => sockets.delete(socket));
+         socket.pipe(socket);
+      });
+      server.on("error", reject);
+      server.listen(0, "127.0.0.1", () => {
+         const addr = server.address() as net.AddressInfo;
+         resolve({
+            port: addr.port,
+            close: () =>
+               new Promise((res, rej) => {
+                  for (const s of sockets) s.destroy();
+                  server.close((err) => (err ? rej(err) : res()));
+               }),
+         });
+      });
+   });
+}
+
+/**
+ * Start a minimal in-process SSH server that accepts the test client key
+ * and fulfils tcpip-forward (forwardOut) requests.
+ */
+function startSshServer(opts: {
+   rejectAuth?: boolean;
+   rejectForward?: boolean;
+}): Promise<{
+   port: number;
+   hostKeyBase64: string;
+   close: () => Promise<void>;
+}> {
+   return new Promise((resolve, reject) => {
+      // Track live client connections and forwarded destination sockets so
+      // close() can force them down; otherwise a lingering forwarded socket
+      // keeps sshd.close() (and the afterEach hook) hanging until the test
+      // timeout, which is what made the tunnel tests flake under --serial CI.
+      const clients = new Set<SshServerConnection>();
+      const destSockets = new Set<net.Socket>();
+      const sshd = new SshServer(
+         { hostKeys: [hostPrivatePem] },
+         (client: SshServerConnection) => {
+            clients.add(client);
+            client.on("close", () => clients.delete(client));
+            client.on("authentication", (ctx) => {
+               if (opts.rejectAuth) {
+                  ctx.reject();
+                  return;
+               }
+               // Accept publickey auth for our test client key.
+               if (ctx.method === "publickey" && !opts.rejectAuth) {
+                  ctx.accept();
+               } else {
+                  ctx.reject();
+               }
+            });
+
+            client.on("ready", () => {
+               client.on("tcpip", (accept, _reject, info) => {
+                  if (opts.rejectForward) {
+                     _reject();
+                     return;
+                  }
+                  const channel = accept();
+                  // Connect to the real destination and pipe.
+                  const dest = net.createConnection(
+                     { host: info.destIP, port: info.destPort },
+                     () => {
+                        channel.pipe(dest).pipe(channel);
+                        channel.on("error", () => dest.destroy());
+                        dest.on("error", () => channel.destroy());
+                     },
+                  );
+                  destSockets.add(dest);
+                  dest.on("close", () => destSockets.delete(dest));
+                  dest.on("error", () => channel.destroy());
+               });
+            });
+
+            client.on("error", () => {
+               /* swallow per-client errors in tests */
+            });
+         },
+      );
+
+      sshd.on("error", reject);
+      sshd.listen(0, "127.0.0.1", () => {
+         const addr = sshd.address() as net.AddressInfo;
+
+         // Capture the server's host key fingerprint (base64 of the raw DER) by
+         // parsing the pem via the ssh2 util so the format matches what
+         // hostVerifier receives.
+         const parsed = sshUtils.parseKey(hostPrivatePem);
+         const hostKeyBase64 = (parsed as { getPublicSSH(): Buffer })
+            .getPublicSSH()
+            .toString("base64");
+
+         resolve({
+            port: addr.port,
+            hostKeyBase64,
+            close: () =>
+               new Promise((res) => {
+                  for (const s of destSockets) s.destroy();
+                  for (const c of clients) c.end();
+                  sshd.close(() => res());
+               }),
+         });
+      });
+   });
+}
+
+function connectAndSend(port: number, message: string): Promise<string> {
+   return new Promise((resolve, reject) => {
+      const socket = net.createConnection({ host: "127.0.0.1", port }, () => {
+         socket.write(message);
+      });
+      const chunks: Buffer[] = [];
+      socket.on("data", (chunk) => {
+         chunks.push(chunk);
+         if (Buffer.concat(chunks).toString().includes(message)) {
+            socket.destroy();
+            resolve(Buffer.concat(chunks).toString());
+         }
+      });
+      socket.on("error", reject);
+   });
+}
+
+// Best-effort, time-bounded teardown. Destroying every ssh2/socket handle is
+// reliable locally but a lingering graceful close can still stall on some CI
+// platforms (Bun + ssh2 on Linux/Windows), which would hang the afterEach hook
+// until the 100s test timeout. The assertions have already run by teardown, so
+// cap each close and move on rather than block.
+function closeQuietly(close: () => Promise<void>, ms = 3000): Promise<void> {
+   return Promise.race([
+      close().catch(() => {}),
+      new Promise<void>((resolve) => {
+         const timer = setTimeout(resolve, ms);
+         (timer as unknown as { unref?: () => void }).unref?.();
+      }),
+   ]);
+}
+
+// ── Tests ─────────────────────────────────────────────────────────────────────
+
+describe("openProxy — SSH tunnel", () => {
+   let echoServer: { port: number; close: () => Promise<void> };
+   let sshServer: {
+      port: number;
+      hostKeyBase64: string;
+      close: () => Promise<void>;
+   };
+
+   beforeEach(async () => {
+      echoServer = await startEchoServer();
+      sshServer = await startSshServer({});
+   });
+
+   afterEach(async () => {
+      await closeQuietly(() => echoServer.close());
+      await closeQuietly(() => sshServer.close());
+   });
+
+   it("round-trips bytes through the tunnel", async () => {
+      const ep = await openProxy(
+         {
+            type: "ssh",
+            ssh: {
+               host: "127.0.0.1",
+               port: sshServer.port,
+               username: "testuser",
+               privateKey: clientPrivatePem,
+               hostKey: sshServer.hostKeyBase64,
+            },
+         },
+         { host: "127.0.0.1", port: echoServer.port },
+      );
+
+      try {
+         const reply = await connectAndSend(ep.port, "hello-proxy");
+         expect(reply).toContain("hello-proxy");
+      } finally {
+         await closeQuietly(() => ep.close());
+      }
+   });
+
+   it("accepts a hostKey given as a full known_hosts line", async () => {
+      // Users typically paste `ssh-keyscan` output, e.g.
+      // `bastion.example.com ssh-rsa AAAA…` — the verifier must extract the blob.
+      const knownHostsLine = `bastion.example.com ssh-rsa ${sshServer.hostKeyBase64}`;
+      const ep = await openProxy(
+         {
+            type: "ssh",
+            ssh: {
+               host: "127.0.0.1",
+               port: sshServer.port,
+               username: "testuser",
+               privateKey: clientPrivatePem,
+               hostKey: knownHostsLine,
+            },
+         },
+         { host: "127.0.0.1", port: echoServer.port },
+      );
+
+      try {
+         const reply = await connectAndSend(ep.port, "known-hosts-line");
+         expect(reply).toContain("known-hosts-line");
+      } finally {
+         await closeQuietly(() => ep.close());
+      }
+   });
+
+   it("accepts a hashed (|1|) known_hosts line — the hash only obscures the hostname", async () => {
+      // `ssh-keyscan -H` output hashes the hostname (`|1|salt|hash`) but leaves
+      // the key blob in the clear; the verifier extracts the blob token and never
+      // matches hostnames, so hashed lines work the same as unhashed ones.
+      const hashedLine = `|1|dNQdBg9dg8u7Vw8vq3B0uZ3example=|kZ2exampleHashValue0000000= ssh-rsa ${sshServer.hostKeyBase64}`;
+      const ep = await openProxy(
+         {
+            type: "ssh",
+            ssh: {
+               host: "127.0.0.1",
+               port: sshServer.port,
+               username: "testuser",
+               privateKey: clientPrivatePem,
+               hostKey: hashedLine,
+            },
+         },
+         { host: "127.0.0.1", port: echoServer.port },
+      );
+
+      try {
+         const reply = await connectAndSend(ep.port, "hashed-line");
+         expect(reply).toContain("hashed-line");
+      } finally {
+         await closeQuietly(() => ep.close());
+      }
+   });
+
+   it("rejects when hostKey does not match", async () => {
+      const wrongKey = Buffer.alloc(32, 0xff).toString("base64");
+
+      await expect(
+         openProxy(
+            {
+               type: "ssh",
+               ssh: {
+                  host: "127.0.0.1",
+                  port: sshServer.port,
+                  username: "testuser",
+                  privateKey: clientPrivatePem,
+                  hostKey: wrongKey,
+               },
+            },
+            { host: "127.0.0.1", port: echoServer.port },
+         ),
+      ).rejects.toThrow(/host-key mismatch/i);
+   });
+
+   it("rejects when hostKey is absent and opt-out env is not set", async () => {
+      const orig = process.env.PUBLISHER_SSH_ALLOW_UNKNOWN_HOSTKEY;
+      delete process.env.PUBLISHER_SSH_ALLOW_UNKNOWN_HOSTKEY;
+
+      try {
+         await expect(
+            openProxy(
+               {
+                  type: "ssh",
+                  ssh: {
+                     host: "127.0.0.1",
+                     port: sshServer.port,
+                     username: "testuser",
+                     privateKey: clientPrivatePem,
+                  },
+               },
+               { host: "127.0.0.1", port: echoServer.port },
+            ),
+         ).rejects.toThrow(/no hostKey provided/i);
+      } finally {
+         if (orig !== undefined) {
+            process.env.PUBLISHER_SSH_ALLOW_UNKNOWN_HOSTKEY = orig;
+         }
+      }
+   });
+
+   it("connects when hostKey is absent and opt-out env is set", async () => {
+      process.env.PUBLISHER_SSH_ALLOW_UNKNOWN_HOSTKEY = "true";
+      try {
+         const ep = await openProxy(
+            {
+               type: "ssh",
+               ssh: {
+                  host: "127.0.0.1",
+                  port: sshServer.port,
+                  username: "testuser",
+                  privateKey: clientPrivatePem,
+               },
+            },
+            { host: "127.0.0.1", port: echoServer.port },
+         );
+         try {
+            const reply = await connectAndSend(ep.port, "allow-unknown");
+            expect(reply).toContain("allow-unknown");
+         } finally {
+            await closeQuietly(() => ep.close());
+         }
+      } finally {
+         delete process.env.PUBLISHER_SSH_ALLOW_UNKNOWN_HOSTKEY;
+      }
+   });
+
+   it("rejects for unsupported proxy type", async () => {
+      await expect(
+         openProxy(
+            // @ts-expect-error — intentionally passing an unsupported type
+            { type: "unsupported" },
+            { host: "127.0.0.1", port: echoServer.port },
+         ),
+      ).rejects.toThrow(/not supported/i);
+   });
+});