@malloy-publisher/server 0.0.218 → 0.0.220

package/src/service/connection_config.ts

@@ -1,10 +1,18 @@
 import { createPrivateKey } from "crypto";
+import { existsSync } from "fs";
 import path from "path";
 import { components } from "../api";
+import { parseHostKeys } from "./proxy";
 
 type ApiConnection = components["schemas"]["Connection"];
 type AttachedDatabase = components["schemas"]["AttachedDatabase"];
 
+// TLS modes accepted on a proxied postgres connection. Canonical here (rather
+// than in connection.ts, which imports this module) so both the config-load
+// validator and the connect-time builder derive from one list. Mirrors the
+// `sslmode` enum in api-doc.yaml.
+export const PROXIED_SSLMODES = ["disable", "no-verify", "verify-ca"] as const;
+
 export type CoreConnectionEntry = {
    is: string;
    [key: string]: unknown;
@@ -265,8 +273,11 @@ function validateConnectionShape(connection: ApiConnection): void {
       // authorized by whoever configures the connection — deliberately NOT gated
       // by an env flag, and kept separate from the `publisher` HTTP multi-hop
       // type's PUBLISHER_ALLOW_PROXY_CONNECTIONS gate below (that flag is about
-      // publisher-to-publisher proxying, a distinct operator decision). Host-key
-      // pinning is fail-closed at connect time (see openProxy).
+      // publisher-to-publisher proxying, a distinct operator decision). Optional
+      // host-key pinning is fail-closed at connect time when configured (see
+      // openProxy); the proxy-specific fields are validated up front below so a
+      // permanent misconfig fails at config load, not by repeatedly dialing the
+      // tenant's bastion at query time.
       if (connection.proxy.type !== "ssh") {
          throw new Error(
             `Connection '${connection.name}' has an unsupported proxy type '${connection.proxy.type}'. Only 'ssh' is supported.`,
@@ -303,6 +314,74 @@ function validateConnectionShape(connection: ApiConnection): void {
                `postgres connection; the connectionString form is not supported with a proxy.`,
          );
       }
+
+      // hostKey is optional (omitted or empty string => connect unpinned), but a
+      // non-empty hostKey that parses to zero keys — only blank lines, whitespace,
+      // or `#` comments, e.g. a paste that grabbed just ssh-keyscan's
+      // `# host:port ...` header — is a misconfigured pin, not a licence to
+      // connect unverified. Reject it here so the operator gets a config error
+      // instead of a silently unpinned tunnel. (Truthiness, not trim(): "" is the
+      // unpinned signal; "   " is a non-empty value that must yield a key.)
+      const hostKey = connection.proxy.ssh?.hostKey;
+      if (hostKey && parseHostKeys(hostKey).size === 0) {
+         throw new Error(
+            `Connection proxy on '${connection.name}' has a hostKey with no usable host-key line ` +
+               `(only blanks/comments). Provide an OpenSSH known_hosts line or base64 blob, or omit ` +
+               `hostKey to connect unpinned.`,
+         );
+      }
+
+      // Validate the proxied TLS mode up front. The tunnel is dialed lazily on
+      // first lookup and a failed build is retried on every subsequent query, so
+      // a permanent sslmode misconfig left to throw at connect time would re-dial
+      // the tenant's bastion indefinitely. Fail at config load instead.
+      // `!= null` (not truthiness) so a present-but-empty sslmode ("") is caught
+      // here as unsupported rather than slipping through to fail at tunnel-build;
+      // null/undefined mean unset (server applies the default).
+      const sslmode = connection.postgresConnection?.sslmode;
+      if (sslmode != null) {
+         if (!(PROXIED_SSLMODES as readonly string[]).includes(sslmode)) {
+            throw new Error(
+               `Connection proxy on '${connection.name}' has unsupported sslmode '${sslmode}' ` +
+                  `(expected ${PROXIED_SSLMODES.join(" | ")}).`,
+            );
+         }
+         if (sslmode === "verify-ca") {
+            const caBundle = process.env.NODE_EXTRA_CA_CERTS;
+            if (!caBundle || !existsSync(caBundle)) {
+               throw new Error(
+                  `Connection proxy on '${connection.name}' uses sslmode 'verify-ca' but no readable ` +
+                     `CA bundle is available (NODE_EXTRA_CA_CERTS is unset or points at a missing file). ` +
+                     `Add the CA bundle to the image or use sslmode 'no-verify'.`,
+               );
+            }
+         }
+      }
+
+      // The proxied path builds a connectionString to the local tunnel endpoint;
+      // pg decodes the database path with decodeURI, which leaves URI-reserved
+      // characters percent-encoded — so a db name containing them would resolve
+      // to the wrong database. Reject it clearly rather than failing later with a
+      // confusing "database does not exist". (user/password use decodeURIComponent
+      // on parse and round-trip fine.)
+      const dbName = connection.postgresConnection?.databaseName;
+      if (dbName && decodeURI(encodeURIComponent(dbName)) !== dbName) {
+         throw new Error(
+            `Connection proxy on '${connection.name}' has a database name with characters that can't ` +
+               `be carried over a proxied connection (${JSON.stringify(dbName)}). Use a database name ` +
+               `without URI-reserved characters (; , / ? : @ & = + $ #).`,
+         );
+      }
+   }
+
+   // sslmode is only honored on the proxied path (the direct path builds TLS from
+   // the deployment PGSSLMODE). Reject it on a non-proxied connection so a tenant
+   // who sets it doesn't silently get a different TLS posture than they asked for.
+   if (!connection.proxy && connection.postgresConnection?.sslmode) {
+      throw new Error(
+         `Connection '${connection.name}' sets postgresConnection.sslmode but has no proxy; sslmode is ` +
+            `only supported for proxied connections (direct connections use the deployment PGSSLMODE).`,
+      );
    }
 
    switch (connection.type) {

package/src/service/connection_fingerprint.spec.ts

@@ -0,0 +1,102 @@
+import { describe, expect, it } from "bun:test";
+import { components } from "../api";
+import {
+   buildEnvironmentMalloyConfig,
+   EnvironmentMalloyConfig,
+} from "./connection";
+
+type ApiConnection = components["schemas"]["Connection"];
+
+/**
+ * The connection `fingerprint` seam: when a connection carries an API
+ * fingerprint, the resolved Malloy connection's getDigest() must return it
+ * verbatim. Every consumer of a connection's identity — build planning, the
+ * package-load worker's digest RPC, and Malloy's serve-time manifest
+ * resolution — reads getDigest() off a connection resolved through the same
+ * environment lookup wrapper, so this one seam is what keeps build-time and
+ * serve-time source ids in agreement.
+ *
+ * Uses a postgres connection because it is constructed lazily (no network I/O
+ * until a query runs), so digests can be read offline.
+ */
+
+function pgConnection(overrides: Partial<ApiConnection> = {}): ApiConnection {
+   return {
+      name: "pg",
+      type: "postgres",
+      postgresConnection: {
+         host: "db.example.com",
+         port: 5432,
+         databaseName: "analytics",
+         userName: "svc_user",
+         password: "hunter2",
+      },
+      ...overrides,
+   };
+}
+
+async function digestOf(connection: ApiConnection): Promise<string> {
+   let config: EnvironmentMalloyConfig | undefined;
+   try {
+      config = buildEnvironmentMalloyConfig([connection]);
+      const resolved = await config.malloyConfig.connections.lookupConnection(
+         connection.name,
+      );
+      return resolved.getDigest();
+   } finally {
+      await config?.releaseConnections();
+   }
+}
+
+describe("connection fingerprint", () => {
+   it("uses the fingerprint verbatim as the connection digest", async () => {
+      const digest = await digestOf(
+         pgConnection({ fingerprint: "fp-opaque-token-1" }),
+      );
+      expect(digest).toBe("fp-opaque-token-1");
+   });
+
+   it("keeps the digest stable across a credential rotation", async () => {
+      const before = await digestOf(pgConnection({ fingerprint: "fp-stable" }));
+      const after = await digestOf(
+         pgConnection({
+            fingerprint: "fp-stable",
+            postgresConnection: {
+               host: "db.example.com",
+               port: 5432,
+               databaseName: "analytics",
+               userName: "rotated_user",
+               password: "rotated-password",
+            },
+         }),
+      );
+      expect(after).toBe(before);
+   });
+
+   it("changes the digest when the fingerprint changes", async () => {
+      const one = await digestOf(pgConnection({ fingerprint: "fp-1" }));
+      const two = await digestOf(pgConnection({ fingerprint: "fp-2" }));
+      expect(one).not.toBe(two);
+   });
+
+   it("falls back to the locally derived digest when omitted", async () => {
+      const digest = await digestOf(pgConnection());
+      expect(typeof digest).toBe("string");
+      expect(digest.length).toBeGreaterThan(0);
+      // Behavior is unchanged from before the fingerprint existed: the digest
+      // is the connection's own derivation, deterministic for one config.
+      expect(await digestOf(pgConnection())).toBe(digest);
+   });
+
+   it("round-trips the fingerprint on the API connection", async () => {
+      const config = buildEnvironmentMalloyConfig([
+         pgConnection({ fingerprint: "fp-round-trip" }),
+      ]);
+      try {
+         const api = config.apiConnections.find((c) => c.name === "pg");
+         expect(api?.fingerprint).toBe("fp-round-trip");
+      } finally {
+         await config.releaseConnections();
+      }
+   });
+});

package/src/service/manifest_loader.spec.ts

@@ -29,9 +29,9 @@ describe("fetchManifestEntries", () => {
          builtAt: new Date().toISOString(),
          strict: false,
          entries: {
-            b1: { buildId: "b1", physicalTableName: "schema.orders_mz" },
+            b1: { sourceEntityId: "b1", physicalTableName: "schema.orders_mz" },
             b2: {
-               buildId: "b2",
+               sourceEntityId: "b2",
                physicalTableName: "schema.daily_mz",
                connectionName: "bq",
             },
@@ -48,7 +48,7 @@ describe("fetchManifestEntries", () => {
 
    it("reads via a file:// URI", async () => {
       const file = await writeManifest({
-         entries: { b1: { buildId: "b1", physicalTableName: "t1" } },
+         entries: { b1: { sourceEntityId: "b1", physicalTableName: "t1" } },
       });
 
       const entries = await fetchManifestEntries(pathToFileURL(file).href);
@@ -59,8 +59,8 @@ describe("fetchManifestEntries", () => {
    it("skips entries without a physicalTableName", async () => {
       const file = await writeManifest({
          entries: {
-            good: { buildId: "good", physicalTableName: "t1" },
-            bad: { buildId: "bad" },
+            good: { sourceEntityId: "good", physicalTableName: "t1" },
+            bad: { sourceEntityId: "bad" },
          },
       });
 

package/src/service/manifest_loader.ts

@@ -58,7 +58,7 @@ async function readManifestBytes(uri: string): Promise<string> {
 
 /**
  * Fetch and parse the control-plane-computed build manifest at `uri`, returning
- * the Malloy-runtime binding map (`buildId -> { tableName }`). The wire manifest
+ * the Malloy-runtime binding map (`sourceEntityId -> { tableName }`). The wire manifest
  * keys physical tables under `physicalTableName`; the Malloy runtime consumes
  * `tableName`, so we translate here. Entries missing a physical table are
  * skipped. Throws if the URI can't be read or parsed.
@@ -80,16 +80,16 @@ export async function fetchManifestEntries(
    }
 
    const entries: BuildManifest["entries"] = {};
-   for (const [buildId, entry] of Object.entries(parsed.entries ?? {})) {
+   for (const [sourceEntityId, entry] of Object.entries(parsed.entries ?? {})) {
       const physicalTableName = entry?.physicalTableName;
       if (!physicalTableName) {
          logger.warn("Manifest entry has no physicalTableName; skipping", {
             uri,
-            buildId,
+            sourceEntityId,
          });
          continue;
       }
-      entries[buildId] = { tableName: physicalTableName };
+      entries[sourceEntityId] = { tableName: physicalTableName };
    }
    return entries;
 }

package/src/service/materialization_service.spec.ts

@@ -236,13 +236,15 @@ describe("MaterializationService", () => {
          ).toMatchObject({ mode: "orchestrated" });
       });
 
-      it("rejects instructions referencing an unknown buildId before creating", async () => {
+      it("rejects instructions referencing an unknown sourceEntityId before creating", async () => {
          setPackage(ctx.environmentStore, {
             getBuildPlan: () => makeBuildPlan(),
          });
          await expect(
             ctx.service.createMaterialization("my-env", "pkg", {
-               buildInstructions: [makeInstruction({ buildId: "ghost" })],
+               buildInstructions: [
+                  makeInstruction({ sourceEntityId: "ghost" }),
+               ],
             }),
          ).rejects.toThrow(BadRequestError);
          expect(ctx.repository.createMaterialization.called).toBe(false);
@@ -347,7 +349,7 @@ describe("MaterializationService", () => {
                   strict: false,
                   entries: {
                      b1: {
-                        buildId: "b1",
+                        sourceEntityId: "b1",
                         physicalTableName: "orders_mz",
                         connectionName: "duckdb",
                      },
@@ -383,7 +385,7 @@ describe("MaterializationService", () => {
                   strict: false,
                   entries: {
                      b1: {
-                        buildId: "b1",
+                        sourceEntityId: "b1",
                         physicalTableName: "orders_mz",
                         connectionName: "duckdb",
                      },
@@ -423,7 +425,7 @@ describe("MaterializationService", () => {
                   strict: false,
                   entries: {
                      b1: {
-                        buildId: "b1",
+                        sourceEntityId: "b1",
                         // Container path on a hyphenated BigQuery project: each
                         // segment must be backtick-quoted independently.
                         physicalTableName: "my-proj.ds.engaged",
@@ -460,7 +462,7 @@ describe("MaterializationService", () => {
                   strict: false,
                   entries: {
                      b1: {
-                        buildId: "b1",
+                        sourceEntityId: "b1",
                         physicalTableName: "orders_mz",
                         connectionName: "duckdb",
                      },
@@ -481,7 +483,7 @@ describe("MaterializationService", () => {
 });
 
 describe("stagingSuffix", () => {
-   it("derives a short, stable suffix from the buildId", () => {
+   it("derives a short, stable suffix from the sourceEntityId", () => {
       expect(stagingSuffix("abcdef1234567890")).toBe("_abcdef123456");
    });
 });
@@ -492,17 +494,17 @@ describe("deriveSelfInstructions", () => {
       ctx = createMocks();
    });
 
-   it("carries forward unchanged buildIds and builds the rest (deduping repeats)", () => {
+   it("carries forward unchanged sourceEntityIds and builds the rest (deduping repeats)", () => {
       const compiled = compiledWith(
          {
-            s1: fakeSource({ name: "s1", buildId: "b1aaaaaaaaaaaaaa" }),
-            s2: fakeSource({ name: "s2", buildId: "b2bbbbbbbbbbbbbb" }),
+            s1: fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" }),
+            s2: fakeSource({ name: "s2", sourceEntityId: "b2bbbbbbbbbbbbbb" }),
          },
          [["s1", "s2"], ["s2"]],
       );
       const priorEntries = {
          b1aaaaaaaaaaaaaa: {
-            buildId: "b1aaaaaaaaaaaaaa",
+            sourceEntityId: "b1aaaaaaaaaaaaaa",
             physicalTableName: "s1_prev",
             connectionName: "duckdb",
          },
@@ -519,7 +521,7 @@ describe("deriveSelfInstructions", () => {
       ).deriveSelfInstructions(compiled, undefined, priorEntries);
 
       expect(instructions).toHaveLength(1);
-      expect(instructions[0].buildId).toBe("b2bbbbbbbbbbbbbb");
+      expect(instructions[0].sourceEntityId).toBe("b2bbbbbbbbbbbbbb");
       expect(instructions[0].physicalTableName).toBe("s2");
       expect(instructions[0].realization).toBe("COPY");
       expect(instructions[0].materializedTableId).toBe("local-b2bbbbbbbbbb");
@@ -531,8 +533,8 @@ describe("deriveSelfInstructions", () => {
    it("honors the sourceNames filter (excluded sources are neither built nor carried)", () => {
       const compiled = compiledWith(
          {
-            s1: fakeSource({ name: "s1", buildId: "b1aaaaaaaaaaaaaa" }),
-            s2: fakeSource({ name: "s2", buildId: "b2bbbbbbbbbbbbbb" }),
+            s1: fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" }),
+            s2: fakeSource({ name: "s2", sourceEntityId: "b2bbbbbbbbbbbbbb" }),
          },
          [["s1", "s2"]],
       );
@@ -547,7 +549,7 @@ describe("deriveSelfInstructions", () => {
       ).deriveSelfInstructions(compiled, ["s2"], {});
 
       expect(instructions).toHaveLength(1);
-      expect(instructions[0].buildId).toBe("b2bbbbbbbbbbbbbb");
+      expect(instructions[0].sourceEntityId).toBe("b2bbbbbbbbbbbbbb");
       expect(Object.keys(carried as Record<string, unknown>)).toHaveLength(0);
    });
 });
@@ -561,7 +563,7 @@ describe("getMostRecentManifestEntries (skip-if-unchanged)", () => {
    it("returns entries from the most recent successful run, excluding the in-flight one", async () => {
       const entries = {
          b1: {
-            buildId: "b1",
+            sourceEntityId: "b1",
             physicalTableName: "orders_v1",
             connectionName: "duckdb",
          },
@@ -599,7 +601,9 @@ describe("getMostRecentManifestEntries (skip-if-unchanged)", () => {
             manifest: {
                builtAt: "t",
                strict: false,
-               entries: { b1: { buildId: "b1", physicalTableName: "t1" } },
+               entries: {
+                  b1: { sourceEntityId: "b1", physicalTableName: "t1" },
+               },
             },
          }),
       ]);
@@ -679,8 +683,8 @@ describe("executeInstructedBuild", () => {
    it("builds instructed sources in dependency order and seeds carried entries", async () => {
       const runSQL = sinon.stub().resolves();
       const connection = { runSQL } as unknown as MalloyConnection;
-      const s1 = fakeSource({ name: "s1", buildId: "b1aaaaaaaaaaaaaa" });
-      const s2 = fakeSource({ name: "s2", buildId: "b2bbbbbbbbbbbbbb" });
+      const s1 = fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" });
+      const s2 = fakeSource({ name: "s2", sourceEntityId: "b2bbbbbbbbbbbbbb" });
       const compiled = compiledWith(
          { s1, s2 },
          [["s1"], ["s2"]],
@@ -691,13 +695,13 @@ describe("executeInstructedBuild", () => {
          compiled,
          [
             {
-               buildId: "b1aaaaaaaaaaaaaa",
+               sourceEntityId: "b1aaaaaaaaaaaaaa",
                materializedTableId: "mt-1",
                physicalTableName: "s1_v1",
                realization: "COPY",
             },
             {
-               buildId: "b2bbbbbbbbbbbbbb",
+               sourceEntityId: "b2bbbbbbbbbbbbbb",
                materializedTableId: "mt-2",
                physicalTableName: "s2_v1",
                realization: "COPY",
@@ -705,7 +709,7 @@ describe("executeInstructedBuild", () => {
          ],
          {
             carried0: {
-               buildId: "carried0",
+               sourceEntityId: "carried0",
                physicalTableName: "carried_tbl",
                connectionName: "duckdb",
             },
@@ -725,8 +729,14 @@ describe("executeInstructedBuild", () => {
       // though the caller instructed it. Both must now build, `mid` first.
       const runSQL = sinon.stub().resolves();
       const connection = { runSQL } as unknown as MalloyConnection;
-      const root = fakeSource({ name: "root", buildId: "brootaaaaaaaaaa" });
-      const mid = fakeSource({ name: "mid", buildId: "bmidbbbbbbbbbbb" });
+      const root = fakeSource({
+         name: "root",
+         sourceEntityId: "brootaaaaaaaaaa",
+      });
+      const mid = fakeSource({
+         name: "mid",
+         sourceEntityId: "bmidbbbbbbbbbbb",
+      });
       const compiled = {
          graphs: [
             {
@@ -750,13 +760,13 @@ describe("executeInstructedBuild", () => {
          compiled,
          [
             {
-               buildId: "brootaaaaaaaaaa",
+               sourceEntityId: "brootaaaaaaaaaa",
                materializedTableId: "mt-r",
                physicalTableName: "root_v1",
                realization: "COPY",
             },
             {
-               buildId: "bmidbbbbbbbbbbb",
+               sourceEntityId: "bmidbbbbbbbbbbb",
                materializedTableId: "mt-m",
                physicalTableName: "mid_v1",
                realization: "COPY",
@@ -779,7 +789,7 @@ describe("executeInstructedBuild", () => {
    });
 
    it("throws when an instructed graph's connection is missing", async () => {
-      const s1 = fakeSource({ name: "s1", buildId: "b1aaaaaaaaaaaaaa" });
+      const s1 = fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" });
       const compiled = compiledWith({ s1 }, [["s1"]], new Map());
 
       await expect(
@@ -787,7 +797,7 @@ describe("executeInstructedBuild", () => {
             compiled,
             [
                {
-                  buildId: "b1aaaaaaaaaaaaaa",
+                  sourceEntityId: "b1aaaaaaaaaaaaaa",
                   materializedTableId: "mt-1",
                   physicalTableName: "s1_v1",
                   realization: "COPY",
@@ -809,15 +819,15 @@ describe("buildOneSource", () => {
       connection: { runSQL: sinon.SinonStub },
       physicalTableName: string,
       dialectName?: string,
-   ): Promise<{ buildId: string; physicalTableName: string }> {
+   ): Promise<{ sourceEntityId: string; physicalTableName: string }> {
       const source = fakeSource({
          name: "orders",
-         buildId: "abcdef1234567890",
+         sourceEntityId: "abcdef1234567890",
          sql: "SELECT * FROM t",
          dialectName,
       });
       const instruction: BuildInstruction = {
-         buildId: "abcdef1234567890",
+         sourceEntityId: "abcdef1234567890",
          materializedTableId: "mt-1",
          physicalTableName,
          realization: "COPY",
@@ -830,7 +840,7 @@ describe("buildOneSource", () => {
                c: unknown,
                d: Record<string, string>,
                m: Manifest,
-            ) => Promise<{ buildId: string; physicalTableName: string }>;
+            ) => Promise<{ sourceEntityId: string; physicalTableName: string }>;
          }
       ).buildOneSource(
          source,
@@ -853,7 +863,7 @@ describe("buildOneSource", () => {
          'ALTER TABLE "orders_v1_abcdef123456" RENAME TO "orders_v1"',
       ]);
       expect(entry.physicalTableName).toBe("orders_v1");
-      expect(entry.buildId).toBe("abcdef1234567890");
+      expect(entry.sourceEntityId).toBe("abcdef1234567890");
    });
 
    it("drops the staging table and rethrows when the build SQL fails", async () => {
@@ -919,7 +929,7 @@ describe("runBuild (branch behavior)", () => {
    function stubEngine(): RunBuildInternals {
       const entries = {
          "build-orders": {
-            buildId: "build-orders",
+            sourceEntityId: "build-orders",
             physicalTableName: '"orders_v1"',
             connectionName: "duckdb",
          },
@@ -995,11 +1005,11 @@ describe("autoLoadManifest", () => {
       const reloadAllModelsForPackage = sinon.stub().resolves();
       const entries = {
          b1: {
-            buildId: "b1",
+            sourceEntityId: "b1",
             physicalTableName: "orders_v1",
             connectionName: "duckdb",
          },
-         b2: { buildId: "b2", physicalTableName: undefined },
+         b2: { sourceEntityId: "b2", physicalTableName: undefined },
       };
 
       await (
@@ -1033,7 +1043,7 @@ describe("autoLoadManifest", () => {
                ) => Promise<void>;
             }
          ).autoLoadManifest({ reloadAllModelsForPackage }, "pkg", {
-            b1: { buildId: "b1", physicalTableName: "t" },
+            b1: { sourceEntityId: "b1", physicalTableName: "t" },
          }),
       ).resolves.toBeUndefined();
    });