@malloy-publisher/server 0.0.219 → 0.0.220

package/src/server.ts

@@ -20,6 +20,7 @@ import {
    BadRequestError,
    internalErrorToHttpError,
    NotImplementedError,
+   ServiceUnavailableError,
 } from "./errors";
 import {
    drainingGuard,
@@ -38,6 +39,7 @@ import { setFilterDeprecationHeaders } from "./filter_deprecation";
 import { checkHeapConfiguration } from "./heap_check";
 import { queryConcurrency } from "./query_concurrency";
 import { MaterializationController } from "./controller/materialization.controller";
+import { ThemeController } from "./controller/theme.controller";
 import { initializeMcpServer } from "./mcp/server";
 import { startAgentMcpServer } from "./mcp/agent_server";
 import { registerLegacyRoutes } from "./server-old";
@@ -45,6 +47,7 @@ import { EnvironmentStore } from "./service/environment_store";
 import { MaterializationService } from "./service/materialization_service";
 import { normalizeQueryArray } from "./query_param_utils";
 import { PackageMemoryGovernor } from "./service/package_memory_governor";
+import { ThemeStore } from "./service/theme_store";
 import { assertSafePackageName, safeJoinUnderRoot } from "./path_safety";
 
 export { normalizeQueryArray } from "./query_param_utils";
@@ -207,6 +210,8 @@ const materializationService = new MaterializationService(environmentStore);
 const materializationController = new MaterializationController(
    materializationService,
 );
+const themeStore = new ThemeStore(environmentStore.storageManager, SERVER_ROOT);
+const themeController = new ThemeController(themeStore, SERVER_ROOT);
 
 export const mcpApp = express();
 
@@ -722,7 +727,20 @@ app.get(
 app.get(`${API_PREFIX}/status`, async (_req, res) => {
    try {
       const status = await environmentStore.getStatus();
-      res.status(200).json(status);
+      // Compose theme onto the status response so the SDK can read both
+      // in one round trip on app boot. ThemeStore is the source of truth;
+      // publisher.config.json is only a boot seed (see ThemeStore). The
+      // field is always present (an empty object means "no overrides
+      // yet"), so the OpenAPI shape and the runtime payload agree.
+      // The theme here is cosmetic, so during the brief window before storage
+      // initializes report no overrides rather than 500 an endpoint the
+      // control plane polls for serving state (themeStore.get() throws until
+      // storage is ready). GET /theme, the editor's authoritative load, is
+      // answered with 503 during that window instead.
+      const theme = environmentStore.storageManager.isInitialized()
+         ? await themeStore.get()
+         : undefined;
+      res.status(200).json({ ...status, theme: theme ?? {} });
    } catch (error) {
       logger.error("Error getting status", { error });
       const { json, status } = internalErrorToHttpError(error as Error);
@@ -730,6 +748,45 @@ app.get(`${API_PREFIX}/status`, async (_req, res) => {
    }
 });
 
+app.get(`${API_PREFIX}/theme`, async (_req, res) => {
+   try {
+      if (!environmentStore.storageManager.isInitialized()) {
+         // Storage is still initializing. Answer 503 (not 200 with an empty
+         // theme) so the Theme Editor's load stays in an error state and
+         // never adopts {} as the authoritative saved baseline, which would
+         // let a subsequent edit auto-save {} over the real persisted theme.
+         throw new ServiceUnavailableError(
+            "Theme storage is still initializing. Retry shortly.",
+         );
+      }
+      res.status(200).json(await themeController.getTheme());
+   } catch (error) {
+      logger.error("Error getting theme", { error });
+      const { json, status } = internalErrorToHttpError(error as Error);
+      res.status(status).json(json);
+   }
+});
+
+app.put(`${API_PREFIX}/theme`, async (req, res) => {
+   try {
+      res.status(200).json(await themeController.putTheme(req.body));
+   } catch (error) {
+      logger.error("Error saving theme", { error });
+      const { json, status } = internalErrorToHttpError(error as Error);
+      res.status(status).json(json);
+   }
+});
+
+app.delete(`${API_PREFIX}/theme`, async (_req, res) => {
+   try {
+      res.status(200).json(await themeController.resetTheme());
+   } catch (error) {
+      logger.error("Error resetting theme", { error });
+      const { json, status } = internalErrorToHttpError(error as Error);
+      res.status(status).json(json);
+   }
+});
+
 app.get(`${API_PREFIX}/watch-mode/status`, watchModeController.getWatchStatus);
 app.post(`${API_PREFIX}/watch-mode/start`, watchModeController.startWatching);
 app.post(`${API_PREFIX}/watch-mode/stop`, watchModeController.stopWatchMode);

package/src/service/build_plan.spec.ts

@@ -4,7 +4,7 @@ import * as sinon from "sinon";
 import type { BuildGraph as MalloyBuildGraph } from "@malloydata/malloy";
 import {
    compilePackageBuildPlan,
-   computeBuildId,
+   computeSourceEntityId,
    computePackageBuildPlan,
    deriveAnnotationFields,
    deriveBuildPlan,
@@ -27,8 +27,8 @@ describe("flattenDependsOn", () => {
 
 describe("iterGraphSources", () => {
    it("yields resolvable sources in dependency order, skipping missing ones", () => {
-      const a = fakeSource({ name: "a", buildId: "ba" });
-      const b = fakeSource({ name: "b", buildId: "bb" });
+      const a = fakeSource({ name: "a", sourceEntityId: "ba" });
+      const b = fakeSource({ name: "b", sourceEntityId: "bb" });
       const graph = {
          connectionName: "duckdb",
          nodes: [
@@ -52,9 +52,9 @@ describe("iterGraphSources", () => {
       // and every transitive persist dependency is nested in dependsOn. All
       // three must be yielded (so all get built), leaf-first so a downstream
       // build reads its upstream's freshly materialized table.
-      const root = fakeSource({ name: "root", buildId: "br" });
-      const mid = fakeSource({ name: "mid", buildId: "bm" });
-      const leaf = fakeSource({ name: "leaf", buildId: "bl" });
+      const root = fakeSource({ name: "root", sourceEntityId: "br" });
+      const mid = fakeSource({ name: "mid", sourceEntityId: "bm" });
+      const leaf = fakeSource({ name: "leaf", sourceEntityId: "bl" });
       const graph = {
          connectionName: "duckdb",
          nodes: [
@@ -85,9 +85,9 @@ describe("iterGraphSources", () => {
    it("deduplicates a shared (diamond) dependency across roots", () => {
       // r1 and r2 both depend on `shared`; it must be yielded exactly once and
       // before both dependents.
-      const r1 = fakeSource({ name: "r1", buildId: "b1" });
-      const r2 = fakeSource({ name: "r2", buildId: "b2" });
-      const shared = fakeSource({ name: "shared", buildId: "bs" });
+      const r1 = fakeSource({ name: "r1", sourceEntityId: "b1" });
+      const r2 = fakeSource({ name: "r2", sourceEntityId: "b2" });
+      const shared = fakeSource({ name: "shared", sourceEntityId: "bs" });
       const graph = {
          connectionName: "duckdb",
          nodes: [
@@ -149,7 +149,7 @@ describe("deriveAnnotationFields", () => {
    });
 });
 
-describe("computeBuildId", () => {
+describe("computeSourceEntityId", () => {
    it("delegates to PersistSource.makeBuildId with the connection digest and SQL", () => {
       const makeBuildId = sinon.stub().returns("computed-id");
       const source = {
@@ -158,7 +158,7 @@ describe("computeBuildId", () => {
          getSQL: () => "SELECT 7",
       } as unknown as PersistSource;
 
-      const id = computeBuildId(source, { duckdb: "dig-1" });
+      const id = computeSourceEntityId(source, { duckdb: "dig-1" });
 
       expect(id).toBe("computed-id");
       expect(makeBuildId.calledOnceWithExactly("dig-1", "SELECT 7")).toBe(true);
@@ -188,7 +188,7 @@ describe("deriveBuildPlan", () => {
    it("projects graphs and sources into the wire build plan", () => {
       const orders = fakeSource({
          name: "orders",
-         buildId: "bid-orders",
+         sourceEntityId: "bid-orders",
          sql: "SELECT 1",
       });
       const plan = deriveBuildPlan(
@@ -206,15 +206,15 @@ describe("deriveBuildPlan", () => {
       expect(plan.sources["orders@m"]).toMatchObject({
          name: "orders",
          connectionName: "duckdb",
-         buildId: "bid-orders",
+         sourceEntityId: "bid-orders",
          sql: "SELECT 1",
          columns: [],
       });
    });
 
    it("honors the sourceNames filter", () => {
-      const a = fakeSource({ name: "a", buildId: "bid-a" });
-      const b = fakeSource({ name: "b", buildId: "bid-b" });
+      const a = fakeSource({ name: "a", sourceEntityId: "bid-a" });
+      const b = fakeSource({ name: "b", sourceEntityId: "bid-b" });
       const plan = deriveBuildPlan(
          [
             {
@@ -231,8 +231,8 @@ describe("deriveBuildPlan", () => {
    });
 
    it("carries the per-source package-relative modelPath", () => {
-      const a = fakeSource({ name: "a", buildId: "bid-a" });
-      const b = fakeSource({ name: "b", buildId: "bid-b" });
+      const a = fakeSource({ name: "a", sourceEntityId: "bid-a" });
+      const b = fakeSource({ name: "b", sourceEntityId: "bid-b" });
       const plan = deriveBuildPlan(
          [
             {

package/src/service/build_plan.ts

@@ -112,8 +112,8 @@ export function flattenDependsOn(node: {
  * persist dependency is nested under a root in its recursive {@code dependsOn}
  * tree (and present in {@code sources}). Walking only {@code graph.nodes}
  * silently skips every intermediate persist source, so it never gets
- * materialized (it only gets a table by coincidence when it shares a buildId
- * with a root). We post-order DFS the {@code dependsOn} tree
+ * materialized (it only gets a table by coincidence when it shares a
+ * sourceEntityId with a root). We post-order DFS the {@code dependsOn} tree
  * so dependencies are built first (a downstream build can then read its upstream
  * source's freshly materialized table), deduplicating shared (diamond)
  * dependencies by sourceID so each is yielded once. This mirrors the canonical
@@ -147,11 +147,23 @@ export function* iterGraphSources(
 }
 
 /**
- * The buildId for a persist source: a stable digest of its connection identity
- * and canonical SQL. Centralizes the (source, connectionDigests) call shape so
- * planning, self-instruction, and build all agree on the same id.
+ * The sourceEntityId for a persist source: a stable content address of its
+ * connection identity and canonical SQL. Centralizes the
+ * (source, connectionDigests) call shape so planning, self-instruction, build,
+ * and serve-time manifest resolution all agree on the same id.
+ *
+ * <p>This is the single seam between the publisher and the Malloy compiler's
+ * source-identity recipe. Today it delegates to
+ * {@code PersistSource.makeBuildId(connectionDigest, sql)} — the compiler's
+ * current hex content hash. The connection's contribution is already
+ * fingerprint-aware: when a connection carries an API `fingerprint`, its
+ * digest IS that fingerprint verbatim (see applyConnectionFingerprint in
+ * connection.ts), so ids stay stable across credential rotation. When the
+ * compiler ships the scoped UUID5 sourceEntityId recipe (scope + connection
+ * fingerprint + canonical SQL), swap the delegation below to the new compiler
+ * API — this function is the only place the publisher derives the id.
  */
-export function computeBuildId(
+export function computeSourceEntityId(
    source: PersistSource,
    connectionDigests: Record<string, string>,
 ): string {
@@ -246,9 +258,9 @@ export async function compilePackageBuildPlan(
       const conn = connections.get(graph.connectionName);
       if (!conn) {
          // The connection failed to resolve (already warned in
-         // resolvePackageConnections). Its buildIds will be computed without a
-         // digest, so surface it as a discrete correctness signal rather than
-         // skipping silently.
+         // resolvePackageConnections). Its sourceEntityIds will be computed
+         // without a digest, so surface it as a discrete correctness signal
+         // rather than skipping silently.
          recordConnectionDigestSkipped();
          logger.warn("Skipping connection digest; connection did not resolve", {
             connectionName: graph.connectionName,
@@ -256,6 +268,9 @@ export async function compilePackageBuildPlan(
          continue;
       }
       if (!connectionDigests[graph.connectionName]) {
+         // getDigest() is fingerprint-aware: a connection configured with an
+         // API `fingerprint` returns it verbatim (applyConnectionFingerprint),
+         // so a credential rotation does not re-address its sources.
          connectionDigests[graph.connectionName] = await conn.getDigest();
       }
    }
@@ -297,7 +312,7 @@ export function deriveBuildPlan(
          sourceID: source.sourceID,
          connectionName: source.connectionName,
          dialect: source.dialectName,
-         buildId: computeBuildId(source, connectionDigests),
+         sourceEntityId: computeSourceEntityId(source, connectionDigests),
          sql: source.getSQL(),
          columns: deriveColumns(source),
          annotationFields: deriveAnnotationFields(source),

package/src/service/connection.ts

@@ -1028,6 +1028,38 @@ function getMetadataForLookup(
    return name ? metadata.get(name) : undefined;
 }
 
+/**
+ * Make a connection's digest the caller-supplied API `fingerprint`, verbatim.
+ *
+ * The fingerprint is the connection's data identity as declared by the caller
+ * (secret-free, stable across credential rotation). When set, it replaces the
+ * locally derived `getDigest()` so it becomes the connection's contribution to
+ * content-addressed source ids everywhere a digest is read:
+ *
+ *  - build planning (`compilePackageBuildPlan` -> `conn.getDigest()`),
+ *  - the package-load worker's connection-metadata RPC, and
+ *  - Malloy's own serve-time manifest resolution (the runtime calls
+ *    `conn.getDigest()` internally when a build manifest is bound).
+ *
+ * Overriding the method on the resolved instance — inside the one
+ * `lookupConnection` wrapper every resolution passes through — is what keeps
+ * build-time and serve-time ids identical; applying it per call site would
+ * invite divergence and a silent live-serving fallback. The value is used
+ * verbatim (never re-hashed or combined with local details) and treated as an
+ * opaque token: it is not a secret, but it is never logged or parsed.
+ * Without a fingerprint the connection keeps its locally derived digest.
+ */
+function applyConnectionFingerprint(
+   connection: Connection,
+   metadata: EnvironmentConnectionMetadata | undefined,
+): Connection {
+   const fingerprint = metadata?.apiConnection.fingerprint;
+   if (fingerprint) {
+      connection.getDigest = () => fingerprint;
+   }
+   return connection;
+}
+
 function isDuckDBConnection(
    connection: Connection,
 ): connection is DuckDBConnection {
@@ -1095,10 +1127,11 @@ export function buildEnvironmentMalloyConfig(
    }
 
    malloyConfig.wrapConnections(
-      (base: LookupConnection<Connection>): LookupConnection<Connection> => ({
-         lookupConnection: async (name?: string): Promise<Connection> => {
-            const metadata = getMetadataForLookup(assembled.metadata, name);
-
+      (base: LookupConnection<Connection>): LookupConnection<Connection> => {
+         const resolveConnection = async (
+            name: string | undefined,
+            metadata: EnvironmentConnectionMetadata | undefined,
+         ): Promise<Connection> => {
             if (metadata?.isDuckLake) {
                let connectionPromise = duckLakeCache.get(name!);
                if (!connectionPromise) {
@@ -1210,8 +1243,21 @@ export function buildEnvironmentMalloyConfig(
                await attachOnce(connection, metadata);
             }
             return connection;
-         },
-      }),
+         };
+
+         return {
+            lookupConnection: async (name?: string): Promise<Connection> => {
+               const metadata = getMetadataForLookup(assembled.metadata, name);
+               // Every resolution branch funnels through this one exit so a
+               // configured fingerprint is applied no matter which wrapper
+               // produced the connection (see applyConnectionFingerprint).
+               return applyConnectionFingerprint(
+                  await resolveConnection(name, metadata),
+                  metadata,
+               );
+            },
+         };
+      },
    );
 
    return {

package/src/service/connection_fingerprint.spec.ts

@@ -0,0 +1,102 @@
+import { describe, expect, it } from "bun:test";
+import { components } from "../api";
+import {
+   buildEnvironmentMalloyConfig,
+   EnvironmentMalloyConfig,
+} from "./connection";
+
+type ApiConnection = components["schemas"]["Connection"];
+
+/**
+ * The connection `fingerprint` seam: when a connection carries an API
+ * fingerprint, the resolved Malloy connection's getDigest() must return it
+ * verbatim. Every consumer of a connection's identity — build planning, the
+ * package-load worker's digest RPC, and Malloy's serve-time manifest
+ * resolution — reads getDigest() off a connection resolved through the same
+ * environment lookup wrapper, so this one seam is what keeps build-time and
+ * serve-time source ids in agreement.
+ *
+ * Uses a postgres connection because it is constructed lazily (no network I/O
+ * until a query runs), so digests can be read offline.
+ */
+
+function pgConnection(overrides: Partial<ApiConnection> = {}): ApiConnection {
+   return {
+      name: "pg",
+      type: "postgres",
+      postgresConnection: {
+         host: "db.example.com",
+         port: 5432,
+         databaseName: "analytics",
+         userName: "svc_user",
+         password: "hunter2",
+      },
+      ...overrides,
+   };
+}
+
+async function digestOf(connection: ApiConnection): Promise<string> {
+   let config: EnvironmentMalloyConfig | undefined;
+   try {
+      config = buildEnvironmentMalloyConfig([connection]);
+      const resolved = await config.malloyConfig.connections.lookupConnection(
+         connection.name,
+      );
+      return resolved.getDigest();
+   } finally {
+      await config?.releaseConnections();
+   }
+}
+
+describe("connection fingerprint", () => {
+   it("uses the fingerprint verbatim as the connection digest", async () => {
+      const digest = await digestOf(
+         pgConnection({ fingerprint: "fp-opaque-token-1" }),
+      );
+      expect(digest).toBe("fp-opaque-token-1");
+   });
+
+   it("keeps the digest stable across a credential rotation", async () => {
+      const before = await digestOf(pgConnection({ fingerprint: "fp-stable" }));
+      const after = await digestOf(
+         pgConnection({
+            fingerprint: "fp-stable",
+            postgresConnection: {
+               host: "db.example.com",
+               port: 5432,
+               databaseName: "analytics",
+               userName: "rotated_user",
+               password: "rotated-password",
+            },
+         }),
+      );
+      expect(after).toBe(before);
+   });
+
+   it("changes the digest when the fingerprint changes", async () => {
+      const one = await digestOf(pgConnection({ fingerprint: "fp-1" }));
+      const two = await digestOf(pgConnection({ fingerprint: "fp-2" }));
+      expect(one).not.toBe(two);
+   });
+
+   it("falls back to the locally derived digest when omitted", async () => {
+      const digest = await digestOf(pgConnection());
+      expect(typeof digest).toBe("string");
+      expect(digest.length).toBeGreaterThan(0);
+      // Behavior is unchanged from before the fingerprint existed: the digest
+      // is the connection's own derivation, deterministic for one config.
+      expect(await digestOf(pgConnection())).toBe(digest);
+   });
+
+   it("round-trips the fingerprint on the API connection", async () => {
+      const config = buildEnvironmentMalloyConfig([
+         pgConnection({ fingerprint: "fp-round-trip" }),
+      ]);
+      try {
+         const api = config.apiConnections.find((c) => c.name === "pg");
+         expect(api?.fingerprint).toBe("fp-round-trip");
+      } finally {
+         await config.releaseConnections();
+      }
+   });
+});

package/src/service/manifest_loader.spec.ts

@@ -29,9 +29,9 @@ describe("fetchManifestEntries", () => {
          builtAt: new Date().toISOString(),
          strict: false,
          entries: {
-            b1: { buildId: "b1", physicalTableName: "schema.orders_mz" },
+            b1: { sourceEntityId: "b1", physicalTableName: "schema.orders_mz" },
             b2: {
-               buildId: "b2",
+               sourceEntityId: "b2",
                physicalTableName: "schema.daily_mz",
                connectionName: "bq",
             },
@@ -48,7 +48,7 @@ describe("fetchManifestEntries", () => {
 
    it("reads via a file:// URI", async () => {
       const file = await writeManifest({
-         entries: { b1: { buildId: "b1", physicalTableName: "t1" } },
+         entries: { b1: { sourceEntityId: "b1", physicalTableName: "t1" } },
       });
 
       const entries = await fetchManifestEntries(pathToFileURL(file).href);
@@ -59,8 +59,8 @@ describe("fetchManifestEntries", () => {
    it("skips entries without a physicalTableName", async () => {
       const file = await writeManifest({
          entries: {
-            good: { buildId: "good", physicalTableName: "t1" },
-            bad: { buildId: "bad" },
+            good: { sourceEntityId: "good", physicalTableName: "t1" },
+            bad: { sourceEntityId: "bad" },
          },
       });
 

package/src/service/manifest_loader.ts

@@ -58,7 +58,7 @@ async function readManifestBytes(uri: string): Promise<string> {
 
 /**
  * Fetch and parse the control-plane-computed build manifest at `uri`, returning
- * the Malloy-runtime binding map (`buildId -> { tableName }`). The wire manifest
+ * the Malloy-runtime binding map (`sourceEntityId -> { tableName }`). The wire manifest
  * keys physical tables under `physicalTableName`; the Malloy runtime consumes
  * `tableName`, so we translate here. Entries missing a physical table are
  * skipped. Throws if the URI can't be read or parsed.
@@ -80,16 +80,16 @@ export async function fetchManifestEntries(
    }
 
    const entries: BuildManifest["entries"] = {};
-   for (const [buildId, entry] of Object.entries(parsed.entries ?? {})) {
+   for (const [sourceEntityId, entry] of Object.entries(parsed.entries ?? {})) {
       const physicalTableName = entry?.physicalTableName;
       if (!physicalTableName) {
          logger.warn("Manifest entry has no physicalTableName; skipping", {
             uri,
-            buildId,
+            sourceEntityId,
          });
          continue;
       }
-      entries[buildId] = { tableName: physicalTableName };
+      entries[sourceEntityId] = { tableName: physicalTableName };
    }
    return entries;
 }