@malloy-publisher/server 0.0.207 → 0.0.209

package/src/server-old.ts

@@ -944,16 +944,7 @@ export function registerLegacyRoutes(
       async (req, res) => {
          try {
             const action = req.query.action;
-            if (action === "build") {
-               const build =
-                  await materializationController.buildMaterialization(
-                     req.params.projectName,
-                     req.params.packageName,
-                     req.params.materializationId,
-                     req.body || {},
-                  );
-               res.status(202).json(remapMaterializationResponse(build));
-            } else if (action === "stop") {
+            if (action === "stop") {
                const build =
                   await materializationController.stopMaterialization(
                      req.params.projectName,
@@ -963,7 +954,7 @@ export function registerLegacyRoutes(
                res.status(200).json(remapMaterializationResponse(build));
             } else {
                throw new BadRequestError(
-                  `Unsupported action '${String(action ?? "")}'. Expected 'build' or 'stop'.`,
+                  `Unsupported action '${String(action ?? "")}'. Expected 'stop'.`,
                );
             }
          } catch (error) {

package/src/server.ts

@@ -39,6 +39,7 @@ import { checkHeapConfiguration } from "./heap_check";
 import { queryConcurrency } from "./query_concurrency";
 import { MaterializationController } from "./controller/materialization.controller";
 import { initializeMcpServer } from "./mcp/server";
+import { startAgentMcpServer } from "./mcp/agent_server";
 import { registerLegacyRoutes } from "./server-old";
 import { EnvironmentStore } from "./service/environment_store";
 import { MaterializationService } from "./service/materialization_service";
@@ -160,6 +161,7 @@ parseArgs();
 const PUBLISHER_PORT = Number(process.env.PUBLISHER_PORT || 4000);
 const PUBLISHER_HOST = process.env.PUBLISHER_HOST || "0.0.0.0";
 const MCP_PORT = Number(process.env.MCP_PORT || 4040);
+const AGENT_MCP_PORT = Number(process.env.AGENT_MCP_PORT || 4041);
 const MCP_ENDPOINT = "/mcp";
 const SHUTDOWN_DRAIN_DURATION_SECONDS = Number(
    process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0,
@@ -1590,15 +1592,7 @@ app.post(
    async (req, res) => {
       try {
          const action = req.query.action;
-         if (action === "build") {
-            const build = await materializationController.buildMaterialization(
-               req.params.environmentName,
-               req.params.packageName,
-               req.params.materializationId,
-               req.body || {},
-            );
-            res.status(202).json(build);
-         } else if (action === "stop") {
+         if (action === "stop") {
             const build = await materializationController.stopMaterialization(
                req.params.environmentName,
                req.params.packageName,
@@ -1607,7 +1601,7 @@ app.post(
             res.status(200).json(build);
          } else {
             throw new BadRequestError(
-               `Unsupported action '${String(action ?? "")}'. Expected 'build' or 'stop'.`,
+               `Unsupported action '${String(action ?? "")}'. Expected 'stop'.`,
             );
          }
       } catch (error) {
@@ -1765,9 +1759,21 @@ mcpServer.timeout = 600000;
 mcpServer.keepAliveTimeout = 600000;
 mcpServer.headersTimeout = 600000;
 
+// Separate, isolated MCP server for the agent retrieval tools (get_context,
+// search_docs) on its own listener. Kept apart from the core MCP server above.
+const agentMcpServer = startAgentMcpServer(
+   environmentStore,
+   PUBLISHER_HOST,
+   AGENT_MCP_PORT,
+);
+agentMcpServer.timeout = 600000;
+agentMcpServer.keepAliveTimeout = 600000;
+agentMcpServer.headersTimeout = 600000;
+
 registerSignalHandlers(
    mainServer,
    mcpServer,
    SHUTDOWN_DRAIN_DURATION_SECONDS,
    SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS,
+   agentMcpServer,
 );

package/src/service/build_plan.spec.ts

@@ -0,0 +1,116 @@
+import type { PersistSource } from "@malloydata/malloy";
+import { describe, expect, it } from "bun:test";
+import * as sinon from "sinon";
+import {
+   computeBuildId,
+   computePackageBuildPlan,
+   deriveBuildPlan,
+   flattenDependsOn,
+   resolvePackageConnections,
+} from "./build_plan";
+import { fakeSource } from "./materialization_test_fixtures";
+
+describe("flattenDependsOn", () => {
+   it("maps nested dependsOn entries to a flat sourceID list", () => {
+      expect(
+         flattenDependsOn({
+            dependsOn: [{ sourceID: "a" }, { sourceID: "b" }],
+         }),
+      ).toEqual(["a", "b"]);
+   });
+});
+
+describe("computeBuildId", () => {
+   it("delegates to PersistSource.makeBuildId with the connection digest and SQL", () => {
+      const makeBuildId = sinon.stub().returns("computed-id");
+      const source = {
+         connectionName: "duckdb",
+         makeBuildId,
+         getSQL: () => "SELECT 7",
+      } as unknown as PersistSource;
+
+      const id = computeBuildId(source, { duckdb: "dig-1" });
+
+      expect(id).toBe("computed-id");
+      expect(makeBuildId.calledOnceWithExactly("dig-1", "SELECT 7")).toBe(true);
+   });
+});
+
+describe("resolvePackageConnections", () => {
+   it("resolves each unique name once and omits failures", async () => {
+      const getMalloyConnection = sinon.stub();
+      getMalloyConnection.withArgs("ok").resolves({ id: "ok-conn" });
+      getMalloyConnection.withArgs("bad").rejects(new Error("nope"));
+
+      const map = await resolvePackageConnections({ getMalloyConnection }, [
+         "ok",
+         "ok",
+         "bad",
+      ]);
+
+      expect(map.has("ok")).toBe(true);
+      expect(map.has("bad")).toBe(false);
+      // "ok" requested twice but resolved once (dedupe).
+      expect(getMalloyConnection.withArgs("ok").callCount).toBe(1);
+   });
+});
+
+describe("deriveBuildPlan", () => {
+   it("projects graphs and sources into the wire build plan", () => {
+      const orders = fakeSource({
+         name: "orders",
+         buildId: "bid-orders",
+         sql: "SELECT 1",
+      });
+      const plan = deriveBuildPlan(
+         [
+            {
+               connectionName: "duckdb",
+               nodes: [[{ sourceID: "orders@m", dependsOn: [] }]],
+            },
+         ] as unknown as Parameters<typeof deriveBuildPlan>[0],
+         { "orders@m": orders },
+         { duckdb: "dig" },
+      );
+
+      expect(plan.graphs[0].connectionName).toBe("duckdb");
+      expect(plan.sources["orders@m"]).toMatchObject({
+         name: "orders",
+         connectionName: "duckdb",
+         buildId: "bid-orders",
+         sql: "SELECT 1",
+         columns: [],
+      });
+   });
+
+   it("honors the sourceNames filter", () => {
+      const a = fakeSource({ name: "a", buildId: "bid-a" });
+      const b = fakeSource({ name: "b", buildId: "bid-b" });
+      const plan = deriveBuildPlan(
+         [
+            {
+               connectionName: "duckdb",
+               nodes: [[{ sourceID: "a@m", dependsOn: [] }]],
+            },
+         ] as unknown as Parameters<typeof deriveBuildPlan>[0],
+         { "a@m": a, "b@m": b },
+         { duckdb: "dig" },
+         ["a"],
+      );
+
+      expect(Object.keys(plan.sources)).toEqual(["a@m"]);
+   });
+});
+
+describe("computePackageBuildPlan", () => {
+   it("returns null when the package declares no persist sources", async () => {
+      const pkg = {
+         getModelPaths: () => [],
+         getPackagePath: () => "/test",
+         getMalloyConfig: () => ({}),
+         getMalloyConnection: async () => ({}),
+      } as unknown as Parameters<typeof computePackageBuildPlan>[0];
+
+      expect(await computePackageBuildPlan(pkg)).toBeNull();
+   });
+});

package/src/service/build_plan.ts

@@ -0,0 +1,238 @@
+import type {
+   AtomicField,
+   BuildGraph as MalloyBuildGraph,
+   MalloyConfig,
+   Connection as MalloyConnection,
+   PersistSource,
+} from "@malloydata/malloy";
+import { components } from "../api";
+import { logger } from "../logger";
+import { recordConnectionDigestSkipped } from "../materialization_metrics";
+import { Model } from "./model";
+
+type WireBuildGraph = components["schemas"]["BuildGraph"];
+type WirePersistSourcePlan = components["schemas"]["PersistSourcePlan"];
+type WireColumn = components["schemas"]["Column"];
+type BuildPlan = components["schemas"]["BuildPlan"];
+
+/**
+ * Minimal surface a package must expose to compile its build plan. Both
+ * {@link Package} (for the read-only `Package.buildPlan`) and the
+ * materialization service (for auto-run and orchestrated builds) satisfy it.
+ */
+export interface BuildPlanPackage {
+   getModelPaths(): string[];
+   getPackagePath(): string;
+   getMalloyConfig(): MalloyConfig;
+   getMalloyConnection(name: string): Promise<MalloyConnection>;
+}
+
+/**
+ * Result of compiling a package's persist sources: the dependency-ordered
+ * build graphs, the persist sources keyed by sourceID, the per-connection
+ * digests, and the resolved live connections. The wire {@link BuildPlan} is a
+ * projection of this (see {@link deriveBuildPlan}); the build engine consumes
+ * the full structure.
+ */
+export interface CompiledBuildPlan {
+   graphs: MalloyBuildGraph[];
+   sources: Record<string, PersistSource>;
+   connectionDigests: Record<string, string>;
+   connections: Map<string, MalloyConnection>;
+}
+
+/** Output columns of a persist source, degrading to [] if unavailable. */
+export function deriveColumns(persistSource: PersistSource): WireColumn[] {
+   try {
+      return persistSource._explore.intrinsicFields
+         .filter((f) => f.isAtomicField())
+         .map((f) => ({
+            name: f.name,
+            type: String((f as AtomicField).type),
+         }));
+   } catch (err) {
+      logger.warn("Failed to derive columns for persist source", {
+         sourceID: persistSource.sourceID,
+         error: err instanceof Error ? err.message : String(err),
+      });
+      return [];
+   }
+}
+
+/** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
+export function flattenDependsOn(node: {
+   dependsOn: { sourceID: string }[];
+}): string[] {
+   return node.dependsOn.map((d) => d.sourceID);
+}
+
+/**
+ * The buildId for a persist source: a stable digest of its connection identity
+ * and canonical SQL. Centralizes the (source, connectionDigests) call shape so
+ * planning, self-instruction, and build all agree on the same id.
+ */
+export function computeBuildId(
+   source: PersistSource,
+   connectionDigests: Record<string, string>,
+): string {
+   return source.makeBuildId(
+      connectionDigests[source.connectionName],
+      source.getSQL(),
+   );
+}
+
+/**
+ * Resolve a Map<name, Connection> for the names a step is about to touch.
+ * The package's MalloyConfig caches each lookup, so repeated calls are cheap.
+ * A failed lookup is logged and omitted; downstream code reports the missing
+ * connection explicitly.
+ */
+export async function resolvePackageConnections(
+   pkg: { getMalloyConnection(name: string): Promise<MalloyConnection> },
+   names: Iterable<string>,
+): Promise<Map<string, MalloyConnection>> {
+   const map = new Map<string, MalloyConnection>();
+   const seen = new Set<string>();
+   for (const name of names) {
+      if (!name || seen.has(name)) continue;
+      seen.add(name);
+      try {
+         map.set(name, await pkg.getMalloyConnection(name));
+      } catch (err) {
+         logger.warn(`Failed to resolve connection ${name}`, {
+            error: err instanceof Error ? err.message : String(err),
+         });
+      }
+   }
+   return map;
+}
+
+/**
+ * Compile every model in the package and collect the dependency-ordered
+ * build graphs, persist sources, connection digests, and resolved
+ * connections. The build plan is a pure function of the compiled model plus
+ * connection config (no warehouse access).
+ */
+export async function compilePackageBuildPlan(
+   pkg: BuildPlanPackage,
+   signal?: AbortSignal,
+): Promise<CompiledBuildPlan> {
+   const allGraphs: MalloyBuildGraph[] = [];
+   const allSources: Record<string, PersistSource> = {};
+
+   for (const modelPath of pkg.getModelPaths()) {
+      if (signal?.aborted) throw new Error("Build cancelled");
+
+      const { runtime, modelURL, importBaseURL } = await Model.getModelRuntime(
+         pkg.getPackagePath(),
+         modelPath,
+         pkg.getMalloyConfig(),
+      );
+      const malloyModel = await runtime
+         .loadModel(modelURL, { importBaseURL })
+         .getModel();
+
+      // getBuildPlan() returns empty graphs for models with no #@ persist
+      // sources, so non-persist models are simply skipped below.
+      const buildPlan = malloyModel.getBuildPlan();
+      for (const msg of buildPlan.tagParseLog) {
+         logger.warn("Persist annotation issue", {
+            modelPath,
+            message: msg.message,
+            severity: msg.severity,
+         });
+      }
+      if (buildPlan.graphs.length === 0) continue;
+
+      allGraphs.push(...buildPlan.graphs);
+      for (const [sourceID, source] of Object.entries(buildPlan.sources)) {
+         allSources[sourceID] = source;
+      }
+   }
+
+   const connections = await resolvePackageConnections(
+      pkg,
+      allGraphs.map((g) => g.connectionName),
+   );
+   const connectionDigests: Record<string, string> = {};
+   for (const graph of allGraphs) {
+      const conn = connections.get(graph.connectionName);
+      if (!conn) {
+         // The connection failed to resolve (already warned in
+         // resolvePackageConnections). Its buildIds will be computed without a
+         // digest, so surface it as a discrete correctness signal rather than
+         // skipping silently.
+         recordConnectionDigestSkipped();
+         logger.warn("Skipping connection digest; connection did not resolve", {
+            connectionName: graph.connectionName,
+         });
+         continue;
+      }
+      if (!connectionDigests[graph.connectionName]) {
+         connectionDigests[graph.connectionName] = await conn.getDigest();
+      }
+   }
+
+   return {
+      graphs: allGraphs,
+      sources: allSources,
+      connectionDigests,
+      connections,
+   };
+}
+
+/** Project the Malloy build plan into the trimmed wire BuildPlan. */
+export function deriveBuildPlan(
+   graphs: MalloyBuildGraph[],
+   sources: Record<string, PersistSource>,
+   connectionDigests: Record<string, string>,
+   sourceNames?: string[],
+): BuildPlan {
+   const include = sourceNames ? new Set(sourceNames) : null;
+
+   const wireGraphs: WireBuildGraph[] = graphs.map((graph) => ({
+      connectionName: graph.connectionName,
+      nodes: graph.nodes.map((level) =>
+         level.map((node) => ({
+            sourceID: node.sourceID,
+            dependsOn: flattenDependsOn(node),
+         })),
+      ),
+   }));
+
+   const wireSources: Record<string, WirePersistSourcePlan> = {};
+   for (const [sourceID, source] of Object.entries(sources)) {
+      if (include && !include.has(source.name)) continue;
+      wireSources[sourceID] = {
+         name: source.name,
+         sourceID: source.sourceID,
+         connectionName: source.connectionName,
+         dialect: source.dialectName,
+         buildId: computeBuildId(source, connectionDigests),
+         sql: source.getSQL(),
+         columns: deriveColumns(source),
+      };
+   }
+
+   return { graphs: wireGraphs, sources: wireSources };
+}
+
+/**
+ * Compile and project a package's build plan, or null when the package
+ * declares no persist source. Convenience for the read-only `Package.buildPlan`
+ * field, which is a deterministic property of the compiled package.
+ */
+export async function computePackageBuildPlan(
+   pkg: BuildPlanPackage,
+   signal?: AbortSignal,
+): Promise<BuildPlan | null> {
+   const compiled = await compilePackageBuildPlan(pkg, signal);
+   if (compiled.graphs.length === 0) {
+      return null;
+   }
+   return deriveBuildPlan(
+      compiled.graphs,
+      compiled.sources,
+      compiled.connectionDigests,
+   );
+}

package/src/service/connection.ts

@@ -7,6 +7,10 @@ import "@malloydata/db-mysql";
 import type { MySQLConnection } from "@malloydata/db-mysql";
 import "@malloydata/db-postgres";
 import type { PostgresConnection } from "@malloydata/db-postgres";
+// Registers the "publisher" connection type (proxies SQL to a remote Publisher
+// dataplane). No live-class branch is needed in lookupConnection — the default
+// path resolves it through the registry like any other registered type.
+import "@malloydata/db-publisher";
 import {
    buildPoolOptions,
    SnowflakeConnection,

package/src/service/connection_config.spec.ts

@@ -1,4 +1,4 @@
-import { describe, expect, it } from "bun:test";
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
 import { generateKeyPairSync } from "crypto";
 import { components } from "../api";
 import {
@@ -168,3 +168,184 @@ describe("normalizeSnowflakePrivateKey", () => {
       expect(result.endsWith("-----END PRIVATE KEY-----\n")).toBe(true);
    });
 });
+
+describe("assembleEnvironmentConnections — publisher", () => {
+   const validBase: ApiConnection = {
+      name: "analytics",
+      type: "publisher",
+      publisherConnection: {
+         connectionUri:
+            "https://org.data.example.com/api/v0/environments/proj/connections/analytics",
+         accessToken: "jwt-token",
+      },
+   };
+
+   // publisher connections are default-deny (SSRF gate); these assembly tests
+   // exercise the enabled path, so opt in for the block and restore after.
+   const priorFlag = process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS;
+   beforeEach(() => {
+      process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS = "true";
+   });
+   afterEach(() => {
+      if (priorFlag === undefined) {
+         delete process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS;
+      } else {
+         process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS = priorFlag;
+      }
+   });
+
+   it("emits a publisher core entry proxying to the remote dataplane", () => {
+      const { pojo } = assembleEnvironmentConnections([validBase]);
+
+      const entry = pojo.connections["analytics"];
+      expect(entry.is).toBe("publisher");
+      expect(entry.connectionUri).toBe(
+         "https://org.data.example.com/api/v0/environments/proj/connections/analytics",
+      );
+      expect(entry.accessToken).toBe("jwt-token");
+   });
+
+   it("does not populate static connection attributes (dialect is resolved at runtime)", () => {
+      const { apiConnections } = assembleEnvironmentConnections([validBase]);
+      expect(apiConnections).toHaveLength(1);
+      expect(apiConnections[0].attributes).toBeUndefined();
+   });
+
+   it("assembles without an accessToken (optional)", () => {
+      const conn: ApiConnection = {
+         name: "analytics",
+         type: "publisher",
+         publisherConnection: {
+            connectionUri:
+               "https://org.data.example.com/api/v0/environments/proj/connections/analytics",
+         },
+      };
+      const { pojo } = assembleEnvironmentConnections([conn]);
+      const entry = pojo.connections["analytics"];
+      expect(entry.is).toBe("publisher");
+      expect(entry.accessToken).toBeUndefined();
+   });
+
+   it("rejects a publisher connection missing connectionUri with an actionable error", () => {
+      const conn: ApiConnection = {
+         name: "analytics",
+         type: "publisher",
+         publisherConnection:
+            {} as components["schemas"]["PublisherConnection"],
+      };
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "Invalid publisher connection 'analytics': missing connectionUri.",
+      );
+   });
+
+   it("rejects a publisher connection missing the publisherConnection block", () => {
+      const conn: ApiConnection = {
+         name: "analytics",
+         type: "publisher",
+      };
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "Invalid publisher connection 'analytics': missing connectionUri.",
+      );
+   });
+
+   it("rejects a publisher connection whose connectionUri is not a valid URL", () => {
+      const conn: ApiConnection = {
+         name: "analytics",
+         type: "publisher",
+         publisherConnection: { connectionUri: "not a url" },
+      };
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "Invalid publisher connection 'analytics': connectionUri is not a valid URL.",
+      );
+   });
+
+   it("rejects a publisher connection whose connectionUri uses a non-http(s) scheme", () => {
+      const conn: ApiConnection = {
+         name: "analytics",
+         type: "publisher",
+         publisherConnection: { connectionUri: "file:///etc/passwd" },
+      };
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "connectionUri must use http or https (got 'file:')",
+      );
+   });
+
+   it("does not echo a credential-bearing connectionUri in the validation error", () => {
+      const conn: ApiConnection = {
+         name: "analytics",
+         type: "publisher",
+         // Userinfo present but the URI is otherwise malformed (space in host)
+         // so it fails to parse and must not be reflected back verbatim.
+         publisherConnection: {
+            connectionUri: "https://user:s3cret@bad host/connections/x",
+         },
+      };
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "connectionUri is not a valid URL",
+      );
+      expect(() => assembleEnvironmentConnections([conn])).not.toThrow(
+         /s3cret/,
+      );
+   });
+
+   it("still rejects the reserved 'duckdb' name for a publisher connection", () => {
+      const conn: ApiConnection = {
+         name: "duckdb",
+         type: "publisher",
+         publisherConnection: { connectionUri: "https://x/connections/duckdb" },
+      };
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "Connection name 'duckdb' is reserved",
+      );
+   });
+
+   it("still rejects a publisher connection with no name", () => {
+      const conn = {
+         type: "publisher",
+         publisherConnection: { connectionUri: "https://x/connections/y" },
+      } as ApiConnection;
+      expect(() => assembleEnvironmentConnections([conn])).toThrow(
+         "Invalid connection configuration. No name.",
+      );
+   });
+
+   describe("SSRF gate (PUBLISHER_ALLOW_PROXY_CONNECTIONS)", () => {
+      it("denies a valid publisher connection when the flag is unset (default-deny)", () => {
+         delete process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS;
+         expect(() => assembleEnvironmentConnections([validBase])).toThrow(
+            "Publisher proxy connection 'analytics' is disabled in this deployment",
+         );
+      });
+
+      it("error names the env var to flip", () => {
+         delete process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS;
+         expect(() => assembleEnvironmentConnections([validBase])).toThrow(
+            "Fix: set the environment variable PUBLISHER_ALLOW_PROXY_CONNECTIONS=true",
+         );
+      });
+
+      it("denies for any non-'true' value (fail-closed)", () => {
+         process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS = "1";
+         expect(() => assembleEnvironmentConnections([validBase])).toThrow(
+            "is disabled in this deployment",
+         );
+      });
+
+      it("gate fires before the connectionUri shape check", () => {
+         // A publisher connection missing connectionUri still surfaces the
+         // disabled error first when the gate is closed — the type is refused
+         // outright, not validated.
+         delete process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS;
+         const conn: ApiConnection = { name: "analytics", type: "publisher" };
+         expect(() => assembleEnvironmentConnections([conn])).toThrow(
+            "is disabled in this deployment",
+         );
+      });
+
+      it("allows a valid publisher connection when the flag is 'true'", () => {
+         process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS = "true";
+         const { pojo } = assembleEnvironmentConnections([validBase]);
+         expect(pojo.connections["analytics"].is).toBe("publisher");
+      });
+   });
+});