@malloy-publisher/server 0.0.206 → 0.0.208

package/src/service/package.ts

@@ -48,8 +48,6 @@ type PackageConnectionInput =
    | Map<string, Connection>
    | (() => MalloyConfig);
 
-const ENABLE_LIST_MODEL_COMPILATION = true;
-
 export class Package {
    private environmentName: string;
    private packageName: string;
@@ -58,6 +56,17 @@ export class Package {
    private models: Map<string, Model> = new Map();
    private packagePath: string;
    private malloyConfig: MalloyConfig;
+   // Build-manifest binding state (Malloy Persistence v0). When bound, these
+   // entries (buildId -> { tableName }) are what served queries use to route
+   // persist sources to their materialized physical tables; they are also reused
+   // by the /compile preview so previewed SQL matches executed SQL. Surfaced on
+   // /status (via getPackageMetadata) so the control plane can confirm a worker
+   // actually bound the distributed manifest instead of inferring it from logs.
+   private buildManifestEntries: BuildManifest["entries"] | undefined;
+   private manifestBindingStatus: "unbound" | "bound" | "live_fallback" =
+      "unbound";
+   private manifestEntryCount = 0;
+   private boundManifestUri: string | null = null;
    private static meter = metrics.getMeter("publisher");
    private static packageLoadHistogram = this.meter.createHistogram(
       "malloy_package_load_duration",
@@ -389,14 +398,76 @@ export class Package {
    }
 
    public getPackageMetadata(): ApiPackage {
-      // Surface explores misconfig so consumers/UI can show it (loading is
-      // fail-safe — the package still serves with the bad entry hidden — so this
-      // is the only non-log signal that it's broken). Computed fresh against the
-      // current models; absent when everything resolves. Returns a copy in that
-      // case so the added field never mutates the stored metadata.
+      // Overlay the server-computed fields onto the stored metadata: the
+      // explores misconfig warnings (loading is fail-safe — the package still
+      // serves with the bad entry hidden — so this is the only non-log signal
+      // that it's broken) and the manifest-binding state (so /status reflects
+      // whether persist sources are actually routed to materialized tables).
+      // Always returns a copy so these overlays never mutate the stored
+      // metadata; the binding fields are authoritative from the private state.
+      //
+      // `name` is the registered package name (the environment's identity for
+      // this package), not the value read from the package's own manifest —
+      // those can differ (e.g. a package installed under a different name than
+      // its publisher.json declares). `listPackages` already overrides it to
+      // the registered name; surfacing it here keeps the single-package GET
+      // consistent without relying on a returned-reference mutation.
+      const metadata: ApiPackage = {
+         ...this.packageMetadata,
+         name: this.packageName,
+         manifestBindingStatus: this.manifestBindingStatus,
+         manifestEntryCount: this.manifestEntryCount,
+         boundManifestUri: this.boundManifestUri,
+      };
       const warnings = this.exploreWarnings();
-      if (warnings.length === 0) return this.packageMetadata;
-      return { ...this.packageMetadata, exploresWarnings: warnings };
+      if (warnings.length > 0) {
+         metadata.exploresWarnings = warnings;
+      }
+      return metadata;
+   }
+
+   /**
+    * The currently-bound build-manifest entries (buildId -> { tableName }), or
+    * undefined when the package is serving live. Reused by the /compile preview
+    * so previewed SQL gets the same persist-source -> physical-table routing as
+    * execution.
+    */
+   public getBuildManifestEntries(): BuildManifest["entries"] | undefined {
+      return this.buildManifestEntries;
+   }
+
+   /**
+    * Record the URI whose manifest is currently bound to the served models. May
+    * differ from `manifestLocation` after an in-memory auto-load following a
+    * materialization build (no URI), in which case it stays null.
+    */
+   public setBoundManifestUri(uri: string | null): void {
+      this.boundManifestUri = uri;
+   }
+
+   /**
+    * Mark that a configured `manifestLocation` could not be fetched/bound, so
+    * the package is serving live despite intending to be materialized-routed.
+    */
+   public markManifestBindFailed(): void {
+      this.manifestBindingStatus = "live_fallback";
+   }
+
+   /**
+    * Store the entries applied by the latest (re)bind for reuse (/compile) and
+    * observability (/status). An empty map means the manifest was cleared, so
+    * the package reverts to live (unbound).
+    */
+   private recordManifestBinding(
+      buildManifest: BuildManifest["entries"],
+   ): void {
+      const count = Object.keys(buildManifest).length;
+      this.buildManifestEntries = count > 0 ? buildManifest : undefined;
+      this.manifestEntryCount = count;
+      this.manifestBindingStatus = count > 0 ? "bound" : "unbound";
+      if (count === 0) {
+         this.boundManifestUri = null;
+      }
    }
 
    /**
@@ -595,6 +666,7 @@ export class Package {
                this.packagePath,
                this.malloyConfig,
                sm,
+               { buildManifest },
             );
             // Validate renderer tags here too (loadViaWorker does it for the
             // create path). Reload keeps per-model placeholders rather than
@@ -636,6 +708,9 @@ export class Package {
          outcome.packageMetadata.manifestLocation ?? null;
       this.applyDiscoveryPolicyToModels();
       this.applyQueryBoundaryToModels();
+      // Remember what we just bound so /compile can route identically and
+      // /status can report the binding. An empty map reverts to live (unbound).
+      this.recordManifestBinding(buildManifest);
       // Re-run the fail-safe warning against the refreshed model set: an edit
       // to publisher.json that introduces a bad entry should surface in the
       // logs on reload too, not only at initial load (loadViaWorker).
@@ -672,15 +747,13 @@ export class Package {
             })
             .map(async (modelPath) => {
                let error: string | undefined;
-               if (ENABLE_LIST_MODEL_COMPILATION) {
-                  try {
-                     await this.models.get(modelPath)?.getModel();
-                  } catch (modelError) {
-                     error =
-                        modelError instanceof Error
-                           ? modelError.message
-                           : undefined;
-                  }
+               try {
+                  await this.models.get(modelPath)?.getModel();
+               } catch (modelError) {
+                  error =
+                     modelError instanceof Error
+                        ? modelError.message
+                        : undefined;
                }
                return {
                   environmentName: this.environmentName,
@@ -700,10 +773,7 @@ export class Package {
                return modelPath.endsWith(NOTEBOOK_FILE_SUFFIX);
             })
             .map(async (modelPath) => {
-               let error: Error | undefined;
-               if (ENABLE_LIST_MODEL_COMPILATION) {
-                  error = this.models.get(modelPath)?.getNotebookError();
-               }
+               const error = this.models.get(modelPath)?.getNotebookError();
                return {
                   environmentName: this.environmentName,
                   packageName: this.packageName,

package/tests/integration/materialization/manifest_binding.integration.spec.ts

@@ -133,25 +133,38 @@ describe("Manifest binding via Package.manifestLocation (E2E)", () => {
       async () => {
          const manifestFile = await writeManifest();
 
-         // Accept on update; the response echoes the bound location.
+         // Accept on update; the response echoes the bound location and the
+         // server-computed binding observability fields (surfaced on /status so
+         // the control plane can confirm the worker actually bound the manifest).
          const patchRes = await patchPackage({
             manifestLocation: manifestFile,
          });
          expect(patchRes.status).toBe(200);
-         expect((await patchRes.json()).manifestLocation).toBe(manifestFile);
+         const patched = await patchRes.json();
+         expect(patched.manifestLocation).toBe(manifestFile);
+         expect(patched.manifestBindingStatus).toBe("bound");
+         expect(patched.manifestEntryCount).toBe(1);
+         expect(patched.boundManifestUri).toBe(manifestFile);
 
          // In-memory metadata reflects it; binding did not break serving.
-         expect((await getPackage()).manifestLocation).toBe(manifestFile);
+         const bound = await getPackage();
+         expect(bound.manifestLocation).toBe(manifestFile);
+         expect(bound.manifestBindingStatus).toBe("bound");
+         expect(bound.manifestEntryCount).toBe(1);
          expect(await queryOrderSummaryStatus()).toBe(200);
 
          // Persisted to publisher.json: a reload re-reads and re-binds it.
          expect((await getPackage(true)).manifestLocation).toBe(manifestFile);
          expect(await queryOrderSummaryStatus()).toBe(200);
 
-         // Clearing reverts to live and survives a reload.
+         // Clearing reverts to live (unbound) and survives a reload.
          const clearRes = await patchPackage({ manifestLocation: null });
          expect(clearRes.status).toBe(200);
-         expect((await clearRes.json()).manifestLocation ?? null).toBeNull();
+         const cleared = await clearRes.json();
+         expect(cleared.manifestLocation ?? null).toBeNull();
+         expect(cleared.manifestBindingStatus).toBe("unbound");
+         expect(cleared.manifestEntryCount).toBe(0);
+         expect(cleared.boundManifestUri ?? null).toBeNull();
          expect((await getPackage(true)).manifestLocation ?? null).toBeNull();
          expect(await queryOrderSummaryStatus()).toBe(200);
       },
@@ -163,7 +176,13 @@ describe("Manifest binding via Package.manifestLocation (E2E)", () => {
 
       const patchRes = await patchPackage({ manifestLocation: missing });
       expect(patchRes.status).toBe(200);
-      expect((await patchRes.json()).manifestLocation).toBe(missing);
+      const patched = await patchRes.json();
+      expect(patched.manifestLocation).toBe(missing);
+      // The configured location is retained, but the binding is reported as a
+      // degraded live fallback (not "bound") with nothing actually bound.
+      expect(patched.manifestBindingStatus).toBe("live_fallback");
+      expect(patched.manifestEntryCount).toBe(0);
+      expect(patched.boundManifestUri ?? null).toBeNull();
 
       // A fetch failure must not brick the package — it still serves live.
       expect(await queryOrderSummaryStatus()).toBe(200);

package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts

@@ -342,6 +342,100 @@ describe("Materialization REST API: auto-run + two-round (E2E)", () => {
       );
    });
 
+   // ── Group A1: Serve-time routing (the v0 payoff) ─────────────────
+   //
+   // Materialization only pays off if *served* queries scan the materialized
+   // table instead of recomputing from the base table. The auto-run path builds
+   // the table and auto-loads the manifest into the serving models in one
+   // process, so routing is provable in-process: capture the live SQL (scans the
+   // base CSV), run an auto-run materialization, then assert both the executed
+   // query SQL and the /compile preview SQL now scan the physical table and no
+   // longer touch the base CSV. The /compile half guards the parity fix
+   // (Environment.compileSource threads the package's bound manifest).
+   describe("serve-time routing (auto-load)", () => {
+      const MODEL_PATH = "persist_test.malloy";
+      const QUERY = "run: order_summary -> { aggregate: c is count() }";
+
+      async function executedSql(): Promise<string> {
+         const res = await fetch(
+            `${baseUrl}${API}/models/${MODEL_PATH}/query`,
+            {
+               method: "POST",
+               headers: { "Content-Type": "application/json" },
+               body: JSON.stringify({ query: QUERY }),
+            },
+         );
+         expect(res.status).toBe(200);
+         const body = (await res.json()) as { result: string };
+         return (JSON.parse(body.result) as { sql: string }).sql;
+      }
+
+      async function compiledSql(): Promise<string> {
+         const res = await fetch(
+            `${baseUrl}${API}/models/${MODEL_PATH}/compile`,
+            {
+               method: "POST",
+               headers: { "Content-Type": "application/json" },
+               body: JSON.stringify({ source: QUERY, includeSql: true }),
+            },
+         );
+         expect(res.status).toBe(200);
+         const body = (await res.json()) as { status: string; sql?: string };
+         expect(body.status).toBe("success");
+         expect(body.sql).toBeDefined();
+         return body.sql as string;
+      }
+
+      it(
+         "routes served + compiled queries to the materialized table after auto-load",
+         async () => {
+            // Reset to live: a prior group may have left an in-memory binding.
+            // Auto-load never writes manifestLocation to publisher.json, so a
+            // reload re-reads the live model.
+            await fetch(`${baseUrl}${API}?reload=true`);
+
+            // Baseline: with nothing materialized, both paths recompute from the
+            // base CSV.
+            expect(await executedSql()).toContain("data/orders.csv");
+            expect(await compiledSql()).toContain("data/orders.csv");
+
+            // Build + auto-load in one pass (self-assigns physicalTableName =
+            // "order_summary" from `#@ persist name="order_summary"`).
+            const createRes = await createMaterialization({
+               pauseBetweenPhases: false,
+            });
+            expect(createRes.status).toBe(201);
+            const { id } = (await createRes.json()) as { id: string };
+            const built = await pollUntil(
+               id,
+               (s) => TERMINAL_STATUSES.includes(s),
+               90_000,
+            );
+            expect(built.status).toBe("MANIFEST_FILE_READY");
+
+            // The payoff: the served query now scans the materialized table and
+            // no longer recomputes from the base CSV, and /compile previews the
+            // same routed SQL it would execute.
+            const routedExecuted = await executedSql();
+            const routedCompiled = await compiledSql();
+            expect(routedExecuted).not.toContain("data/orders.csv");
+            expect(routedExecuted).toContain("order_summary");
+            expect(routedCompiled).not.toContain("data/orders.csv");
+            expect(routedCompiled).toContain("order_summary");
+
+            // Cleanup: drop the table + record, then reload back to live so the
+            // dangling binding doesn't leak into later groups.
+            const deleteRes = await fetch(
+               url(`/materializations/${id}?dropTables=true`),
+               { method: "DELETE" },
+            );
+            expect(deleteRes.status).toBe(204);
+            await fetch(`${baseUrl}${API}?reload=true`);
+         },
+         { timeout: 120_000 },
+      );
+   });
+
    // ── Group B: State machine and error cases ───────────────────────
 
    describe("state machine and errors", () => {

package/tests/integration/watch-mode/watch_mode.integration.spec.ts

@@ -116,6 +116,11 @@ async function startServer(opts: { watch: boolean }): Promise<TestServer> {
    const port = await getFreePort();
    let mcpPort = await getFreePort();
    while (mcpPort === port) mcpPort = await getFreePort();
+   // The agent MCP server binds its own listener; give it a distinct free port
+   // too, otherwise every spawned child binds the fixed default and collides.
+   let agentMcpPort = await getFreePort();
+   while (agentMcpPort === port || agentMcpPort === mcpPort)
+      agentMcpPort = await getFreePort();
    const baseUrl = `http://127.0.0.1:${port}`;
 
    const env: NodeJS.ProcessEnv = {
@@ -124,6 +129,7 @@ async function startServer(opts: { watch: boolean }): Promise<TestServer> {
       PUBLISHER_HOST: "127.0.0.1",
       PUBLISHER_PORT: String(port),
       MCP_PORT: String(mcpPort),
+      AGENT_MCP_PORT: String(agentMcpPort),
    };
    if (opts.watch) env.PUBLISHER_WATCH = ENV_NAME;