@malloy-publisher/server 0.0.225 → 0.0.227

package/src/service/persistence_policy.spec.ts

@@ -0,0 +1,234 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import * as fs from "fs/promises";
+import * as os from "os";
+import * as path from "path";
+
+import { Environment } from "./environment";
+import type { Package } from "./package";
+
+/**
+ * The publish gate for the package's Malloy Persistence policy (persistence.md
+ * §3.1, §9.2–§9.5). Scope is a single package-level mode (`Package.scope`) and a
+ * materialization cron is package-root-only + version-scope-only:
+ *
+ *  - per-source `#@ persist ... sharing=`/`schedule=` are retired (declaring
+ *    either on a source is a manifest error);
+ *  - `materialization.schedule` is legal only when `scope: version`;
+ *  - `materialization.schedule` and freshness (package or source) are mutually
+ *    exclusive.
+ *
+ * These tests run a real `Environment` + `Package.create` over temp dirs, so
+ * they also prove end-to-end that the pinned compiler still ACCEPTS the
+ * `#@ persist` annotation keys (`sharing`/`schedule`/`freshness.*`) at compile
+ * — the rejection is the publisher's manifest-level gate, not a compile error —
+ * and that `scope` is parsed from the manifest root and surfaced on the package.
+ */
+describe("persistence policy gate", () => {
+   let rootDir: string;
+   let envPath: string;
+
+   async function loadPackage(
+      model: string,
+      manifest: Record<string, unknown> = {},
+   ): Promise<Package> {
+      const dir = path.join(envPath, "pkg");
+      await fs.mkdir(dir, { recursive: true });
+      await fs.writeFile(
+         path.join(dir, "publisher.json"),
+         JSON.stringify({ name: "pkg", description: "fixture", ...manifest }),
+      );
+      await fs.writeFile(path.join(dir, "model.malloy"), model);
+      const env = await Environment.create("testEnv", envPath, []);
+      await env.addPackage("pkg");
+      return env.getPackage("pkg", false);
+   }
+
+   function sourceByName(pkg: Package, name: string) {
+      const sources = pkg.getBuildPlan()?.sources ?? {};
+      const found = Object.values(sources).find((s) => s.name === name);
+      if (!found) throw new Error(`persist source '${name}' not in build plan`);
+      return found;
+   }
+
+   beforeEach(async () => {
+      rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-policy-"));
+      envPath = path.join(rootDir, "env");
+      await fs.mkdir(envPath, { recursive: true });
+   });
+
+   afterEach(async () => {
+      await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
+   });
+
+   const PLAIN_MODEL = `##! experimental.persistence
+
+#@ persist name="a_table"
+source: a is duckdb.sql("SELECT 1 as x")
+
+#@ persist name="b_table" refresh=full
+source: b is duckdb.sql("SELECT 2 as x")
+`;
+
+   const SHARING_SOURCE_MODEL = `##! experimental.persistence
+
+#@ persist name="s_table" sharing=private
+source: s is duckdb.sql("SELECT 1 as x")
+`;
+
+   const SCHEDULE_SOURCE_MODEL = `##! experimental.persistence
+
+#@ persist name="s_table" schedule="0 */6 * * *"
+source: s is duckdb.sql("SELECT 1 as x")
+`;
+
+   const FRESHNESS_SOURCE_MODEL = `##! experimental.persistence
+
+#@ persist name="f_table" freshness.window="1h"
+source: f is duckdb.sql("SELECT 1 as x")
+`;
+
+   // ── scope parsing + surfacing ─────────────────────────────────────────
+
+   it(
+      "defaults scope to 'package' when the manifest omits it",
+      async () => {
+         const pkg = await loadPackage(PLAIN_MODEL);
+         expect(pkg.getPackageMetadata().scope).toBe("package");
+      },
+      { timeout: 30000 },
+   );
+
+   it(
+      "surfaces an explicit scope: version from the manifest root",
+      async () => {
+         const pkg = await loadPackage(PLAIN_MODEL, { scope: "version" });
+         expect(pkg.getPackageMetadata().scope).toBe("version");
+      },
+      { timeout: 30000 },
+   );
+
+   it(
+      "fails the load on an invalid scope value",
+      async () => {
+         await expect(
+            loadPackage(PLAIN_MODEL, { scope: "shared" }),
+         ).rejects.toThrow(/scope/i);
+      },
+      { timeout: 30000 },
+   );
+
+   // ── Rule 1: per-source sharing/schedule retired ───────────────────────
+
+   it(
+      "rejects a per-source sharing= annotation, pointing at root scope",
+      async () => {
+         const pkg = await loadPackage(SHARING_SOURCE_MODEL, {
+            scope: "version",
+         });
+         const joined = pkg.formatInvalidPersistencePolicy();
+         expect(joined).toContain('"s"');
+         expect(joined).toContain("sharing");
+         expect(joined).toContain('"scope"');
+      },
+      { timeout: 30000 },
+   );
+
+   it(
+      "rejects a per-source schedule= annotation, pointing at the root schedule",
+      async () => {
+         const pkg = await loadPackage(SCHEDULE_SOURCE_MODEL, {
+            scope: "version",
+         });
+         const joined = pkg.formatInvalidPersistencePolicy();
+         expect(joined).toContain('"s"');
+         expect(joined).toContain("schedule");
+         expect(joined).toContain("materialization.schedule");
+         // The retired field is not emitted on the wire plan.
+         const source = sourceByName(pkg, "s") as Record<string, unknown>;
+         expect(source.schedule).toBeUndefined();
+         expect(source.sharing).toBeUndefined();
+      },
+      { timeout: 30000 },
+   );
+
+   // ── Rule 2: schedule requires scope: version ──────────────────────────
+
+   it(
+      "rejects a package schedule when scope is package (the default)",
+      async () => {
+         const pkg = await loadPackage(PLAIN_MODEL, {
+            materialization: { schedule: "0 6 * * *" },
+         });
+         const joined = pkg.formatInvalidPersistencePolicy();
+         expect(joined).toContain("materialization.schedule");
+         expect(joined).toContain('"scope": "version"');
+      },
+      { timeout: 30000 },
+   );
+
+   it(
+      "accepts a package schedule when scope is version and no freshness is set",
+      async () => {
+         const pkg = await loadPackage(PLAIN_MODEL, {
+            scope: "version",
+            materialization: { schedule: "0 6 * * *" },
+         });
+         expect(pkg.persistencePolicyWarnings()).toEqual([]);
+      },
+      { timeout: 30000 },
+   );
+
+   // ── Rule 3: schedule XOR freshness ────────────────────────────────────
+
+   it(
+      "rejects a schedule declared alongside package-level freshness",
+      async () => {
+         const pkg = await loadPackage(PLAIN_MODEL, {
+            scope: "version",
+            materialization: {
+               schedule: "0 6 * * *",
+               freshness: { window: "24h" },
+            },
+         });
+         const joined = pkg.formatInvalidPersistencePolicy();
+         expect(joined).toContain("mutually exclusive");
+      },
+      { timeout: 30000 },
+   );
+
+   it(
+      "rejects a schedule declared alongside a source-level freshness",
+      async () => {
+         const pkg = await loadPackage(FRESHNESS_SOURCE_MODEL, {
+            scope: "version",
+            materialization: { schedule: "0 6 * * *" },
+         });
+         const joined = pkg.formatInvalidPersistencePolicy();
+         expect(joined).toContain("mutually exclusive");
+      },
+      { timeout: 30000 },
+   );
+
+   // ── inert when nothing is declared ────────────────────────────────────
+
+   it(
+      "is inert for a plain package with no schedule and no forbidden knobs",
+      async () => {
+         const pkg = await loadPackage(PLAIN_MODEL, { scope: "package" });
+         expect(pkg.persistencePolicyWarnings()).toEqual([]);
+      },
+      { timeout: 30000 },
+   );
+
+   it(
+      "accepts source-level freshness without a schedule in either scope",
+      async () => {
+         const pkg = await loadPackage(FRESHNESS_SOURCE_MODEL, {
+            scope: "package",
+         });
+         expect(pkg.persistencePolicyWarnings()).toEqual([]);
+         expect(sourceByName(pkg, "f").freshness).toEqual({ window: "1h" });
+      },
+      { timeout: 30000 },
+   );
+});

package/src/test_helpers/metrics_harness.ts

@@ -62,6 +62,15 @@ export interface MetricsHarness {
     * fired for this gauge yet.
     */
    collectGauge(name: string): Promise<number | undefined>;
+   /**
+    * Force a collection cycle and return the aggregate `count`/`sum` across all
+    * data points for the named histogram (optionally scoped to one label set).
+    * `count` is 0 when the histogram has not been emitted yet.
+    */
+   collectHistogram(
+      name: string,
+      attributeFilter?: Record<string, string | number | boolean>,
+   ): Promise<{ count: number; sum: number; boundaries: number[] }>;
    shutdown(): Promise<void>;
 }
 
@@ -100,6 +109,37 @@ export async function startMetricsHarness(): Promise<MetricsHarness> {
          }
          return total;
       },
+      async collectHistogram(
+         name: string,
+         attributeFilter?: Record<string, string | number | boolean>,
+      ): Promise<{ count: number; sum: number; boundaries: number[] }> {
+         const result = await reader.collect();
+         let count = 0;
+         let sum = 0;
+         let boundaries: number[] = [];
+         for (const rm of result.resourceMetrics.scopeMetrics) {
+            for (const metric of rm.metrics) {
+               if (metric.descriptor.name !== name) continue;
+               for (const dp of metric.dataPoints) {
+                  if (attributeFilter) {
+                     const allMatch = Object.entries(attributeFilter).every(
+                        ([k, v]) => dp.attributes?.[k] === v,
+                     );
+                     if (!allMatch) continue;
+                  }
+                  const point = dp.value as {
+                     count: number;
+                     sum?: number;
+                     buckets?: { boundaries: number[] };
+                  };
+                  count += point.count;
+                  sum += point.sum ?? 0;
+                  if (point.buckets) boundaries = point.buckets.boundaries;
+               }
+            }
+         }
+         return { count, sum, boundaries };
+      },
       async collectGauge(name: string): Promise<number | undefined> {
          const result = await reader.collect();
          for (const rm of result.resourceMetrics.scopeMetrics) {

package/tests/harness/mcp_test_setup.ts

@@ -30,7 +30,7 @@ let portCounter = 0;
 
 // True mutex: the previous Promise-based lock had a TOCTOU race where two beforeAll hooks
 // could both pass the `if (initializationLock)` check and run setup concurrently, sharing
-// one publisher.db / server singleton and causing flaky listResources and port collisions.
+// one publisher.db / server singleton and causing flaky tool calls and port collisions.
 const e2eSetupMutex = new Mutex();
 
 /**

package/tests/integration/mcp/mcp_transport.integration.spec.ts

@@ -34,32 +34,20 @@ describe.serial("MCP Transport Tests (E2E Integration)", () => {
 
    describe("Basic Protocol", () => {
       it(
-         "should handle a simple listResources request",
+         "should handle a simple listTools request",
          async () => {
             if (!env) throw new Error("Test environment not initialized");
-            const result = await mcpClient.listResources();
+            const result = await mcpClient.listTools();
             // Assert based on actual successful response structure
-            expect(result).toHaveProperty("resources");
-            expect(Array.isArray(result.resources)).toBe(true);
+            expect(result).toHaveProperty("tools");
+            expect(Array.isArray(result.tools)).toBe(true);
+            expect(
+               result.tools.some((t) => t.name === "malloy_executeQuery"),
+            ).toBe(true);
          },
          { timeout: 30000 },
       );
 
-      it("should receive InvalidParams error when calling a known method with invalid params", async () => {
-         if (!env) throw new Error("Test environment not initialized");
-         expect.assertions(2);
-         try {
-            // Use the client from the test environment
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            await mcpClient.readResource({} as any); // Force invalid params
-         } catch (error) {
-            expect(error).toBeInstanceOf(Error);
-            // Check error code and message
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            expect((error as any).code).toBe(ErrorCode.InternalError);
-         }
-      });
-
       it("should receive MethodNotFound error for unknown methods", async () => {
          if (!env) throw new Error("Test environment not initialized");
          expect.assertions(2); // Expecting error instance and specific code
@@ -84,18 +72,6 @@ describe.serial("MCP Transport Tests (E2E Integration)", () => {
    });
 
    describe("Connection Management & Bridge Logic", () => {
-      // DELETED - Test irrelevant in stateless mode
-      // it("should maintain session across multiple sequential requests", async () => {
-      //    ...
-      // });
-
-      // DELETED - Test irrelevant in stateless mode
-      // it("should handle reconnection after close", async () => {
-      //    ...
-      // });
-
-      // --- Bridge Error Handling ---
-
       it("should return 405 Method Not Allowed for GET requests", async () => {
          if (!env) throw new Error("Test environment not initialized");
          const getUrl = new URL(`${env.serverUrl}/mcp`); // No session ID needed

package/tests/integration/watch-mode/watch_mode.integration.spec.ts

@@ -116,11 +116,6 @@ async function startServer(opts: { watch: boolean }): Promise<TestServer> {
    const port = await getFreePort();
    let mcpPort = await getFreePort();
    while (mcpPort === port) mcpPort = await getFreePort();
-   // The agent MCP server binds its own listener; give it a distinct free port
-   // too, otherwise every spawned child binds the fixed default and collides.
-   let agentMcpPort = await getFreePort();
-   while (agentMcpPort === port || agentMcpPort === mcpPort)
-      agentMcpPort = await getFreePort();
    const baseUrl = `http://127.0.0.1:${port}`;
 
    const env: NodeJS.ProcessEnv = {
@@ -129,7 +124,6 @@ async function startServer(opts: { watch: boolean }): Promise<TestServer> {
       PUBLISHER_HOST: "127.0.0.1",
       PUBLISHER_PORT: String(port),
       MCP_PORT: String(mcpPort),
-      AGENT_MCP_PORT: String(agentMcpPort),
    };
    if (opts.watch) env.PUBLISHER_WATCH = ENV_NAME;