@malloy-publisher/server 0.0.225 → 0.0.227

package/src/pg_helpers.spec.ts

@@ -1,105 +1,5 @@
-import { afterEach, describe, expect, it } from "bun:test";
-import { ConnectionAuthError } from "./errors";
-import {
-   classifyPgError,
-   handlePgAttachError,
-   pgConnectTimeoutSeconds,
-   redactPgSecrets,
-   withPgConnectTimeout,
-} from "./pg_helpers";
-
-describe("pgConnectTimeoutSeconds", () => {
-   const ORIGINAL_TIMEOUT = process.env.PG_CONNECT_TIMEOUT_SECONDS;
-
-   afterEach(() => {
-      if (ORIGINAL_TIMEOUT === undefined) {
-         delete process.env.PG_CONNECT_TIMEOUT_SECONDS;
-      } else {
-         process.env.PG_CONNECT_TIMEOUT_SECONDS = ORIGINAL_TIMEOUT;
-      }
-   });
-
-   it("defaults to 5 when env unset", () => {
-      delete process.env.PG_CONNECT_TIMEOUT_SECONDS;
-      expect(pgConnectTimeoutSeconds()).toBe(5);
-   });
-
-   it("honors PG_CONNECT_TIMEOUT_SECONDS override", () => {
-      process.env.PG_CONNECT_TIMEOUT_SECONDS = "12";
-      expect(pgConnectTimeoutSeconds()).toBe(12);
-   });
-
-   it("falls back to 5 when env value is invalid", () => {
-      process.env.PG_CONNECT_TIMEOUT_SECONDS = "not-a-number";
-      expect(pgConnectTimeoutSeconds()).toBe(5);
-   });
-
-   it("falls back to 5 when env value is zero or negative", () => {
-      process.env.PG_CONNECT_TIMEOUT_SECONDS = "0";
-      expect(pgConnectTimeoutSeconds()).toBe(5);
-      process.env.PG_CONNECT_TIMEOUT_SECONDS = "-3";
-      expect(pgConnectTimeoutSeconds()).toBe(5);
-   });
-});
-
-describe("withPgConnectTimeout", () => {
-   it("appends to keyword form when missing", () => {
-      expect(withPgConnectTimeout("host=h dbname=d user=u password=p", 5)).toBe(
-         "host=h dbname=d user=u password=p connect_timeout=5",
-      );
-   });
-
-   it("appends to postgres: keyword form (DuckLake catalogUrl shape)", () => {
-      expect(
-         withPgConnectTimeout("postgres:host=h user=u password=p dbname=d", 5),
-      ).toBe("postgres:host=h user=u password=p dbname=d connect_timeout=5");
-   });
-
-   it("does not override a user-supplied connect_timeout in keyword form", () => {
-      expect(withPgConnectTimeout("host=h connect_timeout=30", 99)).toBe(
-         "host=h connect_timeout=30",
-      );
-   });
-
-   it("appends to URI form with no query", () => {
-      expect(withPgConnectTimeout("postgresql://u:p@h:5432/d", 5)).toBe(
-         "postgresql://u:p@h:5432/d?connect_timeout=5",
-      );
-   });
-
-   it("appends to URI form with existing query", () => {
-      expect(
-         withPgConnectTimeout("postgresql://u:p@h/d?sslmode=require", 5),
-      ).toBe("postgresql://u:p@h/d?sslmode=require&connect_timeout=5");
-   });
-
-   it("appends to URI with bare trailing ?", () => {
-      expect(withPgConnectTimeout("postgresql://h/d?", 5)).toBe(
-         "postgresql://h/d?connect_timeout=5",
-      );
-   });
-
-   it("does not double-append when URI already has connect_timeout (?-style)", () => {
-      expect(
-         withPgConnectTimeout("postgresql://h/d?connect_timeout=10", 5),
-      ).toBe("postgresql://h/d?connect_timeout=10");
-   });
-
-   it("does not double-append when URI already has connect_timeout (&-style)", () => {
-      expect(
-         withPgConnectTimeout(
-            "postgresql://h/d?sslmode=require&connect_timeout=10",
-            5,
-         ),
-      ).toBe("postgresql://h/d?sslmode=require&connect_timeout=10");
-   });
-
-   it("recognizes postgres:// (alternative scheme) as URI form", () => {
-      expect(withPgConnectTimeout("postgres://u@h/d", 5)).toBe(
-         "postgres://u@h/d?connect_timeout=5",
-      );
-   });
-});
+import { describe, expect, it } from "bun:test";
+import { redactPgSecrets } from "./pg_helpers";
 
 describe("redactPgSecrets", () => {
    it("redacts bare password values", () => {
@@ -120,107 +20,3 @@ describe("redactPgSecrets", () => {
       );
    });
 });
-
-describe("classifyPgError", () => {
-   it.each([
-      'password authentication failed for user "alice"',
-      "no pg_hba.conf entry for host",
-      'role "alice" does not exist',
-      'database "billing" does not exist',
-      "permission denied for relation foo",
-   ])("classifies '%s' as auth error", (msg) => {
-      const result = classifyPgError(new Error(msg), "PG attach");
-      expect(result).toBeInstanceOf(ConnectionAuthError);
-      expect(result?.message).toContain("PG attach:");
-   });
-
-   it("returns undefined for unrelated errors", () => {
-      expect(
-         classifyPgError(
-            new Error('relation "users" does not exist'),
-            "PG attach",
-         ),
-      ).toBeUndefined();
-      expect(
-         classifyPgError(new Error("connection reset by peer"), "PG attach"),
-      ).toBeUndefined();
-   });
-
-   it("returns undefined for non-Error values", () => {
-      expect(
-         classifyPgError("password authentication failed", "ctx"),
-      ).toBeUndefined();
-      expect(classifyPgError(undefined, "ctx")).toBeUndefined();
-   });
-
-   it("redacts embedded passwords in the wrapped message", () => {
-      const result = classifyPgError(
-         new Error(
-            "password authentication failed: tried host=h password=hunter2",
-         ),
-         "DuckLake attach",
-      );
-      expect(result?.message).toContain("password=***");
-      expect(result?.message).not.toContain("hunter2");
-   });
-});
-
-describe("handlePgAttachError", () => {
-   it("swallows 'already exists' errors", () => {
-      const outcome = handlePgAttachError(
-         new Error('database "db_x" already exists'),
-         "ctx",
-      );
-      expect(outcome.action).toBe("swallow");
-   });
-
-   it("swallows 'already attached' errors", () => {
-      const outcome = handlePgAttachError(
-         new Error("DuckLake catalog db_x is already attached"),
-         "ctx",
-      );
-      expect(outcome.action).toBe("swallow");
-   });
-
-   it("classifies libpq auth failures as ConnectionAuthError", () => {
-      const outcome = handlePgAttachError(
-         new Error('password authentication failed for user "alice"'),
-         "PG attach db_x",
-      );
-      expect(outcome.action).toBe("throw");
-      if (outcome.action === "throw") {
-         expect(outcome.error).toBeInstanceOf(ConnectionAuthError);
-         expect(outcome.error.message).toContain("PG attach db_x:");
-      }
-   });
-
-   it("passes through unrelated Error instances unchanged", () => {
-      const original = new Error("network unreachable");
-      const outcome = handlePgAttachError(original, "ctx");
-      expect(outcome.action).toBe("throw");
-      if (outcome.action === "throw") {
-         expect(outcome.error).toBe(original);
-         expect(outcome.error).not.toBeInstanceOf(ConnectionAuthError);
-      }
-   });
-
-   it("wraps non-Error throwables so callers always get an Error", () => {
-      const outcome = handlePgAttachError("a string was thrown", "ctx");
-      expect(outcome.action).toBe("throw");
-      if (outcome.action === "throw") {
-         expect(outcome.error).toBeInstanceOf(Error);
-         expect(outcome.error.message).toBe("a string was thrown");
-      }
-   });
-
-   it("prefers 'already attached' over auth classification when both keywords appear", () => {
-      // Defensive: if a future DuckDB version emits a combined message,
-      // 'already attached' wins so we don't bubble up a false auth failure
-      // on what is actually a benign idempotent re-attach.
-      const outcome = handlePgAttachError(
-         new Error("already attached; permission denied tail"),
-         "ctx",
-      );
-      expect(outcome.action).toBe("swallow");
-   });
-});

package/src/pg_helpers.ts

@@ -1,64 +1,7 @@
-// Postgres / libpq helpers shared between `service/` (user-facing
-// connections) and `storage/` (materialization-storage catalog). Lives at
-// `src/` root so neither layer takes a dependency on the other — see
-// CLAUDE.md's "Two parallel DuckLake/PG attach paths" note for why this
-// matters.
-import { ConnectionAuthError } from "./errors";
-
-// Default Postgres connect_timeout (seconds), used by the materialization
-// storage catalog ATTACH so a slow or wedged libpq handshake fails the
-// caller in seconds instead of stalling the worker until the K8s liveness
-// probe trips.
-//
-// libpq enforces a documented minimum of 2 seconds — values below 2
-// effectively round up to ~2s wall clock.
-export function pgConnectTimeoutSeconds(): number {
-   const raw = process.env.PG_CONNECT_TIMEOUT_SECONDS;
-   if (!raw) return 5;
-   const parsed = Number.parseInt(raw, 10);
-   return Number.isFinite(parsed) && parsed > 0 ? parsed : 5;
-}
-
-// libpq accepts both keyword=value form ("host=h dbname=d") and URI form
-// ("postgresql://u:p@h/d?param=v"). The materialization-storage catalogUrl
-// can also arrive as `postgres:<keyword=value>` (no `//`). We detect URI
-// form (with `//`) so we know whether to append a new parameter using
-// `?`/`&` or a leading space.
-const URI_FORM_RE = /^[a-z][a-z0-9+.-]*:\/\//i;
-
-// Match an existing connect_timeout key in either form. URI form uses
-// `?key=` or `&key=`; keyword form uses whitespace separation or start-of-
-// string. Without the `[?&]` alternatives a URI-form user-supplied timeout
-// would be missed and we'd double-append, producing an invalid URL.
-const HAS_CONNECT_TIMEOUT_RE = /[?&\s]connect_timeout=|^connect_timeout=/;
-
-// Append `connect_timeout=N` to a libpq-compatible connection string if
-// the caller hasn't already set one. Handles keyword form ("host=h ..."),
-// URI form ("postgresql://..."), and the `postgres:host=h ...` keyword
-// form with a scheme prefix used by DuckLake catalogUrls.
-export function withPgConnectTimeout(
-   connectionString: string,
-   timeout: number,
-): string {
-   if (HAS_CONNECT_TIMEOUT_RE.test(connectionString)) {
-      return connectionString;
-   }
-   if (URI_FORM_RE.test(connectionString)) {
-      // URI form: append as query parameter. `?` if no query string yet,
-      // `&` otherwise. A bare trailing `?` (empty query) gets no extra
-      // separator. We don't try to handle URL fragments — libpq URIs don't
-      // use them.
-      if (!connectionString.includes("?")) {
-         return `${connectionString}?connect_timeout=${timeout}`;
-      }
-      if (connectionString.endsWith("?")) {
-         return `${connectionString}connect_timeout=${timeout}`;
-      }
-      return `${connectionString}&connect_timeout=${timeout}`;
-   }
-   // Keyword=value form (with or without `postgres:` scheme prefix).
-   return `${connectionString} connect_timeout=${timeout}`;
-}
+// Postgres / libpq helper shared between `service/` (user-facing connections)
+// and `storage/` (materialization-storage catalog). Lives at `src/` root so
+// neither layer takes a dependency on the other — see CLAUDE.md's "Two
+// parallel DuckLake/PG attach paths" note for why this matters.
 
 // Redact libpq `password=...` values from a string before it goes into a
 // log line or HTTP response body. Handles bare and quoted values.
@@ -68,62 +11,3 @@ export function withPgConnectTimeout(
 export function redactPgSecrets(s: string): string {
    return s.replace(/password=('[^']*'|"[^"]*"|\S+)/gi, "password=***");
 }
-
-// Substring-match libpq error patterns that indicate a non-retryable
-// auth/permission failure. Returns a ConnectionAuthError when matched so
-// callers can fast-fail with HTTP 422 (semantically "the supplied creds
-// are bad; don't retry") instead of letting the raw error fall through to
-// a generic 500 that retry loops treat as transient.
-export function classifyPgError(
-   error: unknown,
-   context: string,
-): ConnectionAuthError | undefined {
-   if (!(error instanceof Error)) return undefined;
-   const msg = error.message;
-   const patterns = [
-      /password authentication failed/i,
-      /pg_hba\.conf/i,
-      /role ".*" does not exist/i,
-      /database ".*" does not exist/i,
-      /permission denied/i,
-   ];
-   if (!patterns.some((p) => p.test(msg))) return undefined;
-   return new ConnectionAuthError(`${context}: ${redactPgSecrets(msg)}`);
-}
-
-// Outcome of inspecting an error thrown by an `ATTACH` call:
-//   - `{ action: "swallow" }`: DuckDB reported the db is already attached
-//     (idempotent re-attach); caller should log and continue.
-//   - `{ action: "throw", error: ConnectionAuthError }`: classified as a
-//     non-retryable auth failure; caller should warn-log and throw it.
-//   - `{ action: "throw", error: <original> }`: unrecognized; caller
-//     should rethrow as-is to preserve the original cause for diagnosis.
-//
-// Extracted so the decision tree gets a direct unit test without needing
-// to stub DuckDB or run a real ATTACH.
-export type PgAttachErrorOutcome =
-   | { action: "swallow" }
-   | { action: "throw"; error: Error };
-
-export function handlePgAttachError(
-   error: unknown,
-   context: string,
-): PgAttachErrorOutcome {
-   if (
-      error instanceof Error &&
-      (error.message.includes("already exists") ||
-         error.message.includes("already attached"))
-   ) {
-      return { action: "swallow" };
-   }
-   const authErr = classifyPgError(error, context);
-   if (authErr) {
-      return { action: "throw", error: authErr };
-   }
-   if (error instanceof Error) {
-      return { action: "throw", error };
-   }
-   // Non-Error thrown values get wrapped so the catch contract stays
-   // (always throws an Error).
-   return { action: "throw", error: new Error(String(error)) };
-}

package/src/server.ts

@@ -41,7 +41,6 @@ import { queryConcurrency } from "./query_concurrency";
 import { MaterializationController } from "./controller/materialization.controller";
 import { ThemeController } from "./controller/theme.controller";
 import { initializeMcpServer } from "./mcp/server";
-import { startAgentMcpServer } from "./mcp/agent_server";
 import { registerLegacyRoutes } from "./server-old";
 import { EnvironmentStore } from "./service/environment_store";
 import { MaterializationService } from "./service/materialization_service";
@@ -50,8 +49,6 @@ import { PackageMemoryGovernor } from "./service/package_memory_governor";
 import { ThemeStore } from "./service/theme_store";
 import { assertSafePackageName, safeJoinUnderRoot } from "./path_safety";
 
-export { normalizeQueryArray } from "./query_param_utils";
-
 // Parse command line arguments
 function parseArgs() {
    const args = process.argv.slice(2);
@@ -162,7 +159,6 @@ parseArgs();
 const PUBLISHER_PORT = Number(process.env.PUBLISHER_PORT || 4000);
 const PUBLISHER_HOST = process.env.PUBLISHER_HOST || "0.0.0.0";
 const MCP_PORT = Number(process.env.MCP_PORT || 4040);
-const AGENT_MCP_PORT = Number(process.env.AGENT_MCP_PORT || 4041);
 const MCP_ENDPOINT = "/mcp";
 const SHUTDOWN_DRAIN_DURATION_SECONDS = Number(
    process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0,
@@ -1868,21 +1864,9 @@ mcpServer.timeout = 600000;
 mcpServer.keepAliveTimeout = 600000;
 mcpServer.headersTimeout = 600000;
 
-// Separate, isolated MCP server for the agent retrieval tools (get_context,
-// search_docs) on its own listener. Kept apart from the core MCP server above.
-const agentMcpServer = startAgentMcpServer(
-   environmentStore,
-   PUBLISHER_HOST,
-   AGENT_MCP_PORT,
-);
-agentMcpServer.timeout = 600000;
-agentMcpServer.keepAliveTimeout = 600000;
-agentMcpServer.headersTimeout = 600000;
-
 registerSignalHandlers(
    mainServer,
    mcpServer,
    SHUTDOWN_DRAIN_DURATION_SECONDS,
    SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS,
-   agentMcpServer,
 );

package/src/service/build_plan.spec.ts

@@ -10,7 +10,7 @@ import {
    deriveBuildPlan,
    flattenDependsOn,
    iterGraphSources,
-   resolveFreshnessSchedule,
+   resolveFreshness,
    resolvePackageConnections,
 } from "./build_plan";
 import { fakeSource } from "./materialization_test_fixtures";
@@ -213,10 +213,10 @@ describe("deriveBuildPlan", () => {
       });
    });
 
-   it("reports declared sharing/refresh verbatim and null when unset", () => {
-      // The control plane distinguishes unset from an explicit `shared` (it
-      // applies the platform default itself), so the publisher must report
-      // the declared value verbatim — never substitute the default.
+   it("reports declared refresh verbatim (null when unset) and does not emit sharing/schedule", () => {
+      // `refresh` is a metadata pass-through; `sharing`/`schedule` were retired
+      // from the contract and must not be emitted as typed fields (they stay in
+      // the raw annotationFields for the publish-time validator to detect).
       const declared = fakeSource({
          name: "declared",
          sourceEntityId: "bid-d",
@@ -243,17 +243,22 @@ describe("deriveBuildPlan", () => {
          { duckdb: "dig" },
       );
 
-      expect(plan.sources["declared@m"].sharing).toBe("private");
       expect(plan.sources["declared@m"].refresh).toBe("incremental");
-      // The raw annotation map still carries every field alongside the typed
-      // projections.
+      // Retired typed fields are absent from the wire projection.
+      expect(
+         (plan.sources["declared@m"] as Record<string, unknown>).sharing,
+      ).toBeUndefined();
+      expect(
+         (plan.sources["declared@m"] as Record<string, unknown>).schedule,
+      ).toBeUndefined();
+      // The raw annotation map still carries every field (so the validator can
+      // reject a source-level sharing/schedule at publish).
       expect(plan.sources["declared@m"].annotationFields).toEqual({
          name: "d_table",
          sharing: "private",
          refresh: "incremental",
       });
-      // Unset is null — not "shared" — on the wire.
-      expect(plan.sources["unset@m"].sharing).toBeNull();
+      // Unset refresh is null on the wire.
       expect(plan.sources["unset@m"].refresh).toBeNull();
    });
 
@@ -297,71 +302,48 @@ describe("deriveBuildPlan", () => {
    });
 });
 
-describe("resolveFreshnessSchedule", () => {
-   it("reports source-level freshness + schedule verbatim", () => {
+describe("resolveFreshness", () => {
+   it("reports source-level freshness verbatim", () => {
       const source = fakeSource({
          name: "s",
          sourceEntityId: "bid",
          freshnessSchedule: {
             freshness: { window: "1h", fallback: "stale_ok" },
-            schedule: "0 */6 * * *",
          },
       });
-      expect(resolveFreshnessSchedule(source, null)).toEqual({
-         freshness: { window: "1h", fallback: "stale_ok" },
-         schedule: "0 */6 * * *",
+      expect(resolveFreshness(source, null)).toEqual({
+         window: "1h",
+         fallback: "stale_ok",
       });
    });
 
-   it("returns null for both when unset at every level", () => {
+   it("returns null when unset at every level", () => {
       const source = fakeSource({ name: "s", sourceEntityId: "bid" });
-      expect(resolveFreshnessSchedule(source, null)).toEqual({
-         freshness: null,
-         schedule: null,
-      });
+      expect(resolveFreshness(source, null)).toBeNull();
    });
 
    it("falls back to model-file then package per field (most-specific-wins)", () => {
-      // freshness.window from source, freshness.fallback from model-file. The
-      // package-level cron is NOT inherited as the source's effective schedule;
-      // schedule here comes only from the source's own model-file default.
+      // freshness.window from source, freshness.fallback from model-file.
       const source = fakeSource({
          name: "s",
          sourceEntityId: "bid",
          freshnessSchedule: { freshness: { window: "1h" } },
-         modelFreshnessSchedule: {
-            freshness: { fallback: "fail" },
-            schedule: "0 3 * * *",
-         },
+         modelFreshnessSchedule: { freshness: { fallback: "fail" } },
       });
       const pkg = {
-         schedule: "0 0 * * *",
+         schedule: null,
          freshness: { window: "24h", fallback: "live" as const },
       };
-      expect(resolveFreshnessSchedule(source, pkg)).toEqual({
-         freshness: { window: "1h", fallback: "fail" },
-         schedule: "0 3 * * *",
-      });
-   });
-
-   it("does not inherit the package-level cron as the source's schedule", () => {
-      const source = fakeSource({ name: "s", sourceEntityId: "bid" });
-      const pkg = { schedule: "0 0 * * *", freshness: { window: "24h" } };
-      // Freshness inherits from the package; schedule stays null (the package
-      // cron is gated separately, not folded into the per-source cron).
-      expect(resolveFreshnessSchedule(source, pkg)).toEqual({
-         freshness: { window: "24h" },
-         schedule: null,
+      expect(resolveFreshness(source, pkg)).toEqual({
+         window: "1h",
+         fallback: "fail",
       });
    });
 
    it("inherits the package freshness when the source and model are unset", () => {
       const source = fakeSource({ name: "s", sourceEntityId: "bid" });
       const pkg = { schedule: null, freshness: { window: "24h" } };
-      expect(resolveFreshnessSchedule(source, pkg)).toEqual({
-         freshness: { window: "24h" },
-         schedule: null,
-      });
+      expect(resolveFreshness(source, pkg)).toEqual({ window: "24h" });
    });
 
    it("drops an invalid fallback rather than defaulting it", () => {
@@ -370,23 +352,17 @@ describe("resolveFreshnessSchedule", () => {
          sourceEntityId: "bid",
          freshnessSchedule: { freshness: { window: "1h", fallback: "bogus" } },
       });
-      expect(resolveFreshnessSchedule(source, null)).toEqual({
-         freshness: { window: "1h" },
-         schedule: null,
-      });
+      expect(resolveFreshness(source, null)).toEqual({ window: "1h" });
    });
 });
 
-describe("deriveBuildPlan freshness/schedule", () => {
-   it("projects the resolved per-source freshness + schedule onto the plan", () => {
+describe("deriveBuildPlan freshness", () => {
+   it("projects the resolved per-source freshness onto the plan (no schedule/sharing)", () => {
       const source = fakeSource({
          name: "s",
          sourceEntityId: "bid",
-         annotationFields: { name: "s_table", sharing: "private" },
-         freshnessSchedule: {
-            freshness: { window: "1h" },
-            schedule: "0 */6 * * *",
-         },
+         annotationFields: { name: "s_table" },
+         freshnessSchedule: { freshness: { window: "1h" } },
       });
       const plan = deriveBuildPlan(
          [
@@ -406,12 +382,18 @@ describe("deriveBuildPlan freshness/schedule", () => {
       );
 
       // Source window wins over the package default; package fallback fills the
-      // unset source fallback; source schedule reported verbatim.
+      // unset source fallback.
       expect(plan.sources["s@m"].freshness).toEqual({
          window: "1h",
          fallback: "live",
       });
-      expect(plan.sources["s@m"].schedule).toBe("0 */6 * * *");
+      // Retired fields are not emitted.
+      expect(
+         (plan.sources["s@m"] as Record<string, unknown>).schedule,
+      ).toBeUndefined();
+      expect(
+         (plan.sources["s@m"] as Record<string, unknown>).sharing,
+      ).toBeUndefined();
    });
 });