@malloy-publisher/server 0.0.225 → 0.0.226

package/src/controller/package.controller.spec.ts

@@ -18,7 +18,7 @@ describe("PackageController.addPackage explores validation", () => {
          "Invalid explores entry 'missing.malloy' in publisher.json: file not found";
       const mockPackage = {
          formatInvalidExplores: () => invalidMsg,
-         formatInvalidSchedule: () => "",
+         formatInvalidPersistencePolicy: () => "",
       };
       const unloadPackage = sinon.stub().resolves(undefined);
       const deletePackage = sinon.stub().resolves(undefined);
@@ -55,7 +55,7 @@ describe("PackageController.addPackage explores validation", () => {
          "Invalid explores entry 'missing.malloy' in publisher.json: file not found";
       const mockPackage = {
          formatInvalidExplores: () => invalidMsg,
-         formatInvalidSchedule: () => "",
+         formatInvalidPersistencePolicy: () => "",
       };
       // installPackage mimics the real contract: invoke the validator and, if it
       // returns a message, throw BadRequestError (after its internal rollback).
@@ -104,7 +104,7 @@ describe("PackageController.addPackage explores validation", () => {
    it("persists when explores are valid (no-location)", async () => {
       const mockPackage = {
          formatInvalidExplores: () => "",
-         formatInvalidSchedule: () => "",
+         formatInvalidPersistencePolicy: () => "",
       };
       const addPackage = sinon.stub().resolves(mockPackage);
       const getEnvironment = sinon.stub().resolves({ addPackage });
@@ -127,22 +127,22 @@ describe("PackageController.addPackage explores validation", () => {
    });
 });
 
-describe("PackageController.addPackage cron schedule validation", () => {
+describe("PackageController.addPackage persistence policy validation", () => {
    afterEach(() => {
       sinon.restore();
    });
 
-   it("rejects a publish whose cron fails the private-only gate (no-location path)", async () => {
-      // Valid explores but an invalid materialization cron: the publish must
-      // still 400 (strict-at-publish, same split as explores — load merely
-      // warns) and roll back via unloadPackage.
+   it("rejects a publish whose persistence policy is invalid (no-location path)", async () => {
+      // Valid explores but an invalid persistence policy (e.g. a schedule on a
+      // package-scoped package): the publish must still 400 (strict-at-publish,
+      // same split as explores — load merely warns) and roll back via
+      // unloadPackage.
       const cronMsg =
-         "materialization.schedule (cron) in publisher.json requires every " +
-         "persist source to declare '#@ persist ... sharing=private'; source " +
-         "'orders' resolves to unset.";
+         'materialization.schedule (cron) in publisher.json requires "scope": ' +
+         '"version".';
       const mockPackage = {
          formatInvalidExplores: () => "",
-         formatInvalidSchedule: () => cronMsg,
+         formatInvalidPersistencePolicy: () => cronMsg,
       };
       const unloadPackage = sinon.stub().resolves(undefined);
       const addPackage = sinon.stub().resolves(mockPackage);
@@ -165,13 +165,13 @@ describe("PackageController.addPackage cron schedule validation", () => {
       expect(addPackageToDatabase.called).toBe(false);
    });
 
-   it("location path: the cron gate runs inside installPackage's rollback window", async () => {
+   it("location path: the persistence-policy gate runs inside installPackage's rollback window", async () => {
       const cronMsg =
-         "materialization.schedule (cron) in publisher.json requires every " +
-         "persist source to declare '#@ persist ... sharing=private'.";
+         'materialization.schedule (cron) in publisher.json requires "scope": ' +
+         '"version".';
       const mockPackage = {
          formatInvalidExplores: () => "",
-         formatInvalidSchedule: () => cronMsg,
+         formatInvalidPersistencePolicy: () => cronMsg,
       };
       const installPackage = sinon
          .stub()
@@ -225,7 +225,7 @@ describe("PackageController.updatePackage explores validation", () => {
       const mockPackage = {
          formatInvalidExplores: (override?: string[]) =>
             override?.includes("nope.malloy") ? invalidMsg : "",
-         formatInvalidSchedule: () => "",
+         formatInvalidPersistencePolicy: () => "",
       };
       const installPackage = sinon
          .stub()

package/src/controller/package.controller.ts

@@ -8,21 +8,22 @@ type ApiPackage = components["schemas"]["Package"];
 /**
  * Everything that is strict-at-publish, joined into one 400 message (or
  * undefined when the package is publishable): invalid explores entries plus
- * the materialization cron gate (a package-level `schedule` requires every
- * persist source to resolve to explicit `sharing=private` — see
- * Package.scheduleWarnings). At startup/reload both are warn-only instead
- * (fail-safe; see Package.loadViaWorker).
+ * the Malloy Persistence policy gate (scope is package-level; a
+ * `materialization.schedule` is package-root-only + version-scope-only and
+ * mutually exclusive with freshness; per-source `sharing`/`schedule` are
+ * retired — see Package.persistencePolicyWarnings). At startup/reload both are
+ * warn-only instead (fail-safe; see Package.loadViaWorker).
  */
 function formatPublishRejections(
    pkg: {
       formatInvalidExplores(exploresOverride?: string[]): string;
-      formatInvalidSchedule(): string;
+      formatInvalidPersistencePolicy(): string;
    },
    exploresOverride?: string[],
 ): string | undefined {
    const message = [
       pkg.formatInvalidExplores(exploresOverride),
-      pkg.formatInvalidSchedule(),
+      pkg.formatInvalidPersistencePolicy(),
    ]
       .filter(Boolean)
       .join("\n");

package/src/logger.spec.ts

@@ -0,0 +1,193 @@
+import { describe, expect, it } from "bun:test";
+import type { AxiosError } from "axios";
+import { buildAxiosErrorLog, redactSensitive } from "./logger";
+
+describe("redactSensitive", () => {
+   it("masks every known credential field at the top level", () => {
+      const input = {
+         password: "hunter2",
+         connectionString: "postgresql://u:p@h/db",
+         serviceAccountKeyJson: '{"private_key":"-----BEGIN..."}',
+         privateKey: "-----BEGIN PRIVATE KEY-----",
+         privateKeyPass: "passphrase",
+         secret: "gcs-hmac-secret",
+         secretAccessKey: "aws-secret",
+         sessionToken: "aws-session",
+         sasUrl: "https://a.blob.core.windows.net/c?sig=abc",
+         clientSecret: "azure-client-secret",
+         oauthClientSecret: "databricks-oauth-secret",
+         peakaKey: "peaka-api-key",
+         token: "databricks-pat",
+         accessToken: "motherduck-token",
+      };
+      const out = redactSensitive(input) as Record<string, unknown>;
+      for (const key of Object.keys(input)) {
+         expect(out[key]).toBe("[REDACTED]");
+      }
+   });
+
+   it("keeps non-secret fields intact", () => {
+      const input = {
+         name: "bigquery",
+         type: "bigquery",
+         defaultProjectId: "my-project",
+         host: "db.example.com",
+         port: 5432,
+      };
+      expect(redactSensitive(input)).toEqual(input);
+   });
+
+   it("redacts secrets nested inside the environment PATCH payload shape", () => {
+      const input = {
+         name: "demo___malloy-samples",
+         connections: [
+            {
+               name: "bigquery",
+               type: "bigquery",
+               bigqueryConnection: {
+                  defaultProjectId: "vscode-demo",
+                  serviceAccountKeyJson: '{"private_key":"-----BEGIN..."}',
+               },
+            },
+            {
+               name: "warehouse",
+               type: "snowflake",
+               snowflakeConnection: {
+                  account: "acct",
+                  username: "svc",
+                  password: "s3cret",
+                  privateKey: "-----BEGIN PRIVATE KEY-----",
+               },
+            },
+         ],
+      };
+      expect(redactSensitive(input)).toEqual({
+         name: "demo___malloy-samples",
+         connections: [
+            {
+               name: "bigquery",
+               type: "bigquery",
+               bigqueryConnection: {
+                  defaultProjectId: "vscode-demo",
+                  serviceAccountKeyJson: "[REDACTED]",
+               },
+            },
+            {
+               name: "warehouse",
+               type: "snowflake",
+               snowflakeConnection: {
+                  account: "acct",
+                  username: "svc",
+                  password: "[REDACTED]",
+                  privateKey: "[REDACTED]",
+               },
+            },
+         ],
+      });
+   });
+
+   it("recurses through arrays of secrets", () => {
+      const out = redactSensitive([{ password: "a" }, { password: "b" }]);
+      expect(out).toEqual([
+         { password: "[REDACTED]" },
+         { password: "[REDACTED]" },
+      ]);
+   });
+
+   it("passes primitives through unchanged", () => {
+      expect(redactSensitive("plain")).toBe("plain");
+      expect(redactSensitive(42)).toBe(42);
+      expect(redactSensitive(null)).toBeNull();
+      expect(redactSensitive(undefined)).toBeUndefined();
+   });
+
+   it("preserves Date values instead of destructuring them", () => {
+      const d = new Date("2026-07-08T00:00:00.000Z");
+      const out = redactSensitive({ createdAt: d }) as { createdAt: Date };
+      expect(out.createdAt).toBeInstanceOf(Date);
+      expect(out.createdAt.getTime()).toBe(d.getTime());
+   });
+
+   it("does not mutate the input object", () => {
+      const input = { password: "hunter2", nested: { token: "abc" } };
+      redactSensitive(input);
+      expect(input.password).toBe("hunter2");
+      expect(input.nested.token).toBe("abc");
+   });
+
+   it("tolerates circular references", () => {
+      const input: Record<string, unknown> = { password: "hunter2" };
+      input.self = input;
+      const out = redactSensitive(input) as Record<string, unknown>;
+      expect(out.password).toBe("[REDACTED]");
+      expect(out.self).toBe("[Circular]");
+   });
+
+   it("matches credential keys case-insensitively", () => {
+      const out = redactSensitive({
+         Password: "hunter2",
+         Authorization: "Bearer abc",
+      }) as Record<string, unknown>;
+      expect(out.Password).toBe("[REDACTED]");
+      expect(out.Authorization).toBe("[REDACTED]");
+   });
+
+   it("masks credential HTTP headers", () => {
+      const out = redactSensitive({
+         "content-type": "application/json",
+         authorization: "Bearer secret-token",
+         cookie: "session=abc",
+      }) as Record<string, unknown>;
+      expect(out["content-type"]).toBe("application/json");
+      expect(out.authorization).toBe("[REDACTED]");
+      expect(out.cookie).toBe("[REDACTED]");
+   });
+});
+
+describe("buildAxiosErrorLog", () => {
+   it("redacts response data and headers on a server-side error", () => {
+      const error = {
+         response: {
+            config: { url: "http://worker/api" },
+            status: 500,
+            headers: { authorization: "Bearer abc", "content-type": "json" },
+            data: { connection: { serviceAccountKeyJson: "PRIVATE_KEY" } },
+         },
+      } as unknown as AxiosError;
+      const { message, meta } = buildAxiosErrorLog(error);
+      expect(message).toBe("Axios server-side error");
+      expect(meta).toEqual({
+         url: "http://worker/api",
+         status: 500,
+         headers: { authorization: "[REDACTED]", "content-type": "json" },
+         data: { connection: { serviceAccountKeyJson: "[REDACTED]" } },
+      });
+   });
+
+   it("never logs the raw request object on a client-side error", () => {
+      const error = {
+         request: { _header: "GET / HTTP/1.1\r\nAuthorization: Bearer abc" },
+         config: { method: "get", url: "http://worker/api" },
+         code: "ECONNABORTED",
+      } as unknown as AxiosError;
+      const { message, meta } = buildAxiosErrorLog(error);
+      expect(message).toBe("Axios client-side error");
+      expect(meta).toEqual({
+         method: "get",
+         url: "http://worker/api",
+         code: "ECONNABORTED",
+      });
+      expect(JSON.stringify(meta)).not.toContain("Bearer abc");
+   });
+
+   it("logs only the message on a setup error", () => {
+      const error = {
+         message: "getaddrinfo ENOTFOUND",
+         config: { headers: { Authorization: "Bearer abc" } },
+      } as unknown as AxiosError;
+      const { message, meta } = buildAxiosErrorLog(error);
+      expect(message).toBe("Axios unknown error");
+      expect(meta).toEqual({ message: "getaddrinfo ENOTFOUND" });
+      expect(JSON.stringify(meta)).not.toContain("Bearer abc");
+   });
+});

package/src/logger.ts

@@ -84,6 +84,73 @@ export function formatDuration(durationMs: number): string {
    return `${durationMs.toFixed(2)}ms`;
 }
 
+// Connection endpoints carry provider credentials in their request/response
+// bodies (BigQuery serviceAccountKeyJson, Snowflake privateKey/password, Postgres
+// password/connectionString, S3 secretAccessKey/sessionToken, Azure clientSecret/
+// sasUrl, Databricks token, ...), and axios error logs can carry credential HTTP
+// headers (Authorization, Cookie). Those values are masked by key name before
+// they reach a log transport. Matching is case-insensitive so header casing
+// (Authorization vs authorization) does not slip through.
+const SENSITIVE_KEY_NAMES = [
+   // connection config credentials
+   "password",
+   "connectionString",
+   "serviceAccountKeyJson",
+   "privateKey",
+   "privateKeyPass",
+   "secret",
+   "secretAccessKey",
+   "sessionToken",
+   "sasUrl",
+   "clientSecret",
+   "oauthClientSecret",
+   "peakaKey",
+   "token",
+   "accessToken",
+   // credential-bearing HTTP headers
+   "authorization",
+   "proxy-authorization",
+   "cookie",
+   "set-cookie",
+   "x-api-key",
+];
+const SENSITIVE_KEYS: ReadonlySet<string> = new Set(
+   SENSITIVE_KEY_NAMES.map((name) => name.toLowerCase()),
+);
+
+const REDACTED = "[REDACTED]";
+
+/**
+ * Deep-copy `value`, replacing any property whose key names a credential
+ * (SENSITIVE_KEYS) with a placeholder. The match is by key name (case-
+ * insensitive) and recurses through nested objects and arrays, since
+ * connections arrive nested under `connections[]`. Primitives and Dates pass
+ * through unchanged and the input is never mutated; used to keep secrets out of
+ * the request/response and axios-error logs.
+ */
+export function redactSensitive(
+   value: unknown,
+   seen: WeakSet<object> = new WeakSet(),
+): unknown {
+   if (value === null || typeof value !== "object" || value instanceof Date) {
+      return value;
+   }
+   if (seen.has(value)) {
+      return "[Circular]";
+   }
+   seen.add(value);
+   if (Array.isArray(value)) {
+      return value.map((item) => redactSensitive(item, seen));
+   }
+   const result: Record<string, unknown> = {};
+   for (const [key, val] of Object.entries(value as Record<string, unknown>)) {
+      result[key] = SENSITIVE_KEYS.has(key.toLowerCase())
+         ? REDACTED
+         : redactSensitive(val, seen);
+   }
+   return result;
+}
+
 export const loggerMiddleware: RequestHandler = (req, res, next) => {
    const startTime = performance.now();
    const resJson = res.json;
@@ -102,14 +169,14 @@ export const loggerMiddleware: RequestHandler = (req, res, next) => {
       const logMetadata: Record<string, unknown> = {
          statusCode: res.statusCode,
          duration: formatDuration(durationMs),
-         payload: req.body,
+         payload: redactSensitive(req.body),
          params: req.params,
          query: req.query,
       };
 
       // Only include response body if response logging is enabled
       if (!DISABLE_RESPONSE_LOGGING) {
-         logMetadata.response = res.locals.body;
+         logMetadata.response = redactSensitive(res.locals.body);
       }
 
       // Add traceId to log metadata if present
@@ -130,23 +197,53 @@ export const loggerMiddleware: RequestHandler = (req, res, next) => {
    next();
 };
 
-export const logAxiosError = (error: AxiosError) => {
+/**
+ * Build the message + metadata logged for an AxiosError, with secrets removed.
+ * A failing upstream call can echo a connection config in `response.data` or
+ * return credential headers, and the raw `error.request` / `error` objects hold
+ * the outgoing Authorization header. Response fields are redacted and the raw
+ * request object is reduced to a few safe descriptors, so credentials never
+ * reach the log. Split out from `logAxiosError` so it can be unit tested.
+ */
+export function buildAxiosErrorLog(error: AxiosError): {
+   message: string;
+   meta: Record<string, unknown>;
+} {
    if (error.response) {
       // The request was made and the server responded with a status code
-      // that falls out of the range of 2xx
-      logger.error("Axios server-side error", {
-         url: error.response.config.url,
-         status: error.response.status,
-         headers: error.response.headers,
-         data: error.response.data,
-      });
-   } else if (error.request) {
-      // The request was made but no response was received
-      // `error.request` is an instance of XMLHttpRequest in the browser and an instance of
-      // http.ClientRequest in node.js
-      logger.error("Axios client-side error", { error: error.request });
-   } else {
-      // Something happened in setting up the request that triggered an Error
-      logger.error("Axios unknown error", { error });
+      // that falls out of the range of 2xx.
+      return {
+         message: "Axios server-side error",
+         meta: {
+            url: error.response.config.url,
+            status: error.response.status,
+            headers: redactSensitive(error.response.headers),
+            data: redactSensitive(error.response.data),
+         },
+      };
+   }
+   if (error.request) {
+      // The request was made but no response was received. `error.request` is
+      // the raw ClientRequest/XHR and carries the outgoing headers (including
+      // Authorization), so log only safe descriptors from the config.
+      return {
+         message: "Axios client-side error",
+         meta: {
+            method: error.config?.method,
+            url: error.config?.url,
+            code: error.code,
+         },
+      };
    }
+   // Something happened setting up the request. The AxiosError still holds
+   // `config.headers`, so log only the message.
+   return {
+      message: "Axios unknown error",
+      meta: { message: error.message },
+   };
+}
+
+export const logAxiosError = (error: AxiosError) => {
+   const { message, meta } = buildAxiosErrorLog(error);
+   logger.error(message, meta);
 };