@malloy-publisher/server 0.0.224 → 0.0.226

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.224",
+   "version": "0.0.226",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"

package/src/controller/materialization.controller.spec.ts

@@ -85,6 +85,78 @@ describe("MaterializationController.createMaterialization validation", () => {
       expect(await parse({ buildInstructions: null })).toEqual({});
    });
 
+   it("parses referenceManifest and strictUpstreams alongside sources", async () => {
+      const parsed = await parse({
+         buildInstructions: {
+            sources: [
+               {
+                  sourceEntityId: "b2",
+                  materializedTableId: "mt-2",
+                  physicalTableName: "downstream_v1",
+                  realization: "COPY",
+               },
+            ],
+            referenceManifest: [
+               { sourceEntityId: "b1", physicalTableName: "upstream_table" },
+            ],
+            strictUpstreams: true,
+         },
+      });
+      expect(parsed).toEqual({
+         buildInstructions: [
+            {
+               sourceEntityId: "b2",
+               sourceID: undefined,
+               materializedTableId: "mt-2",
+               physicalTableName: "downstream_v1",
+               realization: "COPY",
+            },
+         ],
+         referenceManifest: [
+            { sourceEntityId: "b1", physicalTableName: "upstream_table" },
+         ],
+         strictUpstreams: true,
+      });
+   });
+
+   it("rejects a referenceManifest entry missing a required field", async () => {
+      const { controller } = build();
+      await expect(
+         controller.createMaterialization("env", "pkg", {
+            buildInstructions: {
+               sources: [
+                  {
+                     sourceEntityId: "b2",
+                     materializedTableId: "mt-2",
+                     physicalTableName: "downstream_v1",
+                     realization: "COPY",
+                  },
+               ],
+               referenceManifest: [{ sourceEntityId: "b1" }],
+            },
+         }),
+      ).rejects.toThrow(BadRequestError);
+   });
+
+   it("rejects a non-boolean strictUpstreams", async () => {
+      const { controller } = build();
+      await expect(
+         controller.createMaterialization("env", "pkg", {
+            buildInstructions: {
+               sources: [
+                  {
+                     sourceEntityId: "b2",
+                     materializedTableId: "mt-2",
+                     physicalTableName: "downstream_v1",
+                     realization: "COPY",
+                  },
+               ],
+               strictUpstreams: "yes",
+            },
+         }),
+      ).rejects.toThrow(BadRequestError);
+   });
+
    it("rejects buildInstructions without a non-empty sources array", async () => {
       const { controller } = build();
       await expect(

package/src/controller/materialization.controller.ts

@@ -1,5 +1,8 @@
 import { BadRequestError } from "../errors";
-import { BuildInstruction } from "../storage/DatabaseInterface";
+import {
+   BuildInstruction,
+   ManifestReference,
+} from "../storage/DatabaseInterface";
 import { MaterializationService } from "../service/materialization_service";
 
 export class MaterializationController {
@@ -21,19 +24,28 @@ export class MaterializationController {
       forceRefresh?: boolean;
       sourceNames?: string[];
       buildInstructions?: BuildInstruction[];
+      referenceManifest?: ManifestReference[];
+      strictUpstreams?: boolean;
    } {
       const result: {
          forceRefresh?: boolean;
          sourceNames?: string[];
          buildInstructions?: BuildInstruction[];
+         referenceManifest?: ManifestReference[];
+         strictUpstreams?: boolean;
       } = {};
       if (
          body.buildInstructions !== undefined &&
          body.buildInstructions !== null
       ) {
-         result.buildInstructions = this.validateBuildInstructions(
-            body.buildInstructions,
-         );
+         const parsed = this.validateBuildInstructions(body.buildInstructions);
+         result.buildInstructions = parsed.sources;
+         if (parsed.referenceManifest !== undefined) {
+            result.referenceManifest = parsed.referenceManifest;
+         }
+         if (parsed.strictUpstreams !== undefined) {
+            result.strictUpstreams = parsed.strictUpstreams;
+         }
       }
       if (body.forceRefresh !== undefined) {
          if (typeof body.forceRefresh !== "boolean") {
@@ -57,22 +69,74 @@ export class MaterializationController {
 
    /**
     * Validate the orchestrated `buildInstructions` payload (BuildInstructions:
-    * `{ sources: BuildInstruction[] }`) and flatten it to the instruction list
-    * the service consumes.
+    * `{ sources: BuildInstruction[], referenceManifest?, strictUpstreams? }`)
+    * into the parts the service consumes: the flattened instruction list, the
+    * optional upstream reference manifest, and the strict flag.
     */
-   private validateBuildInstructions(raw: unknown): BuildInstruction[] {
+   private validateBuildInstructions(raw: unknown): {
+      sources: BuildInstruction[];
+      referenceManifest?: ManifestReference[];
+      strictUpstreams?: boolean;
+   } {
       if (typeof raw !== "object" || raw === null) {
          throw new BadRequestError("buildInstructions must be an object");
       }
-      const sources = (raw as Record<string, unknown>).sources;
+      const obj = raw as Record<string, unknown>;
+      const sources = obj.sources;
       if (!Array.isArray(sources) || sources.length === 0) {
          throw new BadRequestError(
             "buildInstructions requires a non-empty 'sources' array of BuildInstruction",
          );
       }
-      return sources.map((instruction) =>
-         this.validateInstruction(instruction),
-      );
+      const result: {
+         sources: BuildInstruction[];
+         referenceManifest?: ManifestReference[];
+         strictUpstreams?: boolean;
+      } = {
+         sources: sources.map((instruction) =>
+            this.validateInstruction(instruction),
+         ),
+      };
+      if (
+         obj.referenceManifest !== undefined &&
+         obj.referenceManifest !== null
+      ) {
+         if (!Array.isArray(obj.referenceManifest)) {
+            throw new BadRequestError(
+               "buildInstructions.referenceManifest must be an array of ManifestReference",
+            );
+         }
+         result.referenceManifest = obj.referenceManifest.map((ref) =>
+            this.validateManifestReference(ref),
+         );
+      }
+      if (obj.strictUpstreams !== undefined) {
+         if (typeof obj.strictUpstreams !== "boolean") {
+            throw new BadRequestError(
+               "buildInstructions.strictUpstreams must be a boolean",
+            );
+         }
+         result.strictUpstreams = obj.strictUpstreams;
+      }
+      return result;
+   }
+
+   private validateManifestReference(raw: unknown): ManifestReference {
+      if (typeof raw !== "object" || raw === null) {
+         throw new BadRequestError("Each manifest reference must be an object");
+      }
+      const ref = raw as Record<string, unknown>;
+      for (const field of ["sourceEntityId", "physicalTableName"] as const) {
+         if (typeof ref[field] !== "string") {
+            throw new BadRequestError(
+               `Manifest reference is missing required string field '${field}'`,
+            );
+         }
+      }
+      return {
+         sourceEntityId: ref.sourceEntityId as string,
+         physicalTableName: ref.physicalTableName as string,
+      };
    }
 
    private validateInstruction(raw: unknown): BuildInstruction {

package/src/controller/package.controller.spec.ts

@@ -18,7 +18,7 @@ describe("PackageController.addPackage explores validation", () => {
          "Invalid explores entry 'missing.malloy' in publisher.json: file not found";
       const mockPackage = {
          formatInvalidExplores: () => invalidMsg,
-         formatInvalidSchedule: () => "",
+         formatInvalidPersistencePolicy: () => "",
       };
       const unloadPackage = sinon.stub().resolves(undefined);
       const deletePackage = sinon.stub().resolves(undefined);
@@ -55,7 +55,7 @@ describe("PackageController.addPackage explores validation", () => {
          "Invalid explores entry 'missing.malloy' in publisher.json: file not found";
       const mockPackage = {
          formatInvalidExplores: () => invalidMsg,
-         formatInvalidSchedule: () => "",
+         formatInvalidPersistencePolicy: () => "",
       };
       // installPackage mimics the real contract: invoke the validator and, if it
       // returns a message, throw BadRequestError (after its internal rollback).
@@ -104,7 +104,7 @@ describe("PackageController.addPackage explores validation", () => {
    it("persists when explores are valid (no-location)", async () => {
       const mockPackage = {
          formatInvalidExplores: () => "",
-         formatInvalidSchedule: () => "",
+         formatInvalidPersistencePolicy: () => "",
       };
       const addPackage = sinon.stub().resolves(mockPackage);
       const getEnvironment = sinon.stub().resolves({ addPackage });
@@ -127,22 +127,22 @@ describe("PackageController.addPackage explores validation", () => {
    });
 });
 
-describe("PackageController.addPackage cron schedule validation", () => {
+describe("PackageController.addPackage persistence policy validation", () => {
    afterEach(() => {
       sinon.restore();
    });
 
-   it("rejects a publish whose cron fails the private-only gate (no-location path)", async () => {
-      // Valid explores but an invalid materialization cron: the publish must
-      // still 400 (strict-at-publish, same split as explores — load merely
-      // warns) and roll back via unloadPackage.
+   it("rejects a publish whose persistence policy is invalid (no-location path)", async () => {
+      // Valid explores but an invalid persistence policy (e.g. a schedule on a
+      // package-scoped package): the publish must still 400 (strict-at-publish,
+      // same split as explores — load merely warns) and roll back via
+      // unloadPackage.
       const cronMsg =
-         "materialization.schedule (cron) in publisher.json requires every " +
-         "persist source to declare '#@ persist ... sharing=private'; source " +
-         "'orders' resolves to unset.";
+         'materialization.schedule (cron) in publisher.json requires "scope": ' +
+         '"version".';
       const mockPackage = {
          formatInvalidExplores: () => "",
-         formatInvalidSchedule: () => cronMsg,
+         formatInvalidPersistencePolicy: () => cronMsg,
       };
       const unloadPackage = sinon.stub().resolves(undefined);
       const addPackage = sinon.stub().resolves(mockPackage);
@@ -165,13 +165,13 @@ describe("PackageController.addPackage cron schedule validation", () => {
       expect(addPackageToDatabase.called).toBe(false);
    });
 
-   it("location path: the cron gate runs inside installPackage's rollback window", async () => {
+   it("location path: the persistence-policy gate runs inside installPackage's rollback window", async () => {
       const cronMsg =
-         "materialization.schedule (cron) in publisher.json requires every " +
-         "persist source to declare '#@ persist ... sharing=private'.";
+         'materialization.schedule (cron) in publisher.json requires "scope": ' +
+         '"version".';
       const mockPackage = {
          formatInvalidExplores: () => "",
-         formatInvalidSchedule: () => cronMsg,
+         formatInvalidPersistencePolicy: () => cronMsg,
       };
       const installPackage = sinon
          .stub()
@@ -225,7 +225,7 @@ describe("PackageController.updatePackage explores validation", () => {
       const mockPackage = {
          formatInvalidExplores: (override?: string[]) =>
             override?.includes("nope.malloy") ? invalidMsg : "",
-         formatInvalidSchedule: () => "",
+         formatInvalidPersistencePolicy: () => "",
       };
       const installPackage = sinon
          .stub()

package/src/controller/package.controller.ts

@@ -8,21 +8,22 @@ type ApiPackage = components["schemas"]["Package"];
 /**
  * Everything that is strict-at-publish, joined into one 400 message (or
  * undefined when the package is publishable): invalid explores entries plus
- * the materialization cron gate (a package-level `schedule` requires every
- * persist source to resolve to explicit `sharing=private` — see
- * Package.scheduleWarnings). At startup/reload both are warn-only instead
- * (fail-safe; see Package.loadViaWorker).
+ * the Malloy Persistence policy gate (scope is package-level; a
+ * `materialization.schedule` is package-root-only + version-scope-only and
+ * mutually exclusive with freshness; per-source `sharing`/`schedule` are
+ * retired — see Package.persistencePolicyWarnings). At startup/reload both are
+ * warn-only instead (fail-safe; see Package.loadViaWorker).
  */
 function formatPublishRejections(
    pkg: {
       formatInvalidExplores(exploresOverride?: string[]): string;
-      formatInvalidSchedule(): string;
+      formatInvalidPersistencePolicy(): string;
    },
    exploresOverride?: string[],
 ): string | undefined {
    const message = [
       pkg.formatInvalidExplores(exploresOverride),
-      pkg.formatInvalidSchedule(),
+      pkg.formatInvalidPersistencePolicy(),
    ]
       .filter(Boolean)
       .join("\n");

package/src/logger.spec.ts

@@ -0,0 +1,193 @@
+import { describe, expect, it } from "bun:test";
+import type { AxiosError } from "axios";
+import { buildAxiosErrorLog, redactSensitive } from "./logger";
+
+describe("redactSensitive", () => {
+   it("masks every known credential field at the top level", () => {
+      const input = {
+         password: "hunter2",
+         connectionString: "postgresql://u:p@h/db",
+         serviceAccountKeyJson: '{"private_key":"-----BEGIN..."}',
+         privateKey: "-----BEGIN PRIVATE KEY-----",
+         privateKeyPass: "passphrase",
+         secret: "gcs-hmac-secret",
+         secretAccessKey: "aws-secret",
+         sessionToken: "aws-session",
+         sasUrl: "https://a.blob.core.windows.net/c?sig=abc",
+         clientSecret: "azure-client-secret",
+         oauthClientSecret: "databricks-oauth-secret",
+         peakaKey: "peaka-api-key",
+         token: "databricks-pat",
+         accessToken: "motherduck-token",
+      };
+      const out = redactSensitive(input) as Record<string, unknown>;
+      for (const key of Object.keys(input)) {
+         expect(out[key]).toBe("[REDACTED]");
+      }
+   });
+
+   it("keeps non-secret fields intact", () => {
+      const input = {
+         name: "bigquery",
+         type: "bigquery",
+         defaultProjectId: "my-project",
+         host: "db.example.com",
+         port: 5432,
+      };
+      expect(redactSensitive(input)).toEqual(input);
+   });
+
+   it("redacts secrets nested inside the environment PATCH payload shape", () => {
+      const input = {
+         name: "demo___malloy-samples",
+         connections: [
+            {
+               name: "bigquery",
+               type: "bigquery",
+               bigqueryConnection: {
+                  defaultProjectId: "vscode-demo",
+                  serviceAccountKeyJson: '{"private_key":"-----BEGIN..."}',
+               },
+            },
+            {
+               name: "warehouse",
+               type: "snowflake",
+               snowflakeConnection: {
+                  account: "acct",
+                  username: "svc",
+                  password: "s3cret",
+                  privateKey: "-----BEGIN PRIVATE KEY-----",
+               },
+            },
+         ],
+      };
+      expect(redactSensitive(input)).toEqual({
+         name: "demo___malloy-samples",
+         connections: [
+            {
+               name: "bigquery",
+               type: "bigquery",
+               bigqueryConnection: {
+                  defaultProjectId: "vscode-demo",
+                  serviceAccountKeyJson: "[REDACTED]",
+               },
+            },
+            {
+               name: "warehouse",
+               type: "snowflake",
+               snowflakeConnection: {
+                  account: "acct",
+                  username: "svc",
+                  password: "[REDACTED]",
+                  privateKey: "[REDACTED]",
+               },
+            },
+         ],
+      });
+   });
+
+   it("recurses through arrays of secrets", () => {
+      const out = redactSensitive([{ password: "a" }, { password: "b" }]);
+      expect(out).toEqual([
+         { password: "[REDACTED]" },
+         { password: "[REDACTED]" },
+      ]);
+   });
+
+   it("passes primitives through unchanged", () => {
+      expect(redactSensitive("plain")).toBe("plain");
+      expect(redactSensitive(42)).toBe(42);
+      expect(redactSensitive(null)).toBeNull();
+      expect(redactSensitive(undefined)).toBeUndefined();
+   });
+
+   it("preserves Date values instead of destructuring them", () => {
+      const d = new Date("2026-07-08T00:00:00.000Z");
+      const out = redactSensitive({ createdAt: d }) as { createdAt: Date };
+      expect(out.createdAt).toBeInstanceOf(Date);
+      expect(out.createdAt.getTime()).toBe(d.getTime());
+   });
+
+   it("does not mutate the input object", () => {
+      const input = { password: "hunter2", nested: { token: "abc" } };
+      redactSensitive(input);
+      expect(input.password).toBe("hunter2");
+      expect(input.nested.token).toBe("abc");
+   });
+
+   it("tolerates circular references", () => {
+      const input: Record<string, unknown> = { password: "hunter2" };
+      input.self = input;
+      const out = redactSensitive(input) as Record<string, unknown>;
+      expect(out.password).toBe("[REDACTED]");
+      expect(out.self).toBe("[Circular]");
+   });
+
+   it("matches credential keys case-insensitively", () => {
+      const out = redactSensitive({
+         Password: "hunter2",
+         Authorization: "Bearer abc",
+      }) as Record<string, unknown>;
+      expect(out.Password).toBe("[REDACTED]");
+      expect(out.Authorization).toBe("[REDACTED]");
+   });
+
+   it("masks credential HTTP headers", () => {
+      const out = redactSensitive({
+         "content-type": "application/json",
+         authorization: "Bearer secret-token",
+         cookie: "session=abc",
+      }) as Record<string, unknown>;
+      expect(out["content-type"]).toBe("application/json");
+      expect(out.authorization).toBe("[REDACTED]");
+      expect(out.cookie).toBe("[REDACTED]");
+   });
+});
+
+describe("buildAxiosErrorLog", () => {
+   it("redacts response data and headers on a server-side error", () => {
+      const error = {
+         response: {
+            config: { url: "http://worker/api" },
+            status: 500,
+            headers: { authorization: "Bearer abc", "content-type": "json" },
+            data: { connection: { serviceAccountKeyJson: "PRIVATE_KEY" } },
+         },
+      } as unknown as AxiosError;
+      const { message, meta } = buildAxiosErrorLog(error);
+      expect(message).toBe("Axios server-side error");
+      expect(meta).toEqual({
+         url: "http://worker/api",
+         status: 500,
+         headers: { authorization: "[REDACTED]", "content-type": "json" },
+         data: { connection: { serviceAccountKeyJson: "[REDACTED]" } },
+      });
+   });
+
+   it("never logs the raw request object on a client-side error", () => {
+      const error = {
+         request: { _header: "GET / HTTP/1.1\r\nAuthorization: Bearer abc" },
+         config: { method: "get", url: "http://worker/api" },
+         code: "ECONNABORTED",
+      } as unknown as AxiosError;
+      const { message, meta } = buildAxiosErrorLog(error);
+      expect(message).toBe("Axios client-side error");
+      expect(meta).toEqual({
+         method: "get",
+         url: "http://worker/api",
+         code: "ECONNABORTED",
+      });
+      expect(JSON.stringify(meta)).not.toContain("Bearer abc");
+   });
+
+   it("logs only the message on a setup error", () => {
+      const error = {
+         message: "getaddrinfo ENOTFOUND",
+         config: { headers: { Authorization: "Bearer abc" } },
+      } as unknown as AxiosError;
+      const { message, meta } = buildAxiosErrorLog(error);
+      expect(message).toBe("Axios unknown error");
+      expect(meta).toEqual({ message: "getaddrinfo ENOTFOUND" });
+      expect(JSON.stringify(meta)).not.toContain("Bearer abc");
+   });
+});