@malloy-publisher/server 0.0.233 → 0.0.235

package/src/service/materialization_config_validation.spec.ts

@@ -0,0 +1,99 @@
+import { describe, expect, it } from "bun:test";
+import type { components } from "../api";
+import { materializationConfigWarnings } from "./materialization_config_validation";
+
+const source = (
+   name: string,
+   queryMetadata: Record<string, string> | null,
+): components["schemas"]["PersistSourcePlan"] => ({
+   name,
+   sourceID: `${name}@m`,
+   connectionName: "duckdb",
+   sourceEntityId: "bid",
+   sql: "SELECT 1",
+   columns: [],
+   queryMetadata,
+});
+
+describe("materializationConfigWarnings", () => {
+   it("says nothing about a clean config", () => {
+      expect(
+         materializationConfigWarnings({
+            packageMaterialization: {
+               schedule: null,
+               freshness: null,
+               queryMetadata: { team: "finance" },
+            },
+            sources: [source("orders", { team: "finance" })],
+         }),
+      ).toEqual([]);
+   });
+
+   it("passes manifest deprecations through", () => {
+      expect(
+         materializationConfigWarnings({
+            manifestWarnings: ['"scope" at the manifest root is deprecated'],
+         }),
+      ).toEqual([{ message: '"scope" at the manifest root is deprecated' }]);
+   });
+
+   it("reports a package-level property that violates the contract", () => {
+      const warnings = materializationConfigWarnings({
+         packageMaterialization: {
+            schedule: null,
+            freshness: null,
+            queryMetadata: { "team.name": "finance" },
+         },
+      });
+      expect(warnings).toHaveLength(1);
+      expect(warnings[0].message).toContain("materialization.queryMetadata");
+      expect(warnings[0].message).toContain("team.name");
+      expect(warnings[0].target).toBeUndefined();
+   });
+
+   it("reports the offending property at the source that resolves it", () => {
+      const warnings = materializationConfigWarnings({
+         sources: [source("orders", { "team.name": "finance" })],
+      });
+      expect(warnings).toHaveLength(1);
+      expect(warnings[0].target).toBe("orders");
+      expect(warnings[0].message).toContain("#@ persist queryMetadata");
+   });
+
+   it("reports a BigQuery-dropped name as a warning, not silence", () => {
+      const warnings = materializationConfigWarnings({
+         sources: [source("orders", { _team: "finance" })],
+      });
+      expect(warnings).toHaveLength(1);
+      expect(warnings[0].message).toMatch(/BigQuery/);
+   });
+
+   it("reports the problem at every level that resolves it", () => {
+      // The package declares it and both sources inherit it. Reporting each
+      // level is what lets an author find the one they can edit; identical
+      // messages are collapsed, different levels are not.
+      const warnings = materializationConfigWarnings({
+         packageMaterialization: {
+            schedule: null,
+            freshness: null,
+            queryMetadata: { "team.name": "finance" },
+         },
+         sources: [
+            source("orders", { "team.name": "finance" }),
+            source("returns", { "team.name": "finance" }),
+         ],
+      });
+      expect(warnings).toHaveLength(3);
+      expect(warnings.map((w) => w.target)).toEqual([
+         undefined,
+         "orders",
+         "returns",
+      ]);
+   });
+
+   it("ignores sources with no metadata", () => {
+      expect(
+         materializationConfigWarnings({ sources: [source("orders", null)] }),
+      ).toEqual([]);
+   });
+});

package/src/service/materialization_config_validation.ts

@@ -0,0 +1,120 @@
+/**
+ * Publish-time validation of a package's materialization config.
+ *
+ * One module for the rules so they cannot drift between the levels a knob can be
+ * declared at. Two principles, both learned from shipped bugs:
+ *
+ *  1. **Nothing is silently dropped.** A declaration that will not do what it
+ *     says has to surface somewhere — a publish error, a publish warning, a 400,
+ *     or a metered runtime drop. A `#@ persist` name that was silently ignored
+ *     (source published, never materialized, no error anywhere) is why
+ *     `persist_annotation_validation.ts` exists; the same rule applies here.
+ *  2. **Strict at declaration, lenient at execution.** Publish and the API have a
+ *     human behind them and report problems; the runtime resolve path clamps and
+ *     meters instead, because metadata must never break a query or a build.
+ *
+ * Today this owns the `queryMetadata` rules and the manifest-shape deprecations,
+ * which are ADVISORY — they surface on the package's operator warnings array.
+ * They deliberately do NOT go through `Package.persistencePolicyWarnings`, which
+ * is a publish REJECTION gate (and disarms the scheduler): a deprecated manifest
+ * shape or a mistyped tag must not stop a package from publishing. The
+ * scope/schedule/freshness coherence rules that do reject still live there;
+ * moving them here is a mechanical follow-up with no behavior change.
+ */
+
+import type { components } from "../api";
+import {
+   queryMetadataAdvisoryWarnings,
+   queryMetadataBudgetWarning,
+   queryMetadataViolations,
+   type QueryMetadata,
+} from "./query_metadata";
+
+type WirePackageMaterialization =
+   components["schemas"]["PackageMaterializationConfig"];
+type WirePersistSourcePlan = components["schemas"]["PersistSourcePlan"];
+
+export interface MaterializationConfigInput {
+   /** The package manifest's `materialization` block, as parsed. */
+   packageMaterialization?: WirePackageMaterialization | null;
+   /** The compiled plan's sources, carrying each one's RESOLVED metadata. */
+   sources?: WirePersistSourcePlan[];
+   /**
+    * Deprecations the manifest parse tolerated (e.g. a root-level `scope`), which
+    * a load keeps working but a publish should report.
+    */
+   manifestWarnings?: string[];
+}
+
+/** One finding, in the shape the wire package's operator warnings array uses. */
+export interface MaterializationConfigWarning {
+   /** The persist source the finding belongs to, absent for a package-level one. */
+   target?: string;
+   message: string;
+}
+
+function metadataWarnings(
+   level: string,
+   metadata: QueryMetadata | null | undefined,
+   target?: string,
+): MaterializationConfigWarning[] {
+   if (!metadata) return [];
+   const budget = queryMetadataBudgetWarning(Object.keys(metadata).length);
+   return [
+      ...queryMetadataViolations(metadata),
+      ...queryMetadataAdvisoryWarnings(metadata),
+      ...(budget ? [budget] : []),
+   ].map((message) => ({
+      // The level is in the message because a reader needs to know WHICH
+      // declaration to edit, and an inherited property has more than one.
+      message: `${level}: ${message}`,
+      ...(target ? { target } : {}),
+   }));
+}
+
+/**
+ * Everything wrong or inadvisable about a package's materialization config, as
+ * actionable messages. Warnings rather than errors: a package published before a
+ * rule existed must keep loading, and a bad metadata property degrades
+ * attribution rather than corrupting anything.
+ *
+ * A source's metadata is checked in its RESOLVED form (the effective bag from
+ * package → model-file → `#@ persist`), because that is what will actually be
+ * attached — a property inherited from the manifest is just as broken at the
+ * source as it is at the package, and reporting it at both is how an author finds
+ * the one they can edit.
+ */
+export function materializationConfigWarnings(
+   input: MaterializationConfigInput,
+): MaterializationConfigWarning[] {
+   const warnings: MaterializationConfigWarning[] = (
+      input.manifestWarnings ?? []
+   ).map((message) => ({ message }));
+
+   warnings.push(
+      ...metadataWarnings(
+         "materialization.queryMetadata",
+         input.packageMaterialization?.queryMetadata,
+      ),
+   );
+
+   for (const source of input.sources ?? []) {
+      warnings.push(
+         ...metadataWarnings(
+            `#@ persist queryMetadata`,
+            source.queryMetadata,
+            source.name,
+         ),
+      );
+   }
+
+   // Identical findings collapse (two sources inheriting one bad package
+   // property produce one message each, not one per level per source).
+   const seen = new Set<string>();
+   return warnings.filter((warning) => {
+      const key = `${warning.target ?? ""}\u0000${warning.message}`;
+      if (seen.has(key)) return false;
+      seen.add(key);
+      return true;
+   });
+}

package/src/service/materialization_schedule_surface.spec.ts

@@ -54,6 +54,7 @@ describe("materialization schedule surfacing", () => {
          expect(pkg.getPackageMetadata().materialization).toEqual({
             schedule: "0 6 * * *",
             freshness: null,
+            queryMetadata: null,
          });
       },
       { timeout: 20000 },
@@ -77,6 +78,7 @@ describe("materialization schedule surfacing", () => {
          expect(pkg.getPackageMetadata().materialization).toEqual({
             schedule: null,
             freshness: { window: "24h", fallback: "stale_ok" },
+            queryMetadata: null,
          });
       },
       { timeout: 20000 },
@@ -119,14 +121,136 @@ describe("materialization schedule surfacing", () => {
          expect(updated.materialization).toEqual({
             schedule: "0 6 * * *",
             freshness: null,
+            queryMetadata: null,
          });
 
          const pkg = await env.getPackage("pkg", false);
          expect(pkg.getPackageMetadata().materialization).toEqual({
             schedule: "0 6 * * *",
             freshness: null,
+            queryMetadata: null,
          });
       },
       { timeout: 20000 },
    );
+
+   async function readManifest(): Promise<Record<string, unknown>> {
+      return JSON.parse(
+         await fs.readFile(path.join(envPath, "pkg", "publisher.json"), "utf8"),
+      );
+   }
+
+   it(
+      "writes scope to both homes so the manifest it authors still loads",
+      async () => {
+         // A scope PATCH used to write only the manifest root. With scope also
+         // living in the materialization block, that authored a manifest whose
+         // two homes disagreed — which the loader refuses, so the package
+         // survived the PATCH and then failed on the next restart.
+         const env = await Environment.create("testEnv", envPath, []);
+         await writePackageDir({
+            materialization: {
+               scope: "package",
+               queryMetadata: { team: "fin" },
+            },
+         });
+         await env.addPackage("pkg");
+
+         await env.updatePackage("pkg", { name: "pkg", scope: "version" });
+
+         const manifest = await readManifest();
+         expect(manifest.scope).toBe("version");
+         expect(manifest.materialization).toMatchObject({
+            scope: "version",
+            queryMetadata: { team: "fin" },
+         });
+
+         // The real assertion: it loads again.
+         const reloaded = await Environment.create("testEnv", envPath, []);
+         await reloaded.addPackage("pkg");
+         const pkg = await reloaded.getPackage("pkg", false);
+         expect(pkg.getPackageMetadata().scope).toBe("version");
+      },
+      { timeout: 20000 },
+   );
+
+   it(
+      "keeps the envelope scope a materialization PATCH cannot express",
+      async () => {
+         // The wire materialization block has no `scope`, so a PATCH that sends
+         // one must not be read as "the author dropped it".
+         const env = await Environment.create("testEnv", envPath, []);
+         await writePackageDir({ materialization: { scope: "version" } });
+         await env.addPackage("pkg");
+
+         await env.updatePackage("pkg", {
+            name: "pkg",
+            materialization: { schedule: "0 6 * * *" },
+         });
+
+         const manifest = await readManifest();
+         expect(manifest.materialization).toMatchObject({
+            scope: "version",
+            schedule: "0 6 * * *",
+         });
+         expect(manifest.scope).toBe("version");
+      },
+      { timeout: 20000 },
+   );
+
+   it(
+      "keeps queryMetadata a schedule PATCH said nothing about",
+      async () => {
+         // The block is replaced wholesale, which is right for the policy the
+         // caller is setting — but queryMetadata is orthogonal to it, so a
+         // client that sets a schedule without re-sending the tags would
+         // silently untag every statement the package's builds issue. The UI
+         // and the control plane are separate clients; preserving it here fixes
+         // all of them at once.
+         const env = await Environment.create("testEnv", envPath, []);
+         await writePackageDir({
+            materialization: {
+               scope: "version",
+               queryMetadata: { team: "finance" },
+            },
+         });
+         await env.addPackage("pkg");
+
+         await env.updatePackage("pkg", {
+            name: "pkg",
+            materialization: { schedule: "0 6 * * *" },
+         });
+
+         expect((await readManifest()).materialization).toMatchObject({
+            schedule: "0 6 * * *",
+            queryMetadata: { team: "finance" },
+         });
+      },
+      { timeout: 20000 },
+   );
+
+   it(
+      "still lets an explicit null clear queryMetadata",
+      async () => {
+         // Preserved-on-omission must not make it unclearable.
+         const env = await Environment.create("testEnv", envPath, []);
+         await writePackageDir({
+            materialization: {
+               scope: "version",
+               queryMetadata: { team: "finance" },
+            },
+         });
+         await env.addPackage("pkg");
+
+         await env.updatePackage("pkg", {
+            name: "pkg",
+            materialization: { schedule: "0 6 * * *", queryMetadata: null },
+         });
+
+         expect(
+            (await readManifest()).materialization as Record<string, unknown>,
+         ).toMatchObject({ schedule: "0 6 * * *", queryMetadata: null });
+      },
+      { timeout: 20000 },
+   );
 });

package/src/service/materialization_service.spec.ts

@@ -39,6 +39,23 @@ import {
 
 type MockRepo = sinon.SinonStubbedInstance<ResourceRepository>;
 
+/**
+ * Turn per-query metadata on for one test and put the environment back after.
+ * The feature ships dark, so a test that asserts what a statement carries has
+ * to enable it — and saying so per test keeps the default honest here: a test
+ * that forgets is testing the shipped default, which is "nothing attached".
+ */
+function withQueryMetadataOn(): void {
+   const original = process.env.PUBLISHER_QUERY_METADATA;
+   beforeEach(() => {
+      process.env.PUBLISHER_QUERY_METADATA = "on";
+   });
+   afterEach(() => {
+      if (original === undefined) delete process.env.PUBLISHER_QUERY_METADATA;
+      else process.env.PUBLISHER_QUERY_METADATA = original;
+   });
+}
+
 describe("redactConnectionSecrets", () => {
    it("strips credential values but keeps the message legible", () => {
       const source = {
@@ -877,6 +894,7 @@ describe("MaterializationService", () => {
 describe("deleteMaterialization telemetry", () => {
    let ctx: ReturnType<typeof createMocks>;
    let harness: MetricsHarness;
+   withQueryMetadataOn();
 
    beforeEach(async () => {
       ctx = createMocks();
@@ -936,6 +954,27 @@ describe("deleteMaterialization telemetry", () => {
       ).toBe(0);
    });
 
+   it("tags the retiring drops as ops work", async () => {
+      // A drop is warehouse work someone accounts for later, and it is the one
+      // statement an operator goes looking for after a table disappears.
+      const runSQL = sinon.stub().resolves();
+      dropTablesFixture(runSQL);
+
+      await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
+         dropTables: true,
+      });
+
+      expect(runSQL.callCount).toBeGreaterThan(0);
+      for (const call of runSQL.getCalls()) {
+         expect(call.args[1]?.queryMetadata).toMatchObject({
+            class: "ops",
+            environment: "my-env",
+            package: "pkg",
+            run_id: "mat-1",
+         });
+      }
+   });
+
    it("meters a failed drop as outcome=failure and still deletes the record", async () => {
       dropTablesFixture(sinon.stub().rejects(new Error("drop boom")));
 
@@ -1472,6 +1511,7 @@ describe("executeInstructedBuild", () => {
 
 describe("buildOneSource", () => {
    let ctx: ReturnType<typeof createMocks>;
+   withQueryMetadataOn();
    beforeEach(() => {
       ctx = createMocks();
    });
@@ -1481,12 +1521,19 @@ describe("buildOneSource", () => {
       physicalTableName: string,
       dialectName?: string,
       manifest: Manifest = new Manifest(),
+      metadata?: {
+         source?: Record<string, string>;
+         package?: Record<string, string>;
+         connection?: Record<string, string>;
+         context?: Record<string, string | undefined>;
+      },
    ): Promise<{ sourceEntityId: string; physicalTableName: string }> {
       const source = fakeSource({
          name: "orders",
          sourceEntityId: "abcdef1234567890",
          sql: "SELECT * FROM t",
          dialectName,
+         queryMetadata: metadata?.source,
       });
       const instruction: BuildInstruction = {
          sourceEntityId: "abcdef1234567890",
@@ -1504,6 +1551,7 @@ describe("buildOneSource", () => {
                m: Manifest,
                env: unknown,
                built: Record<string, unknown>,
+               buildMetadata?: unknown,
             ) => Promise<{ sourceEntityId: string; physicalTableName: string }>;
          }
       ).buildOneSource(
@@ -1514,14 +1562,85 @@ describe("buildOneSource", () => {
          manifest,
          {
             getApiConnection: () => {
+               if (metadata?.connection) {
+                  return { queryMetadata: metadata.connection };
+               }
                throw new Error("no connection config in this colocated test");
             },
             getEnvironmentPath: () => "/tmp/env",
          },
          {}, // builtEntries — colocated build with no prior storage upstreams
+         metadata
+            ? {
+                 packageMaterialization: metadata.package
+                    ? {
+                         schedule: null,
+                         freshness: null,
+                         queryMetadata: metadata.package,
+                      }
+                    : null,
+                 context: metadata.context ?? { queryClass: "materialize" },
+              }
+            : undefined,
       );
    }
 
+   it("tags every statement of the build with the resolved metadata", async () => {
+      // One unit of work: the warehouse's query history has to show the staging
+      // drop, the CTAS, the swap and the rename as the same build.
+      const runSQL = sinon.stub().resolves();
+      await callBuildOneSource({ runSQL }, "orders_v1", undefined, undefined, {
+         connection: { deployment: "eu" },
+         package: { team: "finance", tier: "bronze" },
+         source: { tier: "gold" },
+         context: { queryClass: "materialize", runId: "run-7" },
+      });
+
+      const bags = runSQL.getCalls().map((c) => c.args[1]?.queryMetadata);
+      expect(bags).toHaveLength(4);
+      for (const bag of bags) {
+         expect(bag).toEqual({
+            deployment: "eu",
+            team: "finance",
+            // The source's own declaration wins over the package's.
+            tier: "gold",
+            class: "materialize",
+            run_id: "run-7",
+            source: "orders",
+         });
+      }
+   });
+
+   it("issues untagged statements when nothing declares metadata", async () => {
+      // No metadata context (an internal caller) means the statements are exactly
+      // what they were before this existed.
+      const runSQL = sinon.stub().resolves();
+      await callBuildOneSource({ runSQL }, "orders_v1");
+      for (const call of runSQL.getCalls()) {
+         expect(call.args[1]?.queryMetadata).toBeUndefined();
+      }
+   });
+
+   it("tags the cleanup drop after a failed build", async () => {
+      // The cleanup statement belongs to the same build, and it is the one an
+      // operator goes looking for after a failure.
+      const runSQL = sinon.stub();
+      runSQL.onCall(0).resolves();
+      runSQL.onCall(1).rejects(new Error("create boom"));
+      runSQL.onCall(2).resolves();
+      await expect(
+         callBuildOneSource({ runSQL }, "orders_v1", undefined, undefined, {
+            source: { team: "finance" },
+            context: { queryClass: "materialize" },
+         }),
+      ).rejects.toThrow("create boom");
+      expect(runSQL.lastCall.args[1]?.queryMetadata).toEqual({
+         team: "finance",
+         class: "materialize",
+         source: "orders",
+      });
+   });
+
    it("stages, swaps, and renames in a crash-safe order", async () => {
       const runSQL = sinon.stub().resolves();
       const entry = await callBuildOneSource({ runSQL }, "orders_v1");