@malloy-publisher/server 0.0.232 → 0.0.234

package/src/service/package_manifest.spec.ts

@@ -1,10 +1,63 @@
 import { describe, expect, it } from "bun:test";
 import {
+   packageMaterializationWarnings,
    parsePackageMaterialization,
    parsePackageScope,
+   resolvePackageScope,
 } from "./package_manifest";
 
 describe("service/package_manifest", () => {
+   describe("resolvePackageScope", () => {
+      it("reads the canonical materialization.scope with no warning", () => {
+         expect(resolvePackageScope(undefined, { scope: "version" })).toEqual({
+            scope: "version",
+            warnings: [],
+         });
+      });
+
+      it("defaults to 'package' when neither home declares it", () => {
+         expect(resolvePackageScope(undefined, undefined)).toEqual({
+            scope: "package",
+            warnings: [],
+         });
+         expect(resolvePackageScope(undefined, { freshness: {} })).toEqual({
+            scope: "package",
+            warnings: [],
+         });
+      });
+
+      it("still honors a root scope, with a deprecation warning", () => {
+         const resolved = resolvePackageScope("version", undefined);
+         expect(resolved.scope).toBe("version");
+         expect(resolved.warnings).toHaveLength(1);
+         expect(resolved.warnings[0]).toMatch(/deprecated/i);
+      });
+
+      it("accepts both homes agreeing without a warning", () => {
+         // The transition state the server writes: envelope for this build, root
+         // for an older publisher. Warning would blame the operator for it.
+         expect(resolvePackageScope("version", { scope: "version" })).toEqual({
+            scope: "version",
+            warnings: [],
+         });
+      });
+
+      it("throws when the two homes disagree rather than guessing", () => {
+         expect(() =>
+            resolvePackageScope("package", { scope: "version" }),
+         ).toThrow(/conflicting/i);
+      });
+
+      it("reports an invalid value as invalid, not as a conflict", () => {
+         expect(() =>
+            resolvePackageScope(undefined, { scope: "shared" }),
+         ).toThrow(/expected "version" or "package"/i);
+         expect(() =>
+            resolvePackageScope("shared", { scope: "version" }),
+         ).toThrow(/expected "version" or "package"/i);
+      });
+   });
+
    describe("parsePackageScope", () => {
       it("defaults to 'package' when absent", () => {
          expect(parsePackageScope(undefined)).toBe("package");
@@ -26,7 +79,7 @@ describe("service/package_manifest", () => {
    describe("parsePackageMaterialization", () => {
       it("extracts a string schedule", () => {
          expect(parsePackageMaterialization({ schedule: "0 6 * * *" })).toEqual(
-            { schedule: "0 6 * * *", freshness: null },
+            { schedule: "0 6 * * *", freshness: null, queryMetadata: null },
          );
       });
 
@@ -41,17 +94,23 @@ describe("service/package_manifest", () => {
                schedule: "*/15 * * * *",
                retries: 3,
             }),
-         ).toEqual({ schedule: "*/15 * * * *", freshness: null });
+         ).toEqual({
+            schedule: "*/15 * * * *",
+            freshness: null,
+            queryMetadata: null,
+         });
       });
 
       it("degrades a non-string schedule to null", () => {
          expect(parsePackageMaterialization({ schedule: 42 })).toEqual({
             schedule: null,
             freshness: null,
+            queryMetadata: null,
          });
          expect(parsePackageMaterialization({})).toEqual({
             schedule: null,
             freshness: null,
+            queryMetadata: null,
          });
       });
 
@@ -68,13 +127,18 @@ describe("service/package_manifest", () => {
          ).toEqual({
             schedule: null,
             freshness: { window: "24h", fallback: "stale_ok" },
+            queryMetadata: null,
          });
       });
 
       it("keeps a partial freshness block (window only)", () => {
          expect(
             parsePackageMaterialization({ freshness: { window: "1h" } }),
-         ).toEqual({ schedule: null, freshness: { window: "1h" } });
+         ).toEqual({
+            schedule: null,
+            freshness: { window: "1h" },
+            queryMetadata: null,
+         });
       });
 
       it("drops invalid freshness fields rather than defaulting them", () => {
@@ -84,14 +148,83 @@ describe("service/package_manifest", () => {
             parsePackageMaterialization({
                freshness: { window: 24, fallback: "retry" },
             }),
-         ).toEqual({ schedule: null, freshness: {} });
+         ).toEqual({ schedule: null, freshness: {}, queryMetadata: null });
       });
 
       it("degrades a non-object freshness to null", () => {
          expect(parsePackageMaterialization({ freshness: "24h" })).toEqual({
             schedule: null,
             freshness: null,
+            queryMetadata: null,
+         });
+      });
+
+      it("keeps queryMetadata properties verbatim", () => {
+         expect(
+            parsePackageMaterialization({
+               queryMetadata: { team: "finance", workload: "marts" },
+            }),
+         ).toEqual({
+            schedule: null,
+            freshness: null,
+            queryMetadata: { team: "finance", workload: "marts" },
+         });
+      });
+
+      it("keeps a contract-violating property so publish can report it", () => {
+         // Filtering here would make the author's typo disappear with nothing
+         // to point at; the validator warns and the runtime clamps instead.
+         expect(
+            parsePackageMaterialization({
+               queryMetadata: { "team.name": "finance" },
+            }),
+         ).toEqual({
+            schedule: null,
+            freshness: null,
+            queryMetadata: { "team.name": "finance" },
+         });
+      });
+
+      it("drops non-string values and empties to null", () => {
+         expect(
+            parsePackageMaterialization({
+               queryMetadata: { team: "finance", retries: 3 },
+            }),
+         ).toEqual({
+            schedule: null,
+            freshness: null,
+            queryMetadata: { team: "finance" },
          });
+         expect(
+            parsePackageMaterialization({ queryMetadata: { retries: 3 } }),
+         ).toEqual({ schedule: null, freshness: null, queryMetadata: null });
+         expect(
+            parsePackageMaterialization({ queryMetadata: "team=finance" }),
+         ).toEqual({ schedule: null, freshness: null, queryMetadata: null });
+      });
+   });
+
+   describe("packageMaterializationWarnings", () => {
+      it("names a dropped non-string value", () => {
+         // The drop happens before the config validation that would otherwise
+         // report it, so without this an unquoted `"team": 123` — a plausible
+         // hand-edit — vanishes with nothing anywhere to point at.
+         const warnings = packageMaterializationWarnings({
+            queryMetadata: { team: "finance", retries: 3 },
+         });
+         expect(warnings).toHaveLength(1);
+         expect(warnings[0]).toContain("'retries'");
+         expect(warnings[0]).toContain("got number");
+      });
+
+      it("says nothing about a block that parses cleanly", () => {
+         expect(
+            packageMaterializationWarnings({
+               schedule: "0 6 * * *",
+               queryMetadata: { team: "finance" },
+            }),
+         ).toEqual([]);
+         expect(packageMaterializationWarnings(undefined)).toEqual([]);
       });
    });
 });

package/src/service/package_manifest.ts

@@ -18,10 +18,10 @@ export const PACKAGE_SCOPES = ["version", "package"] as const;
 export type PackageScope = (typeof PACKAGE_SCOPES)[number];
 
 /**
- * Read the manifest root's `scope`, defaulting to `"package"` when absent or
- * null. Any other value is a manifest error: scope is load-bearing (it decides
- * version-owned vs cross-version reuse), so a typo must fail loudly rather than
- * silently pick a default. Throws on an invalid value.
+ * Read a `scope` value, defaulting to `"package"` when absent or null. Any other
+ * value is a manifest error: scope is load-bearing (it decides version-owned vs
+ * cross-version reuse), so a typo must fail loudly rather than silently pick a
+ * default. Throws on an invalid value.
  */
 export function parsePackageScope(raw: unknown): PackageScope {
    if (raw === undefined || raw === null) {
@@ -36,6 +36,69 @@ export function parsePackageScope(raw: unknown): PackageScope {
    );
 }
 
+/**
+ * Resolve the package's scope from its two possible homes: `materialization.scope`
+ * (canonical — every other build-behavioral knob lives in that block) and the
+ * manifest root (the original home, deprecated).
+ *
+ * The root form keeps working because scope rides the published artifact: a
+ * package published before the move must keep parsing until it is republished.
+ * Declaring DIFFERENT values in the two homes throws, because scope decides
+ * whether an artifact is version-owned and guessing which one the author meant
+ * could reuse a table across versions that was never supposed to be shared.
+ *
+ * Only a root-ONLY declaration is deprecated. Both homes agreeing is the
+ * transition state the server itself writes (see `writePackageManifest`): the
+ * envelope for this build, the root for an older publisher that would otherwise
+ * silently default to `package`. Warning about that would be warning an operator
+ * about something the server did on their behalf and they cannot fix.
+ */
+export function resolvePackageScope(
+   rootRaw: unknown,
+   materializationRaw: unknown,
+): { scope: PackageScope; warnings: string[] } {
+   const envelopeRaw =
+      materializationRaw && typeof materializationRaw === "object"
+         ? (materializationRaw as { scope?: unknown }).scope
+         : undefined;
+   const rootDeclared = rootRaw !== undefined && rootRaw !== null;
+   const envelopeDeclared = envelopeRaw !== undefined && envelopeRaw !== null;
+
+   // Validate both homes before comparing, so a typo is reported as a typo
+   // rather than as a conflict.
+   const rootScope = parsePackageScope(rootRaw);
+   const envelopeScope = parsePackageScope(envelopeRaw);
+
+   if (rootDeclared && envelopeDeclared) {
+      if (rootScope !== envelopeScope) {
+         // Names the fix, because this throw fails the package LOAD: the
+         // package is skipped and its only trace is /status loadErrors, so the
+         // message is the whole diagnosis. Guessing instead would be worse —
+         // picking the wrong one reuses a table across versions that was never
+         // meant to be shared, silently.
+         throw new Error(
+            `Conflicting "scope" in publisher.json: root "${rootScope}" vs ` +
+               `"materialization.scope" "${envelopeScope}". The package cannot ` +
+               `load until they agree. Edit "materialization": { "scope": ... } ` +
+               `to the value you want and delete the root-level "scope" (the ` +
+               `server rewrites both homes on its next manifest write).`,
+         );
+      }
+      return { scope: envelopeScope, warnings: [] };
+   }
+   if (rootDeclared) {
+      return { scope: rootScope, warnings: [SCOPE_ROOT_DEPRECATION] };
+   }
+   return { scope: envelopeScope, warnings: [] };
+}
+
+const SCOPE_ROOT_DEPRECATION =
+   `"scope" at the manifest root is deprecated: declare it as ` +
+   `"materialization": { "scope": ... } alongside the other build knobs. The ` +
+   `root form still works and will be removed in a future release; until then ` +
+   `the server keeps both homes in sync when it writes the manifest, so an ` +
+   `older publisher still reads the right value.`;
+
 /**
  * The manifest's `materialization.freshness` block, surfaced verbatim for the
  * control plane (which owns the scheduling and query-time gating logic).
@@ -63,6 +126,18 @@ export interface PackageMaterializationConfig {
     * "no policy" from "policy with no valid fields".
     */
    freshness: PackageFreshnessConfig | null;
+   /**
+    * Package-level per-query metadata: the least specific model-side layer,
+    * overridden per property by a model-file, source or per-request declaration.
+    * Null when the manifest declares none.
+    *
+    * Kept VERBATIM (every string-valued property, conforming or not) rather than
+    * filtered to what Malloy will accept. A property that violates the contract
+    * has to be visible somewhere to be fixable: publish reports it as a warning
+    * from this block, and the runtime clamps it with a metric. Filtering here
+    * would make an author's typo disappear with nothing to point at.
+    */
+   queryMetadata: Record<string, string> | null;
 }
 
 function parseFreshness(raw: unknown): PackageFreshnessConfig | null {
@@ -83,11 +158,55 @@ function parseFreshness(raw: unknown): PackageFreshnessConfig | null {
    return freshness;
 }
 
+/**
+ * The `materialization.queryMetadata` block: string-valued properties verbatim.
+ * A non-object block, or one with no string values, degrades to null so absence
+ * on the wire always means "declared nothing usable". A non-string value is
+ * dropped here because it cannot round-trip as a property at all; contract
+ * violations of the string values are deliberately NOT dropped (see
+ * {@link PackageMaterializationConfig.queryMetadata}).
+ *
+ * Each dropped value is reported, because the drop happens BEFORE the config
+ * validation that would otherwise name it: an unquoted `"team": 123` is a
+ * plausible hand-edit, and without this it disappears with nothing anywhere to
+ * point at.
+ */
+function parseQueryMetadata(raw: unknown): {
+   metadata: Record<string, string> | null;
+   warnings: string[];
+} {
+   if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
+      return { metadata: null, warnings: [] };
+   }
+   const out: Record<string, string> = {};
+   const warnings: string[] = [];
+   for (const [name, value] of Object.entries(raw as Record<string, unknown>)) {
+      if (typeof value === "string") {
+         out[name] = value;
+      } else {
+         warnings.push(
+            `materialization.queryMetadata: property '${name}' must be a ` +
+               `string (got ${value === null ? "null" : typeof value}); it is ` +
+               `not attached to any statement`,
+         );
+      }
+   }
+   return {
+      metadata: Object.keys(out).length > 0 ? out : null,
+      warnings,
+   };
+}
+
 /**
  * Read the manifest's `materialization` object, keeping only recognized fields.
  * Returns null when the block is absent so the API field is null rather than an
  * empty object; a non-string schedule degrades to null, and an absent or
  * non-object freshness degrades to null.
+ *
+ * `materialization.scope` is deliberately NOT read here: scope is a package-level
+ * mode rather than a layered knob, and it has two possible homes to reconcile —
+ * see {@link resolvePackageScope}, which owns that read and surfaces the
+ * resolved value on the package itself.
  */
 export function parsePackageMaterialization(
    raw: unknown,
@@ -95,12 +214,28 @@ export function parsePackageMaterialization(
    if (!raw || typeof raw !== "object") {
       return null;
    }
-   const { schedule, freshness } = raw as {
+   const { schedule, freshness, queryMetadata } = raw as {
       schedule?: unknown;
       freshness?: unknown;
+      queryMetadata?: unknown;
    };
    return {
       schedule: typeof schedule === "string" ? schedule : null,
       freshness: parseFreshness(freshness),
+      queryMetadata: parseQueryMetadata(queryMetadata).metadata,
    };
 }
+
+/**
+ * What the `materialization` parse tolerated but could not keep, for the
+ * operator warnings array. Reads the same parse as
+ * {@link parsePackageMaterialization} rather than re-deriving it, so the two
+ * cannot disagree about what was dropped.
+ */
+export function packageMaterializationWarnings(raw: unknown): string[] {
+   if (!raw || typeof raw !== "object") {
+      return [];
+   }
+   const { queryMetadata } = raw as { queryMetadata?: unknown };
+   return parseQueryMetadata(queryMetadata).warnings;
+}

package/src/service/persist_annotation_validation.spec.ts

@@ -66,6 +66,18 @@ describe("assertPersistNamesQuoted", () => {
       ).not.toThrow();
    });
 
+   it("does not mistake a DOTTED key for the name field", () => {
+      // `.` is a word boundary, so a `\b`-anchored pattern reads
+      // `queryMetadata.name=` as the persist name and fails the whole model
+      // load with a 424 about quoting a name the author never declared.
+      expect(() =>
+         assertPersistNamesQuoted(
+            `#@ persist name="bar" queryMetadata.name=finance`,
+            "m.malloy",
+         ),
+      ).not.toThrow();
+   });
+
    it("reports every offending annotation in the message", () => {
       expect(() =>
          assertPersistNamesQuoted(

package/src/service/persist_annotation_validation.ts

@@ -5,11 +5,16 @@ const PERSIST_LINE_PATTERN = /^\s*#@\s+persist\b/;
 /**
  * A `name=` key whose value is NOT immediately opened by a single or double
  * quote — i.e. a bare `name=engaged_events` (whitespace around `=` tolerated).
- * `name="..."` and `name='...'` pass. The `\bname` word boundary requires a
- * standalone key, so neighbours like `tablename=` / `realization_name=` are
- * never mistaken for it.
+ * `name="..."` and `name='...'` pass.
+ *
+ * The lookbehind requires a standalone key: neither an `_`-joined neighbour
+ * (`tablename=`, `realization_name=`) nor a DOTTED one (`queryMetadata.name=`)
+ * is the persist name. A `\b` word boundary would catch the underscore case but
+ * not the dotted one, because `.` is itself a boundary — so a legitimate
+ * `#@ persist queryMetadata.name=finance` would fail the load with a 424 about
+ * quoting a persist name it never declared.
  */
-const UNQUOTED_NAME_PATTERN = /\bname\s*=\s*(?!["'])/;
+const UNQUOTED_NAME_PATTERN = /(?<![.\w])name\s*=\s*(?!["'])/;
 
 /**
  * Reject `#@ persist name=<value>` annotations whose name is unquoted.