@malloy-publisher/server 0.0.233 → 0.0.235

package/src/service/model.ts

@@ -97,11 +97,44 @@ import {
    resolveModelQueryRowLimit,
 } from "./model_limits";
 import { buildSourceAliasMap, extractRunTargetSourceName } from "./query_text";
+import {
+   mergeQueryMetadata,
+   type QueryClass,
+   type QueryMetadata,
+} from "./query_metadata";
 import {
    extractQueriesFromModelDef,
    extractSourcesFromModelDef,
 } from "./source_extraction";
 
+/**
+ * What a request boundary contributes to a model query's per-query metadata: the
+ * caller's own properties and class, the environment the query runs in, and a
+ * reader for the executing connection's default (the controller owns the
+ * environment, so it supplies the lookup rather than the model reaching for it).
+ */
+export interface ModelQueryMetadataInput {
+   request?: QueryMetadata;
+   queryClass?: QueryClass;
+   environment?: string;
+   version?: string;
+   /**
+    * The id this query is correlated by, minted by the boundary that returns it
+    * (see `mintCorrelationId`). Omitted by callers with no response field to
+    * carry it, which is why it is not minted here.
+    */
+   correlationId?: string;
+   /**
+    * The executing connection's two metadata layers: its overridable default and
+    * the properties the deployment enforces. Supplied by the controller, which
+    * owns the environment; the model only knows the connection by name.
+    */
+   connectionMetadata?: (connectionName: string) => {
+      default?: QueryMetadata | null;
+      enforced?: QueryMetadata | null;
+   } | null;
+}
+
 type ApiCompiledModel = components["schemas"]["CompiledModel"];
 type ApiNotebookCell = components["schemas"]["NotebookCell"];
 type ApiRawNotebook = components["schemas"]["RawNotebook"];
@@ -2056,6 +2089,13 @@ export class Model {
       // background callers (materialization, tests) that own their
       // own deadline.
       abortSignal?: AbortSignal,
+      /**
+       * Per-query metadata inputs from the request boundary (see
+       * {@link ModelQueryMetadataInput}). Omitting it still tags the query with
+       * the server context; what is lost is the caller's own properties, the
+       * connection default, and the correlation id.
+       */
+      queryMetadataInput?: ModelQueryMetadataInput,
    ): Promise<{
       result: Malloy.Result;
       compactResult: QueryData;
@@ -2065,6 +2105,12 @@ export class Model {
       rowLimit: number;
       /** Which of those two the cap came from. */
       rowLimitSource: QueryRowLimitSource;
+      /**
+       * The `query_id` property attached to this query's statements, which is the
+       * caller's join key into the backend's own query record. Null when no
+       * metadata was attached.
+       */
+      queryCorrelationId: string | null;
    }> {
       const startTime = performance.now();
       if (this.compilationError) {
@@ -2327,6 +2373,7 @@ export class Model {
       let rowLimitSource: QueryRowLimitSource = "server_default";
       let executionTime = 0;
       let queryResults;
+      let appliedQueryMetadata: QueryMetadata | undefined;
       // Givens supplied only so a joined source's authorize gate could see
       // them (checked above, against the full unfiltered set) must not reach
       // the real query if this model doesn't itself surface them — see
@@ -2339,18 +2386,24 @@ export class Model {
       // shape can read them; the authorize gate above already saw the full set.
       const effectiveGivens = serveVirtualMap ? undefined : querySurfaceGivens;
       try {
-         const preparedLimit = (
-            await runnable.getPreparedResult({
-               givens: effectiveGivens,
-               buildManifest: effectiveBuildManifest,
-               virtualMap: serveVirtualMap,
-            })
-         ).resultExplore.limit;
+         // The prepared result is also where the executing connection's name
+         // comes from, which is what makes the connection's default metadata
+         // layer resolvable BEFORE the statement is issued.
+         const preparedResult = await runnable.getPreparedResult({
+            givens: effectiveGivens,
+            buildManifest: effectiveBuildManifest,
+            virtualMap: serveVirtualMap,
+         });
+         const preparedLimit = preparedResult.resultExplore.limit;
          rowLimitSource = queryRowLimitSource(preparedLimit);
          rowLimit = resolveModelQueryRowLimit(preparedLimit, {
             defaultLimit: getDefaultQueryRowLimit(),
             maxRows,
          });
+         appliedQueryMetadata = this.resolveQueryMetadata(
+            queryMetadataInput,
+            preparedResult.connectionName,
+         );
          executionTime = performance.now() - startTime;
 
          queryResults = await runnable.run({
@@ -2359,6 +2412,7 @@ export class Model {
             abortSignal,
             buildManifest: effectiveBuildManifest,
             virtualMap: serveVirtualMap,
+            queryMetadata: appliedQueryMetadata,
          });
       } catch (error) {
          // A binding that declares `freshnessFallback=live` is saying the tier is
@@ -2480,22 +2534,32 @@ export class Model {
             // the connector reads as a hard cap and stops before the first row: a
             // successful, EMPTY answer. Asking the live shape is also the honest
             // limit, since the live shape is what runs.
-            const livePreparedLimit = (
-               await liveRunnable!.getPreparedResult({
-                  givens: querySurfaceGivens,
-                  buildManifest,
-               })
-            ).resultExplore.limit;
+            const livePrepared = await liveRunnable!.getPreparedResult({
+               givens: querySurfaceGivens,
+               buildManifest,
+            });
+            const livePreparedLimit = livePrepared.resultExplore.limit;
             rowLimitSource = queryRowLimitSource(livePreparedLimit);
             rowLimit = resolveModelQueryRowLimit(livePreparedLimit, {
                defaultLimit: getDefaultQueryRowLimit(),
                maxRows,
             });
+            // Re-resolve rather than reuse: the bag above was resolved against
+            // the STORAGE connection, which on this tier is routinely not the
+            // one the live shape runs on, so reusing it would stamp another
+            // connection's default and enforced layers on this statement. The
+            // correlation id rides in the input, so the retry keeps the id the
+            // response returns — one API call, one join key, two statements.
+            appliedQueryMetadata = this.resolveQueryMetadata(
+               queryMetadataInput,
+               livePrepared.connectionName,
+            );
             queryResults = await liveRunnable!.run({
                rowLimit,
                givens: querySurfaceGivens,
                abortSignal,
                buildManifest,
+               queryMetadata: appliedQueryMetadata,
             });
          } catch (retryError) {
             failQuery(retryError);
@@ -2558,9 +2622,63 @@ export class Model {
          // default. Only the second means rows were probably left behind; a
          // deliberate `top: 10` returning 10 rows is a complete answer.
          rowLimitSource,
+         // The id from the bag that was actually attached, not the one supplied:
+         // a bag shed under budget pressure sheds context last, but a caller
+         // should be told the truth about what it can look up.
+         queryCorrelationId: appliedQueryMetadata?.query_id ?? null,
       };
    }
 
+   /**
+    * The per-query metadata for one model query: the executing connection's
+    * default, the caller's request override, and the server's context (which
+    * package, which model, which class of work), merged most-specific-wins.
+    *
+    * There is no model-side layer here. `materialization.queryMetadata` describes
+    * how a persist source is BUILT; a live query against the model is a different
+    * unit of work, and inheriting a build's tags would attribute interactive
+    * traffic to the build that happens to share the source.
+    *
+    * Fails open, like every other metadata path: a connection whose config can't
+    * be read contributes no default rather than failing the query.
+    */
+   private resolveQueryMetadata(
+      input: ModelQueryMetadataInput | undefined,
+      connectionName: string | undefined,
+   ): QueryMetadata | undefined {
+      let connectionLayers: {
+         default?: QueryMetadata | null;
+         enforced?: QueryMetadata | null;
+      } | null = null;
+      if (connectionName && input?.connectionMetadata) {
+         try {
+            connectionLayers = input.connectionMetadata(connectionName);
+         } catch {
+            connectionLayers = null;
+         }
+      }
+      const resolved = mergeQueryMetadata({
+         connection: connectionLayers?.default,
+         enforced: connectionLayers?.enforced,
+         request: input?.request,
+         context: {
+            queryClass: input?.queryClass ?? "interactive",
+            environment: input?.environment,
+            package: this.packageName,
+            model: this.modelPath,
+            version: input?.version,
+            correlationId: input?.correlationId,
+         },
+      });
+      if (resolved.drops.length > 0) {
+         logger.warn("Dropped query-metadata properties for a query", {
+            modelPath: this.modelPath,
+            drops: resolved.drops,
+         });
+      }
+      return resolved.metadata;
+   }
+
    private getStandardModel(): ApiCompiledModel {
       return {
          type: "source",
@@ -2655,6 +2773,11 @@ export class Model {
       // See `getQueryResults`: forwarded into `runnable.run` so the
       // publisher's wall-clock timeout actually cancels the query.
       abortSignal?: AbortSignal,
+      // A notebook cell issues real backend SQL on an interactive path, so it is
+      // tagged like any other query. No correlation id: the cell response has no
+      // field to hand one back on, and an id nobody can read costs the result
+      // cache for nothing.
+      queryMetadataInput?: ModelQueryMetadataInput,
    ): Promise<{
       type: "code" | "markdown";
       text: string;
@@ -2734,13 +2857,12 @@ export class Model {
             // See getQueryResults / filterGivensToModelSurface: the gate
             // above already saw the full unfiltered givens.
             const cellSurfaceGivens = this.filterGivensToModelSurface(givens);
+            const preparedCell = await runnableToExecute.getPreparedResult({
+               givens: cellSurfaceGivens,
+               buildManifest,
+            });
             const rowLimit = resolveModelQueryRowLimit(
-               (
-                  await runnableToExecute.getPreparedResult({
-                     givens: cellSurfaceGivens,
-                     buildManifest,
-                  })
-               ).resultExplore.limit,
+               preparedCell.resultExplore.limit,
                {
                   defaultLimit: getDefaultQueryRowLimit(),
                   maxRows: cellMaxRows,
@@ -2751,6 +2873,10 @@ export class Model {
                givens: cellSurfaceGivens,
                abortSignal,
                buildManifest,
+               queryMetadata: this.resolveQueryMetadata(
+                  queryMetadataInput,
+                  preparedCell.connectionName,
+               ),
             });
             const query = (await runnableToExecute.getPreparedQuery())._query;
             queryName = (query as NamedQueryDef).as || query.name;

package/src/service/package.ts

@@ -50,6 +50,7 @@ import { errMessage, ignoreDotfiles } from "../utils";
 import { getPersistCollisionEnforce, getPersistStorageMode } from "../config";
 import { deriveServeBindings } from "./materialization_serve_transform";
 import { computePackageBuildPlan, SourceEligibility } from "./build_plan";
+import { materializationConfigWarnings } from "./materialization_config_validation";
 import { CronEvaluator } from "./cron_evaluator";
 import { filterFreshManifest } from "./freshness";
 import { isQuotedIdentifierPath, quoteManifestTablePath } from "./quoting";
@@ -175,6 +176,12 @@ export class Package {
    // tag does not fail the load (see Model.validateRenderTags); this is the
    // response-level signal that a tag is misconfigured.
    private renderTagWarnings: ApiPackageWarning[] = [];
+   /**
+    * Manifest-shape deprecations the load tolerated (a root-level `scope`), kept
+    * so publish can report a still-parsing-but-outdated manifest. Not on the wire
+    * package: it is a property of the manifest text, not of the loaded package.
+    */
+   private manifestWarnings: string[] = [];
    private static meter = publisherMeter();
    private static packageLoadHistogram = this.meter.createHistogram(
       "malloy_package_load_duration",
@@ -550,6 +557,7 @@ export class Package {
          malloyConfig,
       );
       pkg.renderTagWarnings = renderTagWarnings;
+      pkg.manifestWarnings = outcome.packageMetadata.manifestWarnings ?? [];
       // Install the per-query freshness resolver on the freshly-built models.
       // At create time no manifest is bound yet, so the resolver returns
       // undefined (serve live) until a subsequent bindManifest → reloadAllModels.
@@ -688,6 +696,16 @@ export class Package {
          // (alongside the load-path log) so an operator can see it on the status
          // API like the other persist warnings — see persistenceCollisionWarnings.
          ...this.persistenceCollisionWarnings().map((message) => ({ message })),
+         // Materialization-config findings: a queryMetadata property that will
+         // not do what it says, and manifest shapes that still parse but are
+         // deprecated. Advisory by design — none of these blocks a publish.
+         ...materializationConfigWarnings({
+            packageMaterialization: this.packageMetadata.materialization,
+            sources: this.buildPlan?.sources
+               ? Object.values(this.buildPlan.sources)
+               : [],
+            manifestWarnings: this.manifestWarnings,
+         }),
       ];
       if (allWarnings.length > 0) {
          metadata.warnings = allWarnings;
@@ -1059,6 +1077,10 @@ export class Package {
    }
 
    public persistencePolicyWarnings(): string[] {
+      // REJECTION rules only: a non-empty result fails a publish and disarms the
+      // scheduler. Advisory findings (queryMetadata problems, a deprecated
+      // manifest shape) go to the operator warnings array instead — see
+      // materializationConfigWarnings.
       const warnings: string[] = [];
       const sources = this.buildPlan?.sources
          ? Object.values(this.buildPlan.sources)
@@ -1075,7 +1097,7 @@ export class Package {
             warnings.push(
                `#@ persist source "${source.name}" declares sharing=... which is ` +
                   `no longer supported: scope is a single package-level mode. Set ` +
-                  `the root-level "scope": "version" | "package" in ` +
+                  `"materialization": { "scope": "version" | "package" } in ` +
                   `${PACKAGE_MANIFEST_NAME} instead.`,
             );
          }
@@ -1480,6 +1502,7 @@ export class Package {
       // package's current storage= bindings so a reload preserves serve routing.
       this.pushStorageServeBindingsToModels();
       this.renderTagWarnings = renderTagWarnings;
+      this.manifestWarnings = outcome.packageMetadata.manifestWarnings ?? [];
       // A reload re-reads publisher.json in the worker; pick up any change to
       // the explore set and query-boundary mode so listModels()/the gate
       // reflect edited explores without a full Package.create.

package/src/service/package_manifest.spec.ts

@@ -1,10 +1,63 @@
 import { describe, expect, it } from "bun:test";
 import {
+   packageMaterializationWarnings,
    parsePackageMaterialization,
    parsePackageScope,
+   resolvePackageScope,
 } from "./package_manifest";
 
 describe("service/package_manifest", () => {
+   describe("resolvePackageScope", () => {
+      it("reads the canonical materialization.scope with no warning", () => {
+         expect(resolvePackageScope(undefined, { scope: "version" })).toEqual({
+            scope: "version",
+            warnings: [],
+         });
+      });
+
+      it("defaults to 'package' when neither home declares it", () => {
+         expect(resolvePackageScope(undefined, undefined)).toEqual({
+            scope: "package",
+            warnings: [],
+         });
+         expect(resolvePackageScope(undefined, { freshness: {} })).toEqual({
+            scope: "package",
+            warnings: [],
+         });
+      });
+
+      it("still honors a root scope, with a deprecation warning", () => {
+         const resolved = resolvePackageScope("version", undefined);
+         expect(resolved.scope).toBe("version");
+         expect(resolved.warnings).toHaveLength(1);
+         expect(resolved.warnings[0]).toMatch(/deprecated/i);
+      });
+
+      it("accepts both homes agreeing without a warning", () => {
+         // The transition state the server writes: envelope for this build, root
+         // for an older publisher. Warning would blame the operator for it.
+         expect(resolvePackageScope("version", { scope: "version" })).toEqual({
+            scope: "version",
+            warnings: [],
+         });
+      });
+
+      it("throws when the two homes disagree rather than guessing", () => {
+         expect(() =>
+            resolvePackageScope("package", { scope: "version" }),
+         ).toThrow(/conflicting/i);
+      });
+
+      it("reports an invalid value as invalid, not as a conflict", () => {
+         expect(() =>
+            resolvePackageScope(undefined, { scope: "shared" }),
+         ).toThrow(/expected "version" or "package"/i);
+         expect(() =>
+            resolvePackageScope("shared", { scope: "version" }),
+         ).toThrow(/expected "version" or "package"/i);
+      });
+   });
+
    describe("parsePackageScope", () => {
       it("defaults to 'package' when absent", () => {
          expect(parsePackageScope(undefined)).toBe("package");
@@ -26,7 +79,7 @@ describe("service/package_manifest", () => {
    describe("parsePackageMaterialization", () => {
       it("extracts a string schedule", () => {
          expect(parsePackageMaterialization({ schedule: "0 6 * * *" })).toEqual(
-            { schedule: "0 6 * * *", freshness: null },
+            { schedule: "0 6 * * *", freshness: null, queryMetadata: null },
          );
       });
 
@@ -41,17 +94,23 @@ describe("service/package_manifest", () => {
                schedule: "*/15 * * * *",
                retries: 3,
             }),
-         ).toEqual({ schedule: "*/15 * * * *", freshness: null });
+         ).toEqual({
+            schedule: "*/15 * * * *",
+            freshness: null,
+            queryMetadata: null,
+         });
       });
 
       it("degrades a non-string schedule to null", () => {
          expect(parsePackageMaterialization({ schedule: 42 })).toEqual({
             schedule: null,
             freshness: null,
+            queryMetadata: null,
          });
          expect(parsePackageMaterialization({})).toEqual({
             schedule: null,
             freshness: null,
+            queryMetadata: null,
          });
       });
 
@@ -68,13 +127,18 @@ describe("service/package_manifest", () => {
          ).toEqual({
             schedule: null,
             freshness: { window: "24h", fallback: "stale_ok" },
+            queryMetadata: null,
          });
       });
 
       it("keeps a partial freshness block (window only)", () => {
          expect(
             parsePackageMaterialization({ freshness: { window: "1h" } }),
-         ).toEqual({ schedule: null, freshness: { window: "1h" } });
+         ).toEqual({
+            schedule: null,
+            freshness: { window: "1h" },
+            queryMetadata: null,
+         });
       });
 
       it("drops invalid freshness fields rather than defaulting them", () => {
@@ -84,14 +148,83 @@ describe("service/package_manifest", () => {
             parsePackageMaterialization({
                freshness: { window: 24, fallback: "retry" },
             }),
-         ).toEqual({ schedule: null, freshness: {} });
+         ).toEqual({ schedule: null, freshness: {}, queryMetadata: null });
       });
 
       it("degrades a non-object freshness to null", () => {
          expect(parsePackageMaterialization({ freshness: "24h" })).toEqual({
             schedule: null,
             freshness: null,
+            queryMetadata: null,
+         });
+      });
+
+      it("keeps queryMetadata properties verbatim", () => {
+         expect(
+            parsePackageMaterialization({
+               queryMetadata: { team: "finance", workload: "marts" },
+            }),
+         ).toEqual({
+            schedule: null,
+            freshness: null,
+            queryMetadata: { team: "finance", workload: "marts" },
+         });
+      });
+
+      it("keeps a contract-violating property so publish can report it", () => {
+         // Filtering here would make the author's typo disappear with nothing
+         // to point at; the validator warns and the runtime clamps instead.
+         expect(
+            parsePackageMaterialization({
+               queryMetadata: { "team.name": "finance" },
+            }),
+         ).toEqual({
+            schedule: null,
+            freshness: null,
+            queryMetadata: { "team.name": "finance" },
+         });
+      });
+
+      it("drops non-string values and empties to null", () => {
+         expect(
+            parsePackageMaterialization({
+               queryMetadata: { team: "finance", retries: 3 },
+            }),
+         ).toEqual({
+            schedule: null,
+            freshness: null,
+            queryMetadata: { team: "finance" },
          });
+         expect(
+            parsePackageMaterialization({ queryMetadata: { retries: 3 } }),
+         ).toEqual({ schedule: null, freshness: null, queryMetadata: null });
+         expect(
+            parsePackageMaterialization({ queryMetadata: "team=finance" }),
+         ).toEqual({ schedule: null, freshness: null, queryMetadata: null });
+      });
+   });
+
+   describe("packageMaterializationWarnings", () => {
+      it("names a dropped non-string value", () => {
+         // The drop happens before the config validation that would otherwise
+         // report it, so without this an unquoted `"team": 123` — a plausible
+         // hand-edit — vanishes with nothing anywhere to point at.
+         const warnings = packageMaterializationWarnings({
+            queryMetadata: { team: "finance", retries: 3 },
+         });
+         expect(warnings).toHaveLength(1);
+         expect(warnings[0]).toContain("'retries'");
+         expect(warnings[0]).toContain("got number");
+      });
+
+      it("says nothing about a block that parses cleanly", () => {
+         expect(
+            packageMaterializationWarnings({
+               schedule: "0 6 * * *",
+               queryMetadata: { team: "finance" },
+            }),
+         ).toEqual([]);
+         expect(packageMaterializationWarnings(undefined)).toEqual([]);
       });
    });
 });