@malloy-publisher/server 0.0.232 → 0.0.234

package/src/service/model.spec.ts

@@ -895,12 +895,14 @@ describe("service/model", () => {
    describe("runtime storage failure → freshnessFallback=live", () => {
       const originalMode = process.env.PERSIST_STORAGE_MODE;
       const originalDefaultRows = process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT;
+      const originalMetadata = process.env.PUBLISHER_QUERY_METADATA;
 
       afterEach(() => {
          sinon.restore();
          for (const [name, original] of [
             ["PERSIST_STORAGE_MODE", originalMode],
             ["PUBLISHER_DEFAULT_QUERY_ROW_LIMIT", originalDefaultRows],
+            ["PUBLISHER_QUERY_METADATA", originalMetadata],
          ] as const) {
             if (original === undefined) delete process.env[name];
             else process.env[name] = original;
@@ -936,7 +938,10 @@ describe("service/model", () => {
             getPreparedResult:
                opts.storageFailsAt === "prepare"
                   ? sinon.stub().rejects(storageErr)
-                  : sinon.stub().resolves({ resultExplore: { limit: 0 } }),
+                  : sinon.stub().resolves({
+                       resultExplore: { limit: 0 },
+                       connectionName: "lake",
+                    }),
             run:
                opts.storageFailsAt === "run"
                   ? sinon.stub().rejects(storageErr)
@@ -954,6 +959,7 @@ describe("service/model", () => {
          const liveRunnable = {
             getPreparedResult: sinon.stub().resolves({
                resultExplore: { limit: opts.livePreparedLimit ?? 0 },
+               connectionName: "live_pg",
             }),
             run: liveRun,
          };
@@ -1020,6 +1026,44 @@ describe("service/model", () => {
          expect(liveRun.firstCall.args[0].rowLimit).toBe(250);
       });
 
+      it("tags the live retry, with the LIVE connection's layers and the same id", async () => {
+         // Two failures in one: the statement that actually answered the query
+         // carried no attribution at all, so the full-cost fallback — the one an
+         // operator goes looking for on a bill — landed in the untagged bucket;
+         // and the response still returned the id from the bag resolved before
+         // the storage attempt, pointing a caller at the statement that FAILED.
+         // The layers must come from the live connection: on this tier the store
+         // is routinely a different connection, with different enforced
+         // properties.
+         process.env.PUBLISHER_QUERY_METADATA = "on";
+         const { model, liveRun } = routedModel({
+            shapeBindings: [binding("daily", "live")],
+            storageFailsAt: "run",
+         });
+
+         const result = await model.getQueryResults(
+            undefined,
+            undefined,
+            "run: daily -> x",
+            undefined,
+            undefined,
+            undefined,
+            undefined,
+            {
+               correlationId: "corr-1",
+               connectionMetadata: (connectionName: string) =>
+                  connectionName === "live_pg"
+                     ? { default: null, enforced: { tenant: "acme" } }
+                     : { default: null, enforced: { tenant: "wrong" } },
+            },
+         );
+
+         const attached = liveRun.firstCall.args[0].queryMetadata;
+         expect(attached.tenant).toBe("acme");
+         expect(attached.query_id).toBe("corr-1");
+         expect(result.queryCorrelationId).toBe("corr-1");
+      });
+
       it("does not record a live-served answer as a storage hit", async () => {
          // The hit rate is the tier's headline KPI; counting a fallback as a hit
          // makes it RISE while the tier is broken.

package/src/service/model.ts

@@ -92,14 +92,49 @@ import {
 import { malloyGivenToApi, type MalloyGiven } from "./given";
 import {
    assertWithinModelResponseLimits,
+   type QueryRowLimitSource,
+   queryRowLimitSource,
    resolveModelQueryRowLimit,
 } from "./model_limits";
 import { buildSourceAliasMap, extractRunTargetSourceName } from "./query_text";
+import {
+   mergeQueryMetadata,
+   type QueryClass,
+   type QueryMetadata,
+} from "./query_metadata";
 import {
    extractQueriesFromModelDef,
    extractSourcesFromModelDef,
 } from "./source_extraction";
 
+/**
+ * What a request boundary contributes to a model query's per-query metadata: the
+ * caller's own properties and class, the environment the query runs in, and a
+ * reader for the executing connection's default (the controller owns the
+ * environment, so it supplies the lookup rather than the model reaching for it).
+ */
+export interface ModelQueryMetadataInput {
+   request?: QueryMetadata;
+   queryClass?: QueryClass;
+   environment?: string;
+   version?: string;
+   /**
+    * The id this query is correlated by, minted by the boundary that returns it
+    * (see `mintCorrelationId`). Omitted by callers with no response field to
+    * carry it, which is why it is not minted here.
+    */
+   correlationId?: string;
+   /**
+    * The executing connection's two metadata layers: its overridable default and
+    * the properties the deployment enforces. Supplied by the controller, which
+    * owns the environment; the model only knows the connection by name.
+    */
+   connectionMetadata?: (connectionName: string) => {
+      default?: QueryMetadata | null;
+      enforced?: QueryMetadata | null;
+   } | null;
+}
+
 type ApiCompiledModel = components["schemas"]["CompiledModel"];
 type ApiNotebookCell = components["schemas"]["NotebookCell"];
 type ApiRawNotebook = components["schemas"]["RawNotebook"];
@@ -2054,11 +2089,28 @@ export class Model {
       // background callers (materialization, tests) that own their
       // own deadline.
       abortSignal?: AbortSignal,
+      /**
+       * Per-query metadata inputs from the request boundary (see
+       * {@link ModelQueryMetadataInput}). Omitting it still tags the query with
+       * the server context; what is lost is the caller's own properties, the
+       * connection default, and the correlation id.
+       */
+      queryMetadataInput?: ModelQueryMetadataInput,
    ): Promise<{
       result: Malloy.Result;
       compactResult: QueryData;
       modelInfo: Malloy.ModelInfo;
       dataStyles: DataStyles;
+      /** Row cap pushed into the SQL: the query's own LIMIT, else the default. */
+      rowLimit: number;
+      /** Which of those two the cap came from. */
+      rowLimitSource: QueryRowLimitSource;
+      /**
+       * The `query_id` property attached to this query's statements, which is the
+       * caller's join key into the backend's own query record. Null when no
+       * metadata was attached.
+       */
+      queryCorrelationId: string | null;
    }> {
       const startTime = performance.now();
       if (this.compilationError) {
@@ -2318,8 +2370,10 @@ export class Model {
       // a 500. `executionTime` is still captured after prepare and before run,
       // preserving the pre-existing timing recorded by the success histogram.
       let rowLimit = 0;
+      let rowLimitSource: QueryRowLimitSource = "server_default";
       let executionTime = 0;
       let queryResults;
+      let appliedQueryMetadata: QueryMetadata | undefined;
       // Givens supplied only so a joined source's authorize gate could see
       // them (checked above, against the full unfiltered set) must not reach
       // the real query if this model doesn't itself surface them — see
@@ -2332,15 +2386,23 @@ export class Model {
       // shape can read them; the authorize gate above already saw the full set.
       const effectiveGivens = serveVirtualMap ? undefined : querySurfaceGivens;
       try {
-         rowLimit = resolveModelQueryRowLimit(
-            (
-               await runnable.getPreparedResult({
-                  givens: effectiveGivens,
-                  buildManifest: effectiveBuildManifest,
-                  virtualMap: serveVirtualMap,
-               })
-            ).resultExplore.limit,
-            { defaultLimit: getDefaultQueryRowLimit(), maxRows },
+         // The prepared result is also where the executing connection's name
+         // comes from, which is what makes the connection's default metadata
+         // layer resolvable BEFORE the statement is issued.
+         const preparedResult = await runnable.getPreparedResult({
+            givens: effectiveGivens,
+            buildManifest: effectiveBuildManifest,
+            virtualMap: serveVirtualMap,
+         });
+         const preparedLimit = preparedResult.resultExplore.limit;
+         rowLimitSource = queryRowLimitSource(preparedLimit);
+         rowLimit = resolveModelQueryRowLimit(preparedLimit, {
+            defaultLimit: getDefaultQueryRowLimit(),
+            maxRows,
+         });
+         appliedQueryMetadata = this.resolveQueryMetadata(
+            queryMetadataInput,
+            preparedResult.connectionName,
          );
          executionTime = performance.now() - startTime;
 
@@ -2350,6 +2412,7 @@ export class Model {
             abortSignal,
             buildManifest: effectiveBuildManifest,
             virtualMap: serveVirtualMap,
+            queryMetadata: appliedQueryMetadata,
          });
       } catch (error) {
          // A binding that declares `freshnessFallback=live` is saying the tier is
@@ -2471,20 +2534,32 @@ export class Model {
             // the connector reads as a hard cap and stops before the first row: a
             // successful, EMPTY answer. Asking the live shape is also the honest
             // limit, since the live shape is what runs.
-            rowLimit = resolveModelQueryRowLimit(
-               (
-                  await liveRunnable!.getPreparedResult({
-                     givens: querySurfaceGivens,
-                     buildManifest,
-                  })
-               ).resultExplore.limit,
-               { defaultLimit: getDefaultQueryRowLimit(), maxRows },
+            const livePrepared = await liveRunnable!.getPreparedResult({
+               givens: querySurfaceGivens,
+               buildManifest,
+            });
+            const livePreparedLimit = livePrepared.resultExplore.limit;
+            rowLimitSource = queryRowLimitSource(livePreparedLimit);
+            rowLimit = resolveModelQueryRowLimit(livePreparedLimit, {
+               defaultLimit: getDefaultQueryRowLimit(),
+               maxRows,
+            });
+            // Re-resolve rather than reuse: the bag above was resolved against
+            // the STORAGE connection, which on this tier is routinely not the
+            // one the live shape runs on, so reusing it would stamp another
+            // connection's default and enforced layers on this statement. The
+            // correlation id rides in the input, so the retry keeps the id the
+            // response returns — one API call, one join key, two statements.
+            appliedQueryMetadata = this.resolveQueryMetadata(
+               queryMetadataInput,
+               livePrepared.connectionName,
             );
             queryResults = await liveRunnable!.run({
                rowLimit,
                givens: querySurfaceGivens,
                abortSignal,
                buildManifest,
+               queryMetadata: appliedQueryMetadata,
             });
          } catch (retryError) {
             failQuery(retryError);
@@ -2536,9 +2611,74 @@ export class Model {
          compactResult: queryResults.data.value,
          modelInfo: this.modelInfo,
          dataStyles: this.dataStyles,
+         // The cap actually pushed into the SQL. A caller cannot otherwise tell
+         // a complete result from one the row limit cut off: with no LIMIT of
+         // its own a query silently gets DEFAULT_QUERY_ROW_LIMIT rows, and that
+         // is under maxRows, so assertWithinModelResponseLimits raises nothing.
+         // Returning the number lets a caller compare it against the row count
+         // and say so, with no extra query.
+         rowLimit,
+         // Whether that cap was the author's own limit:/top: or the silent
+         // default. Only the second means rows were probably left behind; a
+         // deliberate `top: 10` returning 10 rows is a complete answer.
+         rowLimitSource,
+         // The id from the bag that was actually attached, not the one supplied:
+         // a bag shed under budget pressure sheds context last, but a caller
+         // should be told the truth about what it can look up.
+         queryCorrelationId: appliedQueryMetadata?.query_id ?? null,
       };
    }
 
+   /**
+    * The per-query metadata for one model query: the executing connection's
+    * default, the caller's request override, and the server's context (which
+    * package, which model, which class of work), merged most-specific-wins.
+    *
+    * There is no model-side layer here. `materialization.queryMetadata` describes
+    * how a persist source is BUILT; a live query against the model is a different
+    * unit of work, and inheriting a build's tags would attribute interactive
+    * traffic to the build that happens to share the source.
+    *
+    * Fails open, like every other metadata path: a connection whose config can't
+    * be read contributes no default rather than failing the query.
+    */
+   private resolveQueryMetadata(
+      input: ModelQueryMetadataInput | undefined,
+      connectionName: string | undefined,
+   ): QueryMetadata | undefined {
+      let connectionLayers: {
+         default?: QueryMetadata | null;
+         enforced?: QueryMetadata | null;
+      } | null = null;
+      if (connectionName && input?.connectionMetadata) {
+         try {
+            connectionLayers = input.connectionMetadata(connectionName);
+         } catch {
+            connectionLayers = null;
+         }
+      }
+      const resolved = mergeQueryMetadata({
+         connection: connectionLayers?.default,
+         enforced: connectionLayers?.enforced,
+         request: input?.request,
+         context: {
+            queryClass: input?.queryClass ?? "interactive",
+            environment: input?.environment,
+            package: this.packageName,
+            model: this.modelPath,
+            version: input?.version,
+            correlationId: input?.correlationId,
+         },
+      });
+      if (resolved.drops.length > 0) {
+         logger.warn("Dropped query-metadata properties for a query", {
+            modelPath: this.modelPath,
+            drops: resolved.drops,
+         });
+      }
+      return resolved.metadata;
+   }
+
    private getStandardModel(): ApiCompiledModel {
       return {
          type: "source",
@@ -2633,6 +2773,11 @@ export class Model {
       // See `getQueryResults`: forwarded into `runnable.run` so the
       // publisher's wall-clock timeout actually cancels the query.
       abortSignal?: AbortSignal,
+      // A notebook cell issues real backend SQL on an interactive path, so it is
+      // tagged like any other query. No correlation id: the cell response has no
+      // field to hand one back on, and an id nobody can read costs the result
+      // cache for nothing.
+      queryMetadataInput?: ModelQueryMetadataInput,
    ): Promise<{
       type: "code" | "markdown";
       text: string;
@@ -2712,13 +2857,12 @@ export class Model {
             // See getQueryResults / filterGivensToModelSurface: the gate
             // above already saw the full unfiltered givens.
             const cellSurfaceGivens = this.filterGivensToModelSurface(givens);
+            const preparedCell = await runnableToExecute.getPreparedResult({
+               givens: cellSurfaceGivens,
+               buildManifest,
+            });
             const rowLimit = resolveModelQueryRowLimit(
-               (
-                  await runnableToExecute.getPreparedResult({
-                     givens: cellSurfaceGivens,
-                     buildManifest,
-                  })
-               ).resultExplore.limit,
+               preparedCell.resultExplore.limit,
                {
                   defaultLimit: getDefaultQueryRowLimit(),
                   maxRows: cellMaxRows,
@@ -2729,6 +2873,10 @@ export class Model {
                givens: cellSurfaceGivens,
                abortSignal,
                buildManifest,
+               queryMetadata: this.resolveQueryMetadata(
+                  queryMetadataInput,
+                  preparedCell.connectionName,
+               ),
             });
             const query = (await runnableToExecute.getPreparedQuery())._query;
             queryName = (query as NamedQueryDef).as || query.name;

package/src/service/model_limits.spec.ts

@@ -3,6 +3,7 @@ import { describe, expect, it } from "bun:test";
 import { PayloadTooLargeError } from "../errors";
 import {
    assertWithinModelResponseLimits,
+   queryRowLimitSource,
    resolveModelQueryRowLimit,
 } from "./model_limits";
 
@@ -179,3 +180,30 @@ describe("assertWithinModelResponseLimits", () => {
       ).not.toThrow();
    });
 });
+
+describe("queryRowLimitSource", () => {
+   /**
+    * Must mirror resolveModelQueryRowLimit's own `requested` condition: if the
+    * two ever disagree, the reported source describes a different limit than
+    * the one actually pushed into the SQL.
+    */
+   it("reports the query when it carried its own positive limit", () => {
+      expect(queryRowLimitSource(10)).toBe("query");
+      expect(queryRowLimitSource(1)).toBe("query");
+   });
+
+   it("reports the server default when the query carried none", () => {
+      expect(queryRowLimitSource(undefined)).toBe("server_default");
+      expect(queryRowLimitSource(0)).toBe("server_default");
+      expect(queryRowLimitSource(-1)).toBe("server_default");
+   });
+
+   it("agrees with which limit resolveModelQueryRowLimit actually used", () => {
+      const config = { defaultLimit: 1000, maxRows: 0 };
+      for (const userLimit of [undefined, 0, -1, 1, 10, 5000]) {
+         const used = resolveModelQueryRowLimit(userLimit, config);
+         const cameFromQuery = queryRowLimitSource(userLimit) === "query";
+         expect(cameFromQuery).toBe(used !== config.defaultLimit);
+      }
+   });
+});

package/src/service/model_limits.ts

@@ -67,6 +67,27 @@ export function resolveModelQueryRowLimit(
    return Math.min(requested, maxRows + 1);
 }
 
+/** Whether the cap came from the query itself or from the server default. */
+export type QueryRowLimitSource = "query" | "server_default";
+
+/**
+ * Which of the two the cap in {@link resolveModelQueryRowLimit} came from.
+ *
+ * The distinction is the whole difference between "the database cut this off
+ * and you were not told" and "you asked for exactly this many". A `limit:` or
+ * `top:` the author wrote is deliberate and complete for what it asked; only the
+ * silently-applied default means rows were probably left behind.
+ *
+ * The condition deliberately mirrors `requested` above and must stay in step
+ * with it, so a change to which limit wins cannot leave the reported source
+ * describing the other one.
+ */
+export function queryRowLimitSource(
+   userLimit: number | undefined,
+): QueryRowLimitSource {
+   return userLimit && userLimit > 0 ? "query" : "server_default";
+}
+
 export interface ModelResponseLimitsConfig {
    /** Result of {@link getMaxQueryRows}. `0` disables the row cap. */
    maxRows: number;

package/src/service/package.ts

@@ -50,6 +50,7 @@ import { errMessage, ignoreDotfiles } from "../utils";
 import { getPersistCollisionEnforce, getPersistStorageMode } from "../config";
 import { deriveServeBindings } from "./materialization_serve_transform";
 import { computePackageBuildPlan, SourceEligibility } from "./build_plan";
+import { materializationConfigWarnings } from "./materialization_config_validation";
 import { CronEvaluator } from "./cron_evaluator";
 import { filterFreshManifest } from "./freshness";
 import { isQuotedIdentifierPath, quoteManifestTablePath } from "./quoting";
@@ -175,6 +176,12 @@ export class Package {
    // tag does not fail the load (see Model.validateRenderTags); this is the
    // response-level signal that a tag is misconfigured.
    private renderTagWarnings: ApiPackageWarning[] = [];
+   /**
+    * Manifest-shape deprecations the load tolerated (a root-level `scope`), kept
+    * so publish can report a still-parsing-but-outdated manifest. Not on the wire
+    * package: it is a property of the manifest text, not of the loaded package.
+    */
+   private manifestWarnings: string[] = [];
    private static meter = publisherMeter();
    private static packageLoadHistogram = this.meter.createHistogram(
       "malloy_package_load_duration",
@@ -550,6 +557,7 @@ export class Package {
          malloyConfig,
       );
       pkg.renderTagWarnings = renderTagWarnings;
+      pkg.manifestWarnings = outcome.packageMetadata.manifestWarnings ?? [];
       // Install the per-query freshness resolver on the freshly-built models.
       // At create time no manifest is bound yet, so the resolver returns
       // undefined (serve live) until a subsequent bindManifest → reloadAllModels.
@@ -688,6 +696,16 @@ export class Package {
          // (alongside the load-path log) so an operator can see it on the status
          // API like the other persist warnings — see persistenceCollisionWarnings.
          ...this.persistenceCollisionWarnings().map((message) => ({ message })),
+         // Materialization-config findings: a queryMetadata property that will
+         // not do what it says, and manifest shapes that still parse but are
+         // deprecated. Advisory by design — none of these blocks a publish.
+         ...materializationConfigWarnings({
+            packageMaterialization: this.packageMetadata.materialization,
+            sources: this.buildPlan?.sources
+               ? Object.values(this.buildPlan.sources)
+               : [],
+            manifestWarnings: this.manifestWarnings,
+         }),
       ];
       if (allWarnings.length > 0) {
          metadata.warnings = allWarnings;
@@ -1059,6 +1077,10 @@ export class Package {
    }
 
    public persistencePolicyWarnings(): string[] {
+      // REJECTION rules only: a non-empty result fails a publish and disarms the
+      // scheduler. Advisory findings (queryMetadata problems, a deprecated
+      // manifest shape) go to the operator warnings array instead — see
+      // materializationConfigWarnings.
       const warnings: string[] = [];
       const sources = this.buildPlan?.sources
          ? Object.values(this.buildPlan.sources)
@@ -1075,7 +1097,7 @@ export class Package {
             warnings.push(
                `#@ persist source "${source.name}" declares sharing=... which is ` +
                   `no longer supported: scope is a single package-level mode. Set ` +
-                  `the root-level "scope": "version" | "package" in ` +
+                  `"materialization": { "scope": "version" | "package" } in ` +
                   `${PACKAGE_MANIFEST_NAME} instead.`,
             );
          }
@@ -1480,6 +1502,7 @@ export class Package {
       // package's current storage= bindings so a reload preserves serve routing.
       this.pushStorageServeBindingsToModels();
       this.renderTagWarnings = renderTagWarnings;
+      this.manifestWarnings = outcome.packageMetadata.manifestWarnings ?? [];
       // A reload re-reads publisher.json in the worker; pick up any change to
       // the explore set and query-boundary mode so listModels()/the gate
       // reflect edited explores without a full Package.create.