@malloy-publisher/server 0.0.233 → 0.0.235

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/README.docker.md +1 -0
  2. package/dist/app/api-doc.yaml +208 -0
  3. package/dist/app/assets/{EnvironmentPage-DutP7T8h.js → EnvironmentPage-BsAnavYN.js} +1 -1
  4. package/dist/app/assets/{HomePage-BcxDrBfl.js → HomePage-CADE138j.js} +1 -1
  5. package/dist/app/assets/{LightMode-BJukGxgz.js → LightMode-Cfh7KzN8.js} +1 -1
  6. package/dist/app/assets/{MainPage-DXbwlMeF.js → MainPage-CO3pRlnV.js} +2 -2
  7. package/dist/app/assets/{MaterializationsPage-BBQksmTU.js → MaterializationsPage-p9YjkRXZ.js} +1 -1
  8. package/dist/app/assets/{ModelPage-C6tK51uU.js → ModelPage-C1OSTv-x.js} +1 -1
  9. package/dist/app/assets/{PackagePage-Bo3cwwZE.js → PackagePage-e4kN75YR.js} +1 -1
  10. package/dist/app/assets/{RouteError-BufkcAKE.js → RouteError-CzbfOkng.js} +1 -1
  11. package/dist/app/assets/{ThemeEditorPage-DICvvKpa.js → ThemeEditorPage-CciagFTq.js} +1 -1
  12. package/dist/app/assets/{WorkbookPage-Dkwt75Nj.js → WorkbookPage-DNWmkCXa.js} +1 -1
  13. package/dist/app/assets/{core-C0nunIQT.es-DlMLKZBK.js → core-Rj_4rRnA.es-BZyvITuO.js} +1 -1
  14. package/dist/app/assets/{index-CmEVVe-8.js → index-CH2AcDzc.js} +4 -4
  15. package/dist/app/assets/{index-qnhU9CGo.js → index-DQa463gC.js} +2 -2
  16. package/dist/app/assets/{index-Cs4WVm2z.js → index-DQpV7MyA.js} +1 -1
  17. package/dist/app/assets/{index-BabP-V-S.js → index-DzaYbhnD.js} +1 -1
  18. package/dist/app/assets/{index-BusxL5Pt.js → index-VBbcc8s6.js} +1 -1
  19. package/dist/app/index.html +1 -1
  20. package/dist/package_load_worker.mjs +53 -3
  21. package/dist/server.mjs +835 -97
  22. package/package.json +12 -12
  23. package/src/config.ts +35 -1
  24. package/src/controller/connection.controller.spec.ts +46 -0
  25. package/src/controller/connection.controller.ts +105 -2
  26. package/src/controller/materialization.controller.spec.ts +25 -0
  27. package/src/controller/materialization.controller.ts +60 -0
  28. package/src/controller/model.controller.ts +24 -0
  29. package/src/controller/query.controller.ts +83 -15
  30. package/src/mcp/handler_utils.ts +10 -2
  31. package/src/mcp/query_envelope.ts +10 -0
  32. package/src/mcp/skills/skills_bundle.json +1 -1
  33. package/src/mcp/tools/execute_query_tool.spec.ts +131 -0
  34. package/src/mcp/tools/execute_query_tool.ts +62 -25
  35. package/src/mcp_config.spec.ts +919 -0
  36. package/src/mcp_config.ts +425 -0
  37. package/src/oom_guards.integration.spec.ts +11 -3
  38. package/src/package_load/package_load_pool.ts +2 -0
  39. package/src/package_load/package_load_worker.ts +17 -5
  40. package/src/package_load/protocol.ts +6 -0
  41. package/src/query_metadata_metrics.ts +49 -0
  42. package/src/server.ts +99 -3
  43. package/src/service/build_plan.spec.ts +125 -0
  44. package/src/service/build_plan.ts +108 -7
  45. package/src/service/connection_config.ts +49 -0
  46. package/src/service/environment.ts +57 -3
  47. package/src/service/materialization_config_validation.spec.ts +99 -0
  48. package/src/service/materialization_config_validation.ts +120 -0
  49. package/src/service/materialization_schedule_surface.spec.ts +124 -0
  50. package/src/service/materialization_service.spec.ts +119 -0
  51. package/src/service/materialization_service.ts +186 -3
  52. package/src/service/materialization_test_fixtures.ts +86 -21
  53. package/src/service/model.spec.ts +45 -1
  54. package/src/service/model.ts +145 -19
  55. package/src/service/package.ts +24 -1
  56. package/src/service/package_manifest.spec.ts +137 -4
  57. package/src/service/package_manifest.ts +140 -5
  58. package/src/service/persist_annotation_validation.spec.ts +12 -0
  59. package/src/service/persist_annotation_validation.ts +9 -4
  60. package/src/service/query_metadata.spec.ts +408 -0
  61. package/src/service/query_metadata.ts +492 -0
  62. package/src/service/query_metadata_identity.spec.ts +149 -0
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@malloy-publisher/server",
3
3
  "description": "Malloy Publisher Server",
4
- "version": "0.0.233",
4
+ "version": "0.0.235",
5
5
  "main": "dist/server.mjs",
6
6
  "bin": {
7
7
  "malloy-publisher": "dist/server.mjs"
@@ -36,17 +36,17 @@
36
36
  "@azure/storage-blob": "^12.26.0",
37
37
  "@duckdb/node-api": "1.5.3-r.2",
38
38
  "@google-cloud/storage": "^7.16.0",
39
- "@malloydata/db-bigquery": "^0.0.426",
40
- "@malloydata/db-databricks": "^0.0.426",
41
- "@malloydata/db-duckdb": "^0.0.426",
42
- "@malloydata/db-mysql": "^0.0.426",
43
- "@malloydata/db-postgres": "^0.0.426",
44
- "@malloydata/db-publisher": "^0.0.426",
45
- "@malloydata/db-snowflake": "^0.0.426",
46
- "@malloydata/db-trino": "^0.0.426",
47
- "@malloydata/malloy": "^0.0.426",
48
- "@malloydata/malloy-sql": "^0.0.426",
49
- "@malloydata/render-validator": "^0.0.426",
39
+ "@malloydata/db-bigquery": "^0.0.427",
40
+ "@malloydata/db-databricks": "^0.0.427",
41
+ "@malloydata/db-duckdb": "^0.0.427",
42
+ "@malloydata/db-mysql": "^0.0.427",
43
+ "@malloydata/db-postgres": "^0.0.427",
44
+ "@malloydata/db-publisher": "^0.0.427",
45
+ "@malloydata/db-snowflake": "^0.0.427",
46
+ "@malloydata/db-trino": "^0.0.427",
47
+ "@malloydata/malloy": "^0.0.427",
48
+ "@malloydata/malloy-sql": "^0.0.427",
49
+ "@malloydata/render-validator": "^0.0.427",
50
50
  "@modelcontextprotocol/sdk": "^1.13.2",
51
51
  "@opentelemetry/api": "^1.9.0",
52
52
  "@opentelemetry/auto-instrumentations-node": "^0.57.0",
package/src/config.ts CHANGED
@@ -189,7 +189,7 @@ function parseFloatEnv(name: string): number | undefined {
189
189
  return value;
190
190
  }
191
191
 
192
- function parseBoolEnv(name: string): boolean | undefined {
192
+ export function parseBoolEnv(name: string): boolean | undefined {
193
193
  const raw = process.env[name];
194
194
  if (raw === undefined || raw.trim() === "") return undefined;
195
195
  const normalised = raw.trim().toLowerCase();
@@ -615,6 +615,40 @@ export const getPersistCollisionEnforce = (): boolean =>
615
615
  // prevent. A typo throws at startup, like every other flag here.
616
616
  parseBoolEnv("PERSIST_COLLISION_ENFORCE") ?? false;
617
617
 
618
+ /**
619
+ * Whether the publisher attaches per-query metadata at all, from
620
+ * `PUBLISHER_QUERY_METADATA` (default `off`).
621
+ *
622
+ * Ships dark for a release, like `PERSIST_STORAGE_MODE` before it, and for the
623
+ * same reason: this is the rare feature that touches EVERY statement the server
624
+ * sends. On a backend with no native tag facility the bag rides as a leading SQL
625
+ * comment, so `on` changes the text of the statement (never its meaning or its
626
+ * results) and puts the bag in query logs and `pg_stat_activity`.
627
+ *
628
+ * The risk that decides the default is upstream, not here. Malloy validates the
629
+ * bag at dispatch and THROWS on one it cannot render, and the contract it
630
+ * validates against is mirrored in `service/query_metadata.ts` against a pinned
631
+ * version. Every mitigation on this path — clamping, shedding, never throwing —
632
+ * is downstream of that mirror being right, so a tightened upstream limit would
633
+ * surface as failing customer queries on a path nobody opted into. `off` for a
634
+ * release means a deployment turns attribution on deliberately, having read
635
+ * what it does to its statements.
636
+ *
637
+ * Case-insensitive; loud-fails on an unrecognized value, so a typo cannot
638
+ * silently leave a deployment that asked for attribution without it.
639
+ */
640
+ export type QueryMetadataMode = "on" | "off";
641
+
642
+ export const getQueryMetadataMode = (): QueryMetadataMode => {
643
+ const raw = process.env.PUBLISHER_QUERY_METADATA;
644
+ if (raw === undefined || raw.trim() === "") return "off";
645
+ const value = raw.trim().toLowerCase();
646
+ if (value === "on" || value === "off") return value;
647
+ throw new Error(
648
+ `PUBLISHER_QUERY_METADATA must be on | off (got ${JSON.stringify(raw)})`,
649
+ );
650
+ };
651
+
618
652
  function substituteEnvVars(value: string): string {
619
653
  const envVarPattern = /\$\{([A-Z_][A-Z0-9_]*)\}/g;
620
654
 
@@ -60,6 +60,52 @@ function buildController(
60
60
  return { controller, runSQL, assertCanAdmitQuery };
61
61
  }
62
62
 
63
+ describe("ConnectionController connection queryMetadata", () => {
64
+ afterEach(() => sinon.restore());
65
+
66
+ it("rejects a connection default the contract cannot render", async () => {
67
+ // The connection is the one metadata layer whose author is right here, so
68
+ // it is the one that can be told. `cost-centre` is the natural spelling
69
+ // and would otherwise be dropped at dispatch, leaving every statement on
70
+ // the connection missing the property its operator configured.
71
+ const { controller } = buildController(sinon.stub().resolves());
72
+ await expect(
73
+ controller.addConnection("env", "warehouse", {
74
+ name: "warehouse",
75
+ type: "postgres",
76
+ queryMetadata: { "cost-centre": "eng" },
77
+ }),
78
+ ).rejects.toThrow(/queryMetadata is invalid/);
79
+ });
80
+
81
+ it("rejects an enforced bag the contract cannot render", async () => {
82
+ const { controller } = buildController(sinon.stub().resolves());
83
+ await expect(
84
+ controller.addConnection("env", "warehouse", {
85
+ name: "warehouse",
86
+ type: "postgres",
87
+ queryMetadataEnforced: { "org.id": "acme" },
88
+ }),
89
+ ).rejects.toThrow(/queryMetadataEnforced is invalid/);
90
+ });
91
+
92
+ it("accepts a conforming connection default", async () => {
93
+ const { controller } = buildController(sinon.stub().resolves());
94
+ const addConnection = sinon.stub().resolves();
95
+ (
96
+ controller as unknown as {
97
+ connectionService: { addConnection: sinon.SinonStub };
98
+ }
99
+ ).connectionService = { addConnection } as never;
100
+ await controller.addConnection("env", "warehouse", {
101
+ name: "warehouse",
102
+ type: "postgres",
103
+ queryMetadata: { cost_centre: "eng" },
104
+ });
105
+ expect(addConnection.calledOnce).toBe(true);
106
+ });
107
+ });
108
+
63
109
  describe("ConnectionController.getConnectionQueryData row cap", () => {
64
110
  const originalEnv = process.env.PUBLISHER_MAX_QUERY_ROWS;
65
111
 
@@ -21,6 +21,15 @@ import {
21
21
  getSchemasForConnection,
22
22
  listTablesForSchema,
23
23
  } from "../service/db_utils";
24
+ import {
25
+ mergeQueryMetadata,
26
+ mintCorrelationId,
27
+ parseQueryClass,
28
+ parseSuppliedQueryMetadata,
29
+ queryMetadataViolations,
30
+ type QueryClass,
31
+ type QueryMetadata,
32
+ } from "../service/query_metadata";
24
33
  import type { Environment } from "../service/environment";
25
34
  import { EnvironmentStore } from "../service/environment_store";
26
35
  import { isStreamingConnection, streamSqlWithBudget } from "../stream_helpers";
@@ -112,6 +121,22 @@ function validateAdminAuthoredConnection(
112
121
  } catch (error) {
113
122
  throw new BadRequestError((error as Error).message);
114
123
  }
124
+
125
+ // The connection is the one metadata layer whose author is right here, so it
126
+ // is the one that can be told. A property the contract rejects (a hyphen in
127
+ // `cost-centre` is the natural spelling and the first thing anyone tries)
128
+ // would otherwise be dropped at dispatch and only ever surface as a metric —
129
+ // every statement on the connection missing the property its operator
130
+ // believes they configured. Config LOAD warns instead of throwing (see
131
+ // assembleEnvironmentConnections): a tag must never fail an environment.
132
+ for (const field of ["queryMetadata", "queryMetadataEnforced"] as const) {
133
+ const violations = queryMetadataViolations(connectionConfig[field]);
134
+ if (violations.length > 0) {
135
+ throw new BadRequestError(
136
+ `Connection "${connectionName}" ${field} is invalid: ${violations.join("; ")}`,
137
+ );
138
+ }
139
+ }
115
140
  }
116
141
 
117
142
  export class ConnectionController {
@@ -140,6 +165,43 @@ export class ConnectionController {
140
165
  return environment.getApiConnection(connectionName);
141
166
  }
142
167
 
168
+ /**
169
+ * A connection's default per-query metadata, or null when it declares none.
170
+ * Fails open: metadata is observability, so a connection whose config can't be
171
+ * read contributes no default rather than failing the query the caller asked
172
+ * for.
173
+ */
174
+ private async connectionQueryMetadata(
175
+ environmentName: string,
176
+ connectionName: string,
177
+ ): Promise<{
178
+ default: QueryMetadata | null;
179
+ enforced: QueryMetadata | null;
180
+ }> {
181
+ try {
182
+ const environment = await this.environmentStore.getEnvironment(
183
+ environmentName,
184
+ false,
185
+ );
186
+ const connection = this.getApiConnectionForLookup(
187
+ environment,
188
+ connectionName,
189
+ );
190
+ return {
191
+ default: connection.queryMetadata ?? null,
192
+ enforced: connection.queryMetadataEnforced ?? null,
193
+ };
194
+ } catch (error) {
195
+ // Fails open like every other metadata path, but not invisibly: the
196
+ // layer lost here is the enforced one, and no metric covers it.
197
+ logger.debug("No query-metadata layers for connection", {
198
+ connectionName,
199
+ error,
200
+ });
201
+ return { default: null, enforced: null };
202
+ }
203
+ }
204
+
143
205
  private async getMalloyConnection(
144
206
  environmentName: string,
145
207
  connectionName: string,
@@ -453,6 +515,12 @@ export class ConnectionController {
453
515
  sqlStatement: string,
454
516
  options: string,
455
517
  packageName?: string,
518
+ /**
519
+ * The request's per-query metadata fields, unvalidated — this controller is
520
+ * the boundary that turns a bad bag into a 400 rather than letting the
521
+ * connector refuse the statement at dispatch.
522
+ */
523
+ metadata?: { queryMetadata?: unknown; queryClass?: unknown },
456
524
  ): Promise<ApiQueryData> {
457
525
  // Express parses repeated query parameters (?sqlStatement=a&sqlStatement=b)
458
526
  // and array-shaped JSON bodies as `string[]`, not `string`. The route
@@ -512,6 +580,41 @@ export class ConnectionController {
512
580
  runSQLOptions.abortSignal = undefined;
513
581
  }
514
582
 
583
+ // Per-query metadata. Validated here, not clamped: a raw-SQL caller gets a
584
+ // 400 telling it which property is wrong instead of a statement the
585
+ // connector refuses at dispatch. `options` is forwarded as RunSQLOptions, so
586
+ // a bag can also arrive inside it — validate that one too, and let the
587
+ // documented field win.
588
+ const suppliedMetadata =
589
+ metadata?.queryMetadata ?? runSQLOptions.queryMetadata;
590
+ let requestMetadata: QueryMetadata | undefined;
591
+ let queryClass: QueryClass | undefined;
592
+ try {
593
+ requestMetadata = parseSuppliedQueryMetadata(suppliedMetadata);
594
+ queryClass = parseQueryClass(metadata?.queryClass);
595
+ } catch (error) {
596
+ throw new BadRequestError((error as Error).message);
597
+ }
598
+ const connectionLayers = await this.connectionQueryMetadata(
599
+ environmentName,
600
+ connectionName,
601
+ );
602
+ const resolvedMetadata = mergeQueryMetadata({
603
+ connection: connectionLayers.default,
604
+ enforced: connectionLayers.enforced,
605
+ request: requestMetadata,
606
+ context: {
607
+ // Raw SQL against a connection is platform maintenance unless the
608
+ // caller says otherwise — it is not a modeled query.
609
+ queryClass: queryClass ?? "ops",
610
+ environment: environmentName,
611
+ package: packageName,
612
+ correlationId: mintCorrelationId(),
613
+ },
614
+ });
615
+ runSQLOptions.queryMetadata = resolvedMetadata.metadata;
616
+ const queryCorrelationId = resolvedMetadata.metadata?.query_id ?? null;
617
+
515
618
  // Bound the response with two layered caps:
516
619
  //
517
620
  // - Row cap (PUBLISHER_MAX_QUERY_ROWS) — pushed to the driver as
@@ -572,7 +675,7 @@ export class ConnectionController {
572
675
  throw new ConnectionError((error as Error).message);
573
676
  }
574
677
  }, getQueryTimeoutMs());
575
- return { data: JSON.stringify(streamed) };
678
+ return { data: JSON.stringify(streamed), queryCorrelationId };
576
679
  }
577
680
 
578
681
  const result = await runWithQueryTimeout(async (signal) => {
@@ -601,7 +704,7 @@ export class ConnectionController {
601
704
  );
602
705
  }
603
706
 
604
- return { data: JSON.stringify(result) };
707
+ return { data: JSON.stringify(result), queryCorrelationId };
605
708
  }
606
709
 
607
710
  public async getConnectionTemporaryTable(
@@ -265,4 +265,29 @@ describe("MaterializationController.createMaterialization validation", () => {
265
265
  }),
266
266
  ).rejects.toThrow(BadRequestError);
267
267
  });
268
+
269
+ it("rejects a runId the metadata contract cannot carry", async () => {
270
+ // runId becomes the `run_id` property on every statement of the build, so
271
+ // an over-long or unrenderable value is silently truncated and rewritten
272
+ // at dispatch — leaving the caller holding an id that joins to nothing.
273
+ // The neighbouring `trigger` is enum-validated; this is the same kind of
274
+ // boundary and gets the same treatment.
275
+ const { controller } = build();
276
+ await expect(
277
+ controller.createMaterialization("env", "pkg", {
278
+ runContext: { runId: "x".repeat(300) },
279
+ }),
280
+ ).rejects.toThrow(BadRequestError);
281
+ await expect(
282
+ controller.createMaterialization("env", "pkg", {
283
+ runContext: { runId: 'has "quotes"' },
284
+ }),
285
+ ).rejects.toThrow(BadRequestError);
286
+ });
287
+
288
+ it("still accepts a conforming runId", async () => {
289
+ expect(
290
+ await parse({ runContext: { trigger: "publish", runId: "run-42" } }),
291
+ ).toEqual({ runContext: { trigger: "publish", runId: "run-42" } });
292
+ });
268
293
  });
@@ -1,9 +1,15 @@
1
+ import type { components } from "../api";
1
2
  import { BadRequestError } from "../errors";
2
3
  import {
3
4
  BuildInstruction,
4
5
  ManifestReference,
5
6
  } from "../storage/DatabaseInterface";
6
7
  import { MaterializationService } from "../service/materialization_service";
8
+ import { queryMetadataViolations } from "../service/query_metadata";
9
+
10
+ type RunContext = components["schemas"]["RunContext"];
11
+
12
+ const RUN_TRIGGERS = ["publish", "on_demand", "scheduler"] as const;
7
13
 
8
14
  export class MaterializationController {
9
15
  constructor(private materializationService: MaterializationService) {}
@@ -31,6 +37,7 @@ export class MaterializationController {
31
37
  buildInstructions?: BuildInstruction[];
32
38
  referenceManifest?: ManifestReference[];
33
39
  strictUpstreams?: boolean;
40
+ runContext?: RunContext;
34
41
  } {
35
42
  const result: {
36
43
  forceRefresh?: boolean;
@@ -38,7 +45,11 @@ export class MaterializationController {
38
45
  buildInstructions?: BuildInstruction[];
39
46
  referenceManifest?: ManifestReference[];
40
47
  strictUpstreams?: boolean;
48
+ runContext?: RunContext;
41
49
  } = {};
50
+ if (body.runContext !== undefined && body.runContext !== null) {
51
+ result.runContext = this.validateRunContext(body.runContext);
52
+ }
42
53
  if (
43
54
  body.buildInstructions !== undefined &&
44
55
  body.buildInstructions !== null
@@ -72,6 +83,55 @@ export class MaterializationController {
72
83
  return result;
73
84
  }
74
85
 
86
+ /**
87
+ * Validate `runContext`, the caller's observability context for one run.
88
+ * `trigger` is a closed enum here even though it feeds a metadata property:
89
+ * the whole point is that a reader can group runs by how they started, which a
90
+ * free-form value would quietly break.
91
+ *
92
+ * Note this is NOT the service-level `trigger` the parser above deliberately
93
+ * refuses. That one decides whether the run counts as scheduled; this one only
94
+ * labels the statements the run issues, so accepting `publish` from a caller
95
+ * forges nothing.
96
+ */
97
+ private validateRunContext(raw: unknown): RunContext {
98
+ if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
99
+ throw new BadRequestError("runContext must be an object");
100
+ }
101
+ const obj = raw as Record<string, unknown>;
102
+ const context: RunContext = {};
103
+ if (obj.trigger !== undefined && obj.trigger !== null) {
104
+ if (
105
+ typeof obj.trigger !== "string" ||
106
+ !RUN_TRIGGERS.includes(obj.trigger as (typeof RUN_TRIGGERS)[number])
107
+ ) {
108
+ throw new BadRequestError(
109
+ `runContext.trigger must be one of ${RUN_TRIGGERS.join(" | ")}`,
110
+ );
111
+ }
112
+ context.trigger = obj.trigger as RunContext["trigger"];
113
+ }
114
+ if (obj.runId !== undefined && obj.runId !== null) {
115
+ if (typeof obj.runId !== "string") {
116
+ throw new BadRequestError("runContext.runId must be a string");
117
+ }
118
+ // Held to the metadata contract like any other caller-supplied
119
+ // property: it becomes the `run_id` on every statement of the build,
120
+ // and a value the contract rejects would otherwise be truncated and
121
+ // rewritten in silence — leaving the caller with an id it cannot join
122
+ // on and no way to know why.
123
+ const violations = queryMetadataViolations({ run_id: obj.runId });
124
+ if (violations.length > 0) {
125
+ throw new BadRequestError(
126
+ `runContext.runId is attached to every statement as the run_id ` +
127
+ `property: ${violations.join("; ")}`,
128
+ );
129
+ }
130
+ context.runId = obj.runId;
131
+ }
132
+ return context;
133
+ }
134
+
75
135
  /**
76
136
  * Validate the orchestrated `buildInstructions` payload (BuildInstructions:
77
137
  * `{ sources: BuildInstruction[], referenceManifest?, strictUpstreams? }`)
@@ -1,6 +1,7 @@
1
1
  import { components } from "../api";
2
2
  import { getQueryTimeoutMs } from "../config";
3
3
  import { ModelNotFoundError } from "../errors";
4
+ import { logger } from "../logger";
4
5
  import { runWithQueryTimeout } from "../query_timeout";
5
6
  import { EnvironmentStore } from "../service/environment_store";
6
7
  import type { FilterParams } from "../service/filter";
@@ -133,6 +134,29 @@ export class ModelController {
133
134
  bypassFilters,
134
135
  givens,
135
136
  abortSignal,
137
+ {
138
+ environment: environmentName,
139
+ // The environment owns the connection configs, so the default
140
+ // and enforced layers are read here rather than from the model.
141
+ connectionMetadata: (connectionName) => {
142
+ try {
143
+ const connection =
144
+ environment.getApiConnection(connectionName);
145
+ return {
146
+ default: connection.queryMetadata,
147
+ enforced: connection.queryMetadataEnforced,
148
+ };
149
+ } catch (error) {
150
+ // Fails open, and says so: what an unreadable
151
+ // connection costs is the enforced layer.
152
+ logger.debug(
153
+ "No query-metadata layers for connection",
154
+ { connectionName, error },
155
+ );
156
+ return null;
157
+ }
158
+ },
159
+ },
136
160
  ),
137
161
  getQueryTimeoutMs(),
138
162
  );
@@ -2,8 +2,16 @@ import { validateRenderTags } from "@malloydata/render-validator";
2
2
  import { components } from "../api";
3
3
  import { getQueryTimeoutMs } from "../config";
4
4
  import { API_PREFIX } from "../constants";
5
- import { ModelNotFoundError } from "../errors";
5
+ import { BadRequestError, ModelNotFoundError } from "../errors";
6
6
  import { bigIntReplacer } from "../json_utils";
7
+ import { logger } from "../logger";
8
+ import {
9
+ mintCorrelationId,
10
+ parseQueryClass,
11
+ parseSuppliedQueryMetadata,
12
+ type QueryClass,
13
+ type QueryMetadata,
14
+ } from "../service/query_metadata";
7
15
  import { runWithQueryTimeout } from "../query_timeout";
8
16
  import { EnvironmentStore } from "../service/environment_store";
9
17
  import type { FilterParams } from "../service/filter";
@@ -29,7 +37,26 @@ export class QueryController {
29
37
  filterParams?: FilterParams,
30
38
  bypassFilters?: boolean,
31
39
  givens?: Record<string, GivenValue>,
40
+ /**
41
+ * The request's per-query metadata fields, unvalidated — this controller is
42
+ * the boundary that turns a bad bag into a 400 rather than letting the
43
+ * connector refuse the statement at dispatch.
44
+ */
45
+ metadata?: {
46
+ queryMetadata?: unknown;
47
+ queryClass?: unknown;
48
+ versionId?: string;
49
+ },
32
50
  ): Promise<ApiQuery> {
51
+ let requestMetadata: QueryMetadata | undefined;
52
+ let queryClass: QueryClass | undefined;
53
+ try {
54
+ requestMetadata = parseSuppliedQueryMetadata(metadata?.queryMetadata);
55
+ queryClass = parseQueryClass(metadata?.queryClass);
56
+ } catch (error) {
57
+ throw new BadRequestError((error as Error).message);
58
+ }
59
+
33
60
  const environment = await this.environmentStore.getEnvironment(
34
61
  environmentName,
35
62
  false,
@@ -46,20 +73,60 @@ export class QueryController {
46
73
  if (!model) {
47
74
  throw new ModelNotFoundError(`${modelPath} does not exist`);
48
75
  } else {
49
- const { result, compactResult, rowLimit, rowLimitSource } =
50
- await runWithQueryTimeout(
51
- (abortSignal) =>
52
- model.getQueryResults(
53
- sourceName,
54
- queryName,
55
- query,
56
- filterParams,
57
- bypassFilters,
58
- givens,
59
- abortSignal,
60
- ),
61
- getQueryTimeoutMs(),
62
- );
76
+ const {
77
+ result,
78
+ compactResult,
79
+ rowLimit,
80
+ rowLimitSource,
81
+ queryCorrelationId,
82
+ } = await runWithQueryTimeout(
83
+ (abortSignal) =>
84
+ model.getQueryResults(
85
+ sourceName,
86
+ queryName,
87
+ query,
88
+ filterParams,
89
+ bypassFilters,
90
+ givens,
91
+ abortSignal,
92
+ {
93
+ request: requestMetadata,
94
+ queryClass,
95
+ environment: environmentName,
96
+ // Always undefined today: the route 501s any versionId
97
+ // before this runs. Wired so that lifting that rejection
98
+ // is the whole change.
99
+ version: metadata?.versionId,
100
+ // Minted here because this is the boundary that returns
101
+ // it; a path with nowhere to put it does not mint one.
102
+ correlationId: mintCorrelationId(),
103
+ // The environment owns the connection configs, so the
104
+ // default and enforced layers are read here rather than
105
+ // from the model.
106
+ connectionMetadata: (connectionName) => {
107
+ try {
108
+ const connection =
109
+ environment.getApiConnection(connectionName);
110
+ return {
111
+ default: connection.queryMetadata,
112
+ enforced: connection.queryMetadataEnforced,
113
+ };
114
+ } catch (error) {
115
+ // Failing open is right — a tag must not fail a
116
+ // query — but this is the one drop with no metric
117
+ // behind it, and what it costs is the ENFORCED
118
+ // layer. Log it so it is diagnosable.
119
+ logger.debug(
120
+ "No query-metadata layers for connection",
121
+ { connectionName, error },
122
+ );
123
+ return null;
124
+ }
125
+ },
126
+ },
127
+ ),
128
+ getQueryTimeoutMs(),
129
+ );
63
130
  const renderLogs = validateRenderTags(result);
64
131
  return {
65
132
  result: compactJson
@@ -78,6 +145,7 @@ export class QueryController {
78
145
  // deliberate `limit:`/`top:` from the silently-applied default, and
79
146
  // so cannot reproduce the MCP envelope's _limit_hit.
80
147
  queryRowLimitSource: rowLimitSource,
148
+ queryCorrelationId,
81
149
  } as ApiQuery;
82
150
  }
83
151
  }
@@ -19,6 +19,7 @@ import {
19
19
  type ErrorDetails,
20
20
  } from "./error_messages";
21
21
  import type { Model } from "../service/model";
22
+ import type { Environment } from "../service/environment";
22
23
  import { logger } from "../logger";
23
24
 
24
25
  /**
@@ -116,6 +117,11 @@ export function classifyToolError(
116
117
  /**
117
118
  * Fetches and validates the Package and Model instances needed for query execution.
118
119
  * Handles errors related to package/model access and initial compilation.
120
+ *
121
+ * The resolved Environment comes back with the model because a query needs it for
122
+ * more than the lookup: it owns the connection configs the per-query metadata
123
+ * layers are read from.
124
+ *
119
125
  * @returns An object containing the Model instance or a pre-formatted ErrorDetails object.
120
126
  */
121
127
  export async function getModelForQuery(
@@ -123,7 +129,9 @@ export async function getModelForQuery(
123
129
  environmentName: string,
124
130
  packageName: string,
125
131
  modelPath: string,
126
- ): Promise<{ model: Model } | { error: ErrorDetails }> {
132
+ ): Promise<
133
+ { model: Model; environment: Environment } | { error: ErrorDetails }
134
+ > {
127
135
  try {
128
136
  const environment = await environmentStore.getEnvironment(
129
137
  environmentName,
@@ -143,7 +151,7 @@ export async function getModelForQuery(
143
151
  }
144
152
  // Attempt to get the model definition early to catch initial compilation errors
145
153
  await model.getModel(); // This might throw ModelCompilationError
146
- return { model };
154
+ return { model, environment };
147
155
  } catch (error) {
148
156
  // Handle errors during package/model access or initial compilation
149
157
  let errorDetails: ErrorDetails;
@@ -65,6 +65,14 @@ export interface QueryEnvelope {
65
65
  _rows_truncated?: boolean;
66
66
  _total_rows?: number;
67
67
  _returned_rows?: number;
68
+ /**
69
+ * The `query_id` property attached to this query's statements: the join key
70
+ * into the backend's own query record (`QUERY_HISTORY`, `JOBS.labels`, the
71
+ * statement text). Publisher-side and new to both products, like the two
72
+ * `_limit_...` fields. Absent when nothing was attached, which is every query
73
+ * on a deployment that has not enabled query metadata.
74
+ */
75
+ _query_id?: string;
68
76
  warning?: string;
69
77
  renderLogErrors?: string[];
70
78
  }
@@ -97,6 +105,7 @@ export function buildQueryEnvelope(
97
105
  renderLogErrors: string[] = [],
98
106
  limit = MAX_RESULT_CHARS,
99
107
  rowLimitSource: QueryRowLimitSource = "server_default",
108
+ queryCorrelationId: string | null = null,
100
109
  ): QueryEnvelope {
101
110
  const rowCount = Array.isArray(rows) ? rows.length : 0;
102
111
  // Equality, not >=: the cap is pushed into the SQL, so the database cannot
@@ -132,6 +141,7 @@ export function buildQueryEnvelope(
132
141
  _query_row_limit: rowLimit,
133
142
  _limit_source: rowLimitSource,
134
143
  _limit_hit: limitHit,
144
+ ...(queryCorrelationId !== null && { _query_id: queryCorrelationId }),
135
145
  ...(renderLogErrors.length > 0 && { renderLogErrors }),
136
146
  };
137
147