@malloy-publisher/server 0.0.232 → 0.0.234

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.docker.md +1 -0
  2. package/dist/app/api-doc.yaml +269 -10
  3. package/dist/app/assets/{EnvironmentPage-DXEaZIPx.js → EnvironmentPage-DTZQ4Gxc.js} +1 -1
  4. package/dist/app/assets/{HomePage-kofsqpZt.js → HomePage-C5mlDPXK.js} +1 -1
  5. package/dist/app/assets/{LightMode-CNhIlIlJ.js → LightMode-DGNmhG0u.js} +1 -1
  6. package/dist/app/assets/{MainPage-Bgqo8jCy.js → MainPage-CVL_wmP4.js} +1 -1
  7. package/dist/app/assets/{MaterializationsPage-CgBlgGz2.js → MaterializationsPage-DmzMBCpy.js} +1 -1
  8. package/dist/app/assets/{ModelPage-B0TjoDtf.js → ModelPage-Dbvf4QbB.js} +1 -1
  9. package/dist/app/assets/{PackagePage-BL8vnFj1.js → PackagePage-DxdHc2Qs.js} +1 -1
  10. package/dist/app/assets/{RouteError-BzPby0X2.js → RouteError-OJdT4tCd.js} +1 -1
  11. package/dist/app/assets/{ThemeEditorPage-CTEP_9r3.js → ThemeEditorPage-Bk7s0KXY.js} +1 -1
  12. package/dist/app/assets/{WorkbookPage-BwM3BmKw.js → WorkbookPage-j_vCWdN3.js} +1 -1
  13. package/dist/app/assets/{core-CK68iv6w.es-CpRxXBt7.js → core-Rj_4rRnA.es-DoIfLxDJ.js} +1 -1
  14. package/dist/app/assets/{index-B33zGctF.js → index-B_jKMR35.js} +4 -4
  15. package/dist/app/assets/{index-CmkW1MiE.js → index-D-rDyK11.js} +1 -1
  16. package/dist/app/assets/{index-tXJXwdyj.js → index-DWIe_hK0.js} +1 -1
  17. package/dist/app/assets/{index-BkiWKaAF.js → index-hw-xn0X7.js} +1 -1
  18. package/dist/app/index.html +1 -1
  19. package/dist/package_load_worker.mjs +53 -3
  20. package/dist/server.mjs +20277 -925
  21. package/package.json +1 -1
  22. package/src/config.ts +35 -1
  23. package/src/controller/connection.controller.spec.ts +46 -0
  24. package/src/controller/connection.controller.ts +105 -2
  25. package/src/controller/materialization.controller.spec.ts +25 -0
  26. package/src/controller/materialization.controller.ts +60 -0
  27. package/src/controller/model.controller.ts +24 -0
  28. package/src/controller/query.controller.ts +83 -10
  29. package/src/json_utils.spec.ts +51 -0
  30. package/src/json_utils.ts +33 -0
  31. package/src/mcp/handler_utils.ts +10 -2
  32. package/src/mcp/query_envelope.spec.ts +229 -0
  33. package/src/mcp/query_envelope.ts +240 -0
  34. package/src/mcp/server.protocol.spec.ts +128 -16
  35. package/src/mcp/skills/build_skills_bundle.ts +94 -4
  36. package/src/mcp/skills/skills_bundle.json +1 -1
  37. package/src/mcp/skills/skills_bundle.spec.ts +113 -4
  38. package/src/mcp/tool_response.spec.ts +108 -0
  39. package/src/mcp/tool_response.ts +138 -0
  40. package/src/mcp/tools/compile_tool.spec.ts +112 -4
  41. package/src/mcp/tools/compile_tool.ts +61 -30
  42. package/src/mcp/tools/docs_search_tool.ts +6 -16
  43. package/src/mcp/tools/execute_query_tool.spec.ts +154 -3
  44. package/src/mcp/tools/execute_query_tool.ts +131 -155
  45. package/src/mcp/tools/get_context_tool.spec.ts +63 -3
  46. package/src/mcp/tools/get_context_tool.ts +43 -46
  47. package/src/mcp/tools/reload_package_tool.ts +3 -29
  48. package/src/mcp_config.spec.ts +919 -0
  49. package/src/mcp_config.ts +425 -0
  50. package/src/oom_guards.integration.spec.ts +11 -3
  51. package/src/package_load/package_load_pool.ts +2 -0
  52. package/src/package_load/package_load_worker.ts +17 -5
  53. package/src/package_load/protocol.ts +6 -0
  54. package/src/query_metadata_metrics.ts +49 -0
  55. package/src/server.ts +99 -3
  56. package/src/service/build_plan.spec.ts +125 -0
  57. package/src/service/build_plan.ts +108 -7
  58. package/src/service/compile_fragment_techniques.spec.ts +156 -0
  59. package/src/service/connection.spec.ts +371 -1
  60. package/src/service/connection.ts +77 -14
  61. package/src/service/connection_config.spec.ts +60 -0
  62. package/src/service/connection_config.ts +75 -0
  63. package/src/service/duckdb_instance_isolation.spec.ts +137 -0
  64. package/src/service/environment.ts +57 -3
  65. package/src/service/materialization_config_validation.spec.ts +99 -0
  66. package/src/service/materialization_config_validation.ts +120 -0
  67. package/src/service/materialization_schedule_surface.spec.ts +124 -0
  68. package/src/service/materialization_service.spec.ts +119 -0
  69. package/src/service/materialization_service.ts +186 -3
  70. package/src/service/materialization_test_fixtures.ts +86 -21
  71. package/src/service/model.spec.ts +45 -1
  72. package/src/service/model.ts +171 -23
  73. package/src/service/model_limits.spec.ts +28 -0
  74. package/src/service/model_limits.ts +21 -0
  75. package/src/service/package.ts +24 -1
  76. package/src/service/package_manifest.spec.ts +137 -4
  77. package/src/service/package_manifest.ts +140 -5
  78. package/src/service/persist_annotation_validation.spec.ts +12 -0
  79. package/src/service/persist_annotation_validation.ts +9 -4
  80. package/src/service/query_metadata.spec.ts +408 -0
  81. package/src/service/query_metadata.ts +492 -0
  82. package/src/service/query_metadata_identity.spec.ts +149 -0
  83. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +37 -12
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@malloy-publisher/server",
3
3
  "description": "Malloy Publisher Server",
4
- "version": "0.0.232",
4
+ "version": "0.0.234",
5
5
  "main": "dist/server.mjs",
6
6
  "bin": {
7
7
  "malloy-publisher": "dist/server.mjs"
package/src/config.ts CHANGED
@@ -189,7 +189,7 @@ function parseFloatEnv(name: string): number | undefined {
189
189
  return value;
190
190
  }
191
191
 
192
- function parseBoolEnv(name: string): boolean | undefined {
192
+ export function parseBoolEnv(name: string): boolean | undefined {
193
193
  const raw = process.env[name];
194
194
  if (raw === undefined || raw.trim() === "") return undefined;
195
195
  const normalised = raw.trim().toLowerCase();
@@ -615,6 +615,40 @@ export const getPersistCollisionEnforce = (): boolean =>
615
615
  // prevent. A typo throws at startup, like every other flag here.
616
616
  parseBoolEnv("PERSIST_COLLISION_ENFORCE") ?? false;
617
617
 
618
+ /**
619
+ * Whether the publisher attaches per-query metadata at all, from
620
+ * `PUBLISHER_QUERY_METADATA` (default `off`).
621
+ *
622
+ * Ships dark for a release, like `PERSIST_STORAGE_MODE` before it, and for the
623
+ * same reason: this is the rare feature that touches EVERY statement the server
624
+ * sends. On a backend with no native tag facility the bag rides as a leading SQL
625
+ * comment, so `on` changes the text of the statement (never its meaning or its
626
+ * results) and puts the bag in query logs and `pg_stat_activity`.
627
+ *
628
+ * The risk that decides the default is upstream, not here. Malloy validates the
629
+ * bag at dispatch and THROWS on one it cannot render, and the contract it
630
+ * validates against is mirrored in `service/query_metadata.ts` against a pinned
631
+ * version. Every mitigation on this path — clamping, shedding, never throwing —
632
+ * is downstream of that mirror being right, so a tightened upstream limit would
633
+ * surface as failing customer queries on a path nobody opted into. `off` for a
634
+ * release means a deployment turns attribution on deliberately, having read
635
+ * what it does to its statements.
636
+ *
637
+ * Case-insensitive; loud-fails on an unrecognized value, so a typo cannot
638
+ * silently leave a deployment that asked for attribution without it.
639
+ */
640
+ export type QueryMetadataMode = "on" | "off";
641
+
642
+ export const getQueryMetadataMode = (): QueryMetadataMode => {
643
+ const raw = process.env.PUBLISHER_QUERY_METADATA;
644
+ if (raw === undefined || raw.trim() === "") return "off";
645
+ const value = raw.trim().toLowerCase();
646
+ if (value === "on" || value === "off") return value;
647
+ throw new Error(
648
+ `PUBLISHER_QUERY_METADATA must be on | off (got ${JSON.stringify(raw)})`,
649
+ );
650
+ };
651
+
618
652
  function substituteEnvVars(value: string): string {
619
653
  const envVarPattern = /\$\{([A-Z_][A-Z0-9_]*)\}/g;
620
654
 
@@ -60,6 +60,52 @@ function buildController(
60
60
  return { controller, runSQL, assertCanAdmitQuery };
61
61
  }
62
62
 
63
+ describe("ConnectionController connection queryMetadata", () => {
64
+ afterEach(() => sinon.restore());
65
+
66
+ it("rejects a connection default the contract cannot render", async () => {
67
+ // The connection is the one metadata layer whose author is right here, so
68
+ // it is the one that can be told. `cost-centre` is the natural spelling
69
+ // and would otherwise be dropped at dispatch, leaving every statement on
70
+ // the connection missing the property its operator configured.
71
+ const { controller } = buildController(sinon.stub().resolves());
72
+ await expect(
73
+ controller.addConnection("env", "warehouse", {
74
+ name: "warehouse",
75
+ type: "postgres",
76
+ queryMetadata: { "cost-centre": "eng" },
77
+ }),
78
+ ).rejects.toThrow(/queryMetadata is invalid/);
79
+ });
80
+
81
+ it("rejects an enforced bag the contract cannot render", async () => {
82
+ const { controller } = buildController(sinon.stub().resolves());
83
+ await expect(
84
+ controller.addConnection("env", "warehouse", {
85
+ name: "warehouse",
86
+ type: "postgres",
87
+ queryMetadataEnforced: { "org.id": "acme" },
88
+ }),
89
+ ).rejects.toThrow(/queryMetadataEnforced is invalid/);
90
+ });
91
+
92
+ it("accepts a conforming connection default", async () => {
93
+ const { controller } = buildController(sinon.stub().resolves());
94
+ const addConnection = sinon.stub().resolves();
95
+ (
96
+ controller as unknown as {
97
+ connectionService: { addConnection: sinon.SinonStub };
98
+ }
99
+ ).connectionService = { addConnection } as never;
100
+ await controller.addConnection("env", "warehouse", {
101
+ name: "warehouse",
102
+ type: "postgres",
103
+ queryMetadata: { cost_centre: "eng" },
104
+ });
105
+ expect(addConnection.calledOnce).toBe(true);
106
+ });
107
+ });
108
+
63
109
  describe("ConnectionController.getConnectionQueryData row cap", () => {
64
110
  const originalEnv = process.env.PUBLISHER_MAX_QUERY_ROWS;
65
111
 
@@ -21,6 +21,15 @@ import {
21
21
  getSchemasForConnection,
22
22
  listTablesForSchema,
23
23
  } from "../service/db_utils";
24
+ import {
25
+ mergeQueryMetadata,
26
+ mintCorrelationId,
27
+ parseQueryClass,
28
+ parseSuppliedQueryMetadata,
29
+ queryMetadataViolations,
30
+ type QueryClass,
31
+ type QueryMetadata,
32
+ } from "../service/query_metadata";
24
33
  import type { Environment } from "../service/environment";
25
34
  import { EnvironmentStore } from "../service/environment_store";
26
35
  import { isStreamingConnection, streamSqlWithBudget } from "../stream_helpers";
@@ -112,6 +121,22 @@ function validateAdminAuthoredConnection(
112
121
  } catch (error) {
113
122
  throw new BadRequestError((error as Error).message);
114
123
  }
124
+
125
+ // The connection is the one metadata layer whose author is right here, so it
126
+ // is the one that can be told. A property the contract rejects (a hyphen in
127
+ // `cost-centre` is the natural spelling and the first thing anyone tries)
128
+ // would otherwise be dropped at dispatch and only ever surface as a metric —
129
+ // every statement on the connection missing the property its operator
130
+ // believes they configured. Config LOAD warns instead of throwing (see
131
+ // assembleEnvironmentConnections): a tag must never fail an environment.
132
+ for (const field of ["queryMetadata", "queryMetadataEnforced"] as const) {
133
+ const violations = queryMetadataViolations(connectionConfig[field]);
134
+ if (violations.length > 0) {
135
+ throw new BadRequestError(
136
+ `Connection "${connectionName}" ${field} is invalid: ${violations.join("; ")}`,
137
+ );
138
+ }
139
+ }
115
140
  }
116
141
 
117
142
  export class ConnectionController {
@@ -140,6 +165,43 @@ export class ConnectionController {
140
165
  return environment.getApiConnection(connectionName);
141
166
  }
142
167
 
168
+ /**
169
+ * A connection's default per-query metadata, or null when it declares none.
170
+ * Fails open: metadata is observability, so a connection whose config can't be
171
+ * read contributes no default rather than failing the query the caller asked
172
+ * for.
173
+ */
174
+ private async connectionQueryMetadata(
175
+ environmentName: string,
176
+ connectionName: string,
177
+ ): Promise<{
178
+ default: QueryMetadata | null;
179
+ enforced: QueryMetadata | null;
180
+ }> {
181
+ try {
182
+ const environment = await this.environmentStore.getEnvironment(
183
+ environmentName,
184
+ false,
185
+ );
186
+ const connection = this.getApiConnectionForLookup(
187
+ environment,
188
+ connectionName,
189
+ );
190
+ return {
191
+ default: connection.queryMetadata ?? null,
192
+ enforced: connection.queryMetadataEnforced ?? null,
193
+ };
194
+ } catch (error) {
195
+ // Fails open like every other metadata path, but not invisibly: the
196
+ // layer lost here is the enforced one, and no metric covers it.
197
+ logger.debug("No query-metadata layers for connection", {
198
+ connectionName,
199
+ error,
200
+ });
201
+ return { default: null, enforced: null };
202
+ }
203
+ }
204
+
143
205
  private async getMalloyConnection(
144
206
  environmentName: string,
145
207
  connectionName: string,
@@ -453,6 +515,12 @@ export class ConnectionController {
453
515
  sqlStatement: string,
454
516
  options: string,
455
517
  packageName?: string,
518
+ /**
519
+ * The request's per-query metadata fields, unvalidated — this controller is
520
+ * the boundary that turns a bad bag into a 400 rather than letting the
521
+ * connector refuse the statement at dispatch.
522
+ */
523
+ metadata?: { queryMetadata?: unknown; queryClass?: unknown },
456
524
  ): Promise<ApiQueryData> {
457
525
  // Express parses repeated query parameters (?sqlStatement=a&sqlStatement=b)
458
526
  // and array-shaped JSON bodies as `string[]`, not `string`. The route
@@ -512,6 +580,41 @@ export class ConnectionController {
512
580
  runSQLOptions.abortSignal = undefined;
513
581
  }
514
582
 
583
+ // Per-query metadata. Validated here, not clamped: a raw-SQL caller gets a
584
+ // 400 telling it which property is wrong instead of a statement the
585
+ // connector refuses at dispatch. `options` is forwarded as RunSQLOptions, so
586
+ // a bag can also arrive inside it — validate that one too, and let the
587
+ // documented field win.
588
+ const suppliedMetadata =
589
+ metadata?.queryMetadata ?? runSQLOptions.queryMetadata;
590
+ let requestMetadata: QueryMetadata | undefined;
591
+ let queryClass: QueryClass | undefined;
592
+ try {
593
+ requestMetadata = parseSuppliedQueryMetadata(suppliedMetadata);
594
+ queryClass = parseQueryClass(metadata?.queryClass);
595
+ } catch (error) {
596
+ throw new BadRequestError((error as Error).message);
597
+ }
598
+ const connectionLayers = await this.connectionQueryMetadata(
599
+ environmentName,
600
+ connectionName,
601
+ );
602
+ const resolvedMetadata = mergeQueryMetadata({
603
+ connection: connectionLayers.default,
604
+ enforced: connectionLayers.enforced,
605
+ request: requestMetadata,
606
+ context: {
607
+ // Raw SQL against a connection is platform maintenance unless the
608
+ // caller says otherwise — it is not a modeled query.
609
+ queryClass: queryClass ?? "ops",
610
+ environment: environmentName,
611
+ package: packageName,
612
+ correlationId: mintCorrelationId(),
613
+ },
614
+ });
615
+ runSQLOptions.queryMetadata = resolvedMetadata.metadata;
616
+ const queryCorrelationId = resolvedMetadata.metadata?.query_id ?? null;
617
+
515
618
  // Bound the response with two layered caps:
516
619
  //
517
620
  // - Row cap (PUBLISHER_MAX_QUERY_ROWS) — pushed to the driver as
@@ -572,7 +675,7 @@ export class ConnectionController {
572
675
  throw new ConnectionError((error as Error).message);
573
676
  }
574
677
  }, getQueryTimeoutMs());
575
- return { data: JSON.stringify(streamed) };
678
+ return { data: JSON.stringify(streamed), queryCorrelationId };
576
679
  }
577
680
 
578
681
  const result = await runWithQueryTimeout(async (signal) => {
@@ -601,7 +704,7 @@ export class ConnectionController {
601
704
  );
602
705
  }
603
706
 
604
- return { data: JSON.stringify(result) };
707
+ return { data: JSON.stringify(result), queryCorrelationId };
605
708
  }
606
709
 
607
710
  public async getConnectionTemporaryTable(
@@ -265,4 +265,29 @@ describe("MaterializationController.createMaterialization validation", () => {
265
265
  }),
266
266
  ).rejects.toThrow(BadRequestError);
267
267
  });
268
+
269
+ it("rejects a runId the metadata contract cannot carry", async () => {
270
+ // runId becomes the `run_id` property on every statement of the build, so
271
+ // an over-long or unrenderable value is silently truncated and rewritten
272
+ // at dispatch — leaving the caller holding an id that joins to nothing.
273
+ // The neighbouring `trigger` is enum-validated; this is the same kind of
274
+ // boundary and gets the same treatment.
275
+ const { controller } = build();
276
+ await expect(
277
+ controller.createMaterialization("env", "pkg", {
278
+ runContext: { runId: "x".repeat(300) },
279
+ }),
280
+ ).rejects.toThrow(BadRequestError);
281
+ await expect(
282
+ controller.createMaterialization("env", "pkg", {
283
+ runContext: { runId: 'has "quotes"' },
284
+ }),
285
+ ).rejects.toThrow(BadRequestError);
286
+ });
287
+
288
+ it("still accepts a conforming runId", async () => {
289
+ expect(
290
+ await parse({ runContext: { trigger: "publish", runId: "run-42" } }),
291
+ ).toEqual({ runContext: { trigger: "publish", runId: "run-42" } });
292
+ });
268
293
  });
@@ -1,9 +1,15 @@
1
+ import type { components } from "../api";
1
2
  import { BadRequestError } from "../errors";
2
3
  import {
3
4
  BuildInstruction,
4
5
  ManifestReference,
5
6
  } from "../storage/DatabaseInterface";
6
7
  import { MaterializationService } from "../service/materialization_service";
8
+ import { queryMetadataViolations } from "../service/query_metadata";
9
+
10
+ type RunContext = components["schemas"]["RunContext"];
11
+
12
+ const RUN_TRIGGERS = ["publish", "on_demand", "scheduler"] as const;
7
13
 
8
14
  export class MaterializationController {
9
15
  constructor(private materializationService: MaterializationService) {}
@@ -31,6 +37,7 @@ export class MaterializationController {
31
37
  buildInstructions?: BuildInstruction[];
32
38
  referenceManifest?: ManifestReference[];
33
39
  strictUpstreams?: boolean;
40
+ runContext?: RunContext;
34
41
  } {
35
42
  const result: {
36
43
  forceRefresh?: boolean;
@@ -38,7 +45,11 @@ export class MaterializationController {
38
45
  buildInstructions?: BuildInstruction[];
39
46
  referenceManifest?: ManifestReference[];
40
47
  strictUpstreams?: boolean;
48
+ runContext?: RunContext;
41
49
  } = {};
50
+ if (body.runContext !== undefined && body.runContext !== null) {
51
+ result.runContext = this.validateRunContext(body.runContext);
52
+ }
42
53
  if (
43
54
  body.buildInstructions !== undefined &&
44
55
  body.buildInstructions !== null
@@ -72,6 +83,55 @@ export class MaterializationController {
72
83
  return result;
73
84
  }
74
85
 
86
+ /**
87
+ * Validate `runContext`, the caller's observability context for one run.
88
+ * `trigger` is a closed enum here even though it feeds a metadata property:
89
+ * the whole point is that a reader can group runs by how they started, which a
90
+ * free-form value would quietly break.
91
+ *
92
+ * Note this is NOT the service-level `trigger` the parser above deliberately
93
+ * refuses. That one decides whether the run counts as scheduled; this one only
94
+ * labels the statements the run issues, so accepting `publish` from a caller
95
+ * forges nothing.
96
+ */
97
+ private validateRunContext(raw: unknown): RunContext {
98
+ if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
99
+ throw new BadRequestError("runContext must be an object");
100
+ }
101
+ const obj = raw as Record<string, unknown>;
102
+ const context: RunContext = {};
103
+ if (obj.trigger !== undefined && obj.trigger !== null) {
104
+ if (
105
+ typeof obj.trigger !== "string" ||
106
+ !RUN_TRIGGERS.includes(obj.trigger as (typeof RUN_TRIGGERS)[number])
107
+ ) {
108
+ throw new BadRequestError(
109
+ `runContext.trigger must be one of ${RUN_TRIGGERS.join(" | ")}`,
110
+ );
111
+ }
112
+ context.trigger = obj.trigger as RunContext["trigger"];
113
+ }
114
+ if (obj.runId !== undefined && obj.runId !== null) {
115
+ if (typeof obj.runId !== "string") {
116
+ throw new BadRequestError("runContext.runId must be a string");
117
+ }
118
+ // Held to the metadata contract like any other caller-supplied
119
+ // property: it becomes the `run_id` on every statement of the build,
120
+ // and a value the contract rejects would otherwise be truncated and
121
+ // rewritten in silence — leaving the caller with an id it cannot join
122
+ // on and no way to know why.
123
+ const violations = queryMetadataViolations({ run_id: obj.runId });
124
+ if (violations.length > 0) {
125
+ throw new BadRequestError(
126
+ `runContext.runId is attached to every statement as the run_id ` +
127
+ `property: ${violations.join("; ")}`,
128
+ );
129
+ }
130
+ context.runId = obj.runId;
131
+ }
132
+ return context;
133
+ }
134
+
75
135
  /**
76
136
  * Validate the orchestrated `buildInstructions` payload (BuildInstructions:
77
137
  * `{ sources: BuildInstruction[], referenceManifest?, strictUpstreams? }`)
@@ -1,6 +1,7 @@
1
1
  import { components } from "../api";
2
2
  import { getQueryTimeoutMs } from "../config";
3
3
  import { ModelNotFoundError } from "../errors";
4
+ import { logger } from "../logger";
4
5
  import { runWithQueryTimeout } from "../query_timeout";
5
6
  import { EnvironmentStore } from "../service/environment_store";
6
7
  import type { FilterParams } from "../service/filter";
@@ -133,6 +134,29 @@ export class ModelController {
133
134
  bypassFilters,
134
135
  givens,
135
136
  abortSignal,
137
+ {
138
+ environment: environmentName,
139
+ // The environment owns the connection configs, so the default
140
+ // and enforced layers are read here rather than from the model.
141
+ connectionMetadata: (connectionName) => {
142
+ try {
143
+ const connection =
144
+ environment.getApiConnection(connectionName);
145
+ return {
146
+ default: connection.queryMetadata,
147
+ enforced: connection.queryMetadataEnforced,
148
+ };
149
+ } catch (error) {
150
+ // Fails open, and says so: what an unreadable
151
+ // connection costs is the enforced layer.
152
+ logger.debug(
153
+ "No query-metadata layers for connection",
154
+ { connectionName, error },
155
+ );
156
+ return null;
157
+ }
158
+ },
159
+ },
136
160
  ),
137
161
  getQueryTimeoutMs(),
138
162
  );
@@ -2,7 +2,16 @@ import { validateRenderTags } from "@malloydata/render-validator";
2
2
  import { components } from "../api";
3
3
  import { getQueryTimeoutMs } from "../config";
4
4
  import { API_PREFIX } from "../constants";
5
- import { ModelNotFoundError } from "../errors";
5
+ import { BadRequestError, ModelNotFoundError } from "../errors";
6
+ import { bigIntReplacer } from "../json_utils";
7
+ import { logger } from "../logger";
8
+ import {
9
+ mintCorrelationId,
10
+ parseQueryClass,
11
+ parseSuppliedQueryMetadata,
12
+ type QueryClass,
13
+ type QueryMetadata,
14
+ } from "../service/query_metadata";
6
15
  import { runWithQueryTimeout } from "../query_timeout";
7
16
  import { EnvironmentStore } from "../service/environment_store";
8
17
  import type { FilterParams } from "../service/filter";
@@ -10,14 +19,6 @@ import type { GivenValue } from "@malloydata/malloy";
10
19
 
11
20
  type ApiQuery = components["schemas"]["QueryResult"];
12
21
 
13
- // Replacer function to handle BigInt serialization
14
- function bigIntReplacer(_key: string, value: unknown): unknown {
15
- if (typeof value === "bigint") {
16
- return Number(value);
17
- }
18
- return value;
19
- }
20
-
21
22
  export class QueryController {
22
23
  private environmentStore: EnvironmentStore;
23
24
 
@@ -36,7 +37,26 @@ export class QueryController {
36
37
  filterParams?: FilterParams,
37
38
  bypassFilters?: boolean,
38
39
  givens?: Record<string, GivenValue>,
40
+ /**
41
+ * The request's per-query metadata fields, unvalidated — this controller is
42
+ * the boundary that turns a bad bag into a 400 rather than letting the
43
+ * connector refuse the statement at dispatch.
44
+ */
45
+ metadata?: {
46
+ queryMetadata?: unknown;
47
+ queryClass?: unknown;
48
+ versionId?: string;
49
+ },
39
50
  ): Promise<ApiQuery> {
51
+ let requestMetadata: QueryMetadata | undefined;
52
+ let queryClass: QueryClass | undefined;
53
+ try {
54
+ requestMetadata = parseSuppliedQueryMetadata(metadata?.queryMetadata);
55
+ queryClass = parseQueryClass(metadata?.queryClass);
56
+ } catch (error) {
57
+ throw new BadRequestError((error as Error).message);
58
+ }
59
+
40
60
  const environment = await this.environmentStore.getEnvironment(
41
61
  environmentName,
42
62
  false,
@@ -53,7 +73,13 @@ export class QueryController {
53
73
  if (!model) {
54
74
  throw new ModelNotFoundError(`${modelPath} does not exist`);
55
75
  } else {
56
- const { result, compactResult } = await runWithQueryTimeout(
76
+ const {
77
+ result,
78
+ compactResult,
79
+ rowLimit,
80
+ rowLimitSource,
81
+ queryCorrelationId,
82
+ } = await runWithQueryTimeout(
57
83
  (abortSignal) =>
58
84
  model.getQueryResults(
59
85
  sourceName,
@@ -63,6 +89,41 @@ export class QueryController {
63
89
  bypassFilters,
64
90
  givens,
65
91
  abortSignal,
92
+ {
93
+ request: requestMetadata,
94
+ queryClass,
95
+ environment: environmentName,
96
+ // Always undefined today: the route 501s any versionId
97
+ // before this runs. Wired so that lifting that rejection
98
+ // is the whole change.
99
+ version: metadata?.versionId,
100
+ // Minted here because this is the boundary that returns
101
+ // it; a path with nowhere to put it does not mint one.
102
+ correlationId: mintCorrelationId(),
103
+ // The environment owns the connection configs, so the
104
+ // default and enforced layers are read here rather than
105
+ // from the model.
106
+ connectionMetadata: (connectionName) => {
107
+ try {
108
+ const connection =
109
+ environment.getApiConnection(connectionName);
110
+ return {
111
+ default: connection.queryMetadata,
112
+ enforced: connection.queryMetadataEnforced,
113
+ };
114
+ } catch (error) {
115
+ // Failing open is right — a tag must not fail a
116
+ // query — but this is the one drop with no metric
117
+ // behind it, and what it costs is the ENFORCED
118
+ // layer. Log it so it is diagnosable.
119
+ logger.debug(
120
+ "No query-metadata layers for connection",
121
+ { connectionName, error },
122
+ );
123
+ return null;
124
+ }
125
+ },
126
+ },
66
127
  ),
67
128
  getQueryTimeoutMs(),
68
129
  );
@@ -73,6 +134,18 @@ export class QueryController {
73
134
  : JSON.stringify(result),
74
135
  resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}/models/${modelPath}/query`,
75
136
  renderLogs: renderLogs.length > 0 ? renderLogs : undefined,
137
+ // The cap the database applied. A caller counting the rows it got
138
+ // back cannot otherwise tell a complete result from one the limit
139
+ // cut off: a query with no LIMIT of its own silently gets the
140
+ // server default, which is well under the hard ceiling, so nothing
141
+ // raises. Deriving it client-side is not possible, since it depends
142
+ // on server config and on the query's own LIMIT.
143
+ queryRowLimit: rowLimit,
144
+ // Which of the two that cap was. Without it a caller cannot tell a
145
+ // deliberate `limit:`/`top:` from the silently-applied default, and
146
+ // so cannot reproduce the MCP envelope's _limit_hit.
147
+ queryRowLimitSource: rowLimitSource,
148
+ queryCorrelationId,
76
149
  } as ApiQuery;
77
150
  }
78
151
  }
@@ -0,0 +1,51 @@
1
+ import { describe, expect, it } from "bun:test";
2
+ import { bigIntReplacer } from "./json_utils";
3
+
4
+ const ser = (v: unknown) => JSON.stringify(v, bigIntReplacer);
5
+
6
+ describe("bigIntReplacer", () => {
7
+ it("still renders ordinary BigInt ids as numbers", () => {
8
+ // The common case, and the one that must not change: a DuckDB BIGINT
9
+ // column of ordinary ids has always reached callers as JSON numbers, and
10
+ // the MCP rows are byte-compared against the REST payload.
11
+ expect(ser({ id: 100001n })).toBe('{"id":100001}');
12
+ expect(ser({ c: 0n })).toBe('{"c":0}');
13
+ expect(ser({ c: -42n })).toBe('{"c":-42}');
14
+ });
15
+
16
+ it("keeps the largest exactly-representable values as numbers", () => {
17
+ expect(ser({ v: 9007199254740991n })).toBe('{"v":9007199254740991}');
18
+ expect(ser({ v: -9007199254740991n })).toBe('{"v":-9007199254740991}');
19
+ });
20
+
21
+ /**
22
+ * The regression this guards. Number() rounds past 2^53, so an id came back
23
+ * altered with no error: an agent filtering or joining on it gets a wrong or
24
+ * empty answer. Malloy's own wrapResult keeps an exact string for the same
25
+ * case, so a string here matches what the typed-cell path already did.
26
+ */
27
+ it("preserves values past the safe range exactly, as strings", () => {
28
+ expect(ser({ id: 9007199254740993n })).toBe('{"id":"9007199254740993"}');
29
+ expect(ser({ id: 1234567890123456789n })).toBe(
30
+ '{"id":"1234567890123456789"}',
31
+ );
32
+ expect(ser({ id: -9007199254740993n })).toBe(
33
+ '{"id":"-9007199254740993"}',
34
+ );
35
+ });
36
+
37
+ it("round-trips a large id without losing a digit", () => {
38
+ const exact = "9223372036854775807"; // int64 max
39
+ expect(JSON.parse(ser({ id: BigInt(exact) })).id).toBe(exact);
40
+ });
41
+
42
+ it("leaves non-BigInt values alone", () => {
43
+ expect(ser({ a: 1, b: "x", c: null, d: 1.5 })).toBe(
44
+ '{"a":1,"b":"x","c":null,"d":1.5}',
45
+ );
46
+ });
47
+
48
+ it("does not throw on the BigInt a plain stringify would reject", () => {
49
+ expect(() => ser({ c: 150930n })).not.toThrow();
50
+ });
51
+ });
@@ -0,0 +1,33 @@
1
+ /**
2
+ * JSON.stringify replacer for raw driver output, which can carry BigInt.
3
+ *
4
+ * Load-bearing wherever `compactResult` is serialized: it is
5
+ * `queryResults.data.value` straight from the connector, and a DuckDB BIGINT
6
+ * column arrives as a JS BigInt, so a plain JSON.stringify throws
7
+ * `TypeError: Do not know how to serialize a BigInt`. The wrapped
8
+ * `Malloy.Result` path does not need this, because API.util.wrapResult has
9
+ * already normalized values into typed cells.
10
+ *
11
+ * Values inside the safe integer range become numbers, which is what callers
12
+ * have always received and what keeps the MCP and REST payloads byte-identical.
13
+ * Outside it `Number()` rounds silently: 9007199254740993 becomes ...992, and an
14
+ * agent that filters or joins on that id gets a wrong or empty answer with no
15
+ * error. Those are emitted as strings instead, which is what Malloy itself does
16
+ * for the same case (API.util.wrapResult attaches an exact `string_value`
17
+ * beside the lossy `number_value` whenever the subtype is bigint).
18
+ *
19
+ * So the JSON type varies with magnitude. That is deliberate: a column only
20
+ * yields a string for a value a JSON number cannot represent, and every such
21
+ * value was previously being corrupted, so nothing a caller correctly receives
22
+ * today changes.
23
+ */
24
+ const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);
25
+
26
+ export function bigIntReplacer(_key: string, value: unknown): unknown {
27
+ if (typeof value === "bigint") {
28
+ return value > MAX_SAFE_BIGINT || value < -MAX_SAFE_BIGINT
29
+ ? value.toString()
30
+ : Number(value);
31
+ }
32
+ return value;
33
+ }