@malloy-publisher/server 0.0.233 → 0.0.234

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/README.docker.md +1 -0
  2. package/dist/app/api-doc.yaml +208 -0
  3. package/dist/app/assets/{EnvironmentPage-DutP7T8h.js → EnvironmentPage-DTZQ4Gxc.js} +1 -1
  4. package/dist/app/assets/{HomePage-BcxDrBfl.js → HomePage-C5mlDPXK.js} +1 -1
  5. package/dist/app/assets/{LightMode-BJukGxgz.js → LightMode-DGNmhG0u.js} +1 -1
  6. package/dist/app/assets/{MainPage-DXbwlMeF.js → MainPage-CVL_wmP4.js} +2 -2
  7. package/dist/app/assets/{MaterializationsPage-BBQksmTU.js → MaterializationsPage-DmzMBCpy.js} +1 -1
  8. package/dist/app/assets/{ModelPage-C6tK51uU.js → ModelPage-Dbvf4QbB.js} +1 -1
  9. package/dist/app/assets/{PackagePage-Bo3cwwZE.js → PackagePage-DxdHc2Qs.js} +1 -1
  10. package/dist/app/assets/{RouteError-BufkcAKE.js → RouteError-OJdT4tCd.js} +1 -1
  11. package/dist/app/assets/{ThemeEditorPage-DICvvKpa.js → ThemeEditorPage-Bk7s0KXY.js} +1 -1
  12. package/dist/app/assets/{WorkbookPage-Dkwt75Nj.js → WorkbookPage-j_vCWdN3.js} +1 -1
  13. package/dist/app/assets/{core-C0nunIQT.es-DlMLKZBK.js → core-Rj_4rRnA.es-DoIfLxDJ.js} +1 -1
  14. package/dist/app/assets/{index-CmEVVe-8.js → index-B_jKMR35.js} +4 -4
  15. package/dist/app/assets/{index-BusxL5Pt.js → index-D-rDyK11.js} +1 -1
  16. package/dist/app/assets/{index-Cs4WVm2z.js → index-DWIe_hK0.js} +1 -1
  17. package/dist/app/assets/{index-qnhU9CGo.js → index-hw-xn0X7.js} +1 -1
  18. package/dist/app/index.html +1 -1
  19. package/dist/package_load_worker.mjs +53 -3
  20. package/dist/server.mjs +835 -97
  21. package/package.json +1 -1
  22. package/src/config.ts +35 -1
  23. package/src/controller/connection.controller.spec.ts +46 -0
  24. package/src/controller/connection.controller.ts +105 -2
  25. package/src/controller/materialization.controller.spec.ts +25 -0
  26. package/src/controller/materialization.controller.ts +60 -0
  27. package/src/controller/model.controller.ts +24 -0
  28. package/src/controller/query.controller.ts +83 -15
  29. package/src/mcp/handler_utils.ts +10 -2
  30. package/src/mcp/query_envelope.ts +10 -0
  31. package/src/mcp/skills/skills_bundle.json +1 -1
  32. package/src/mcp/tools/execute_query_tool.spec.ts +131 -0
  33. package/src/mcp/tools/execute_query_tool.ts +62 -25
  34. package/src/mcp_config.spec.ts +919 -0
  35. package/src/mcp_config.ts +425 -0
  36. package/src/oom_guards.integration.spec.ts +11 -3
  37. package/src/package_load/package_load_pool.ts +2 -0
  38. package/src/package_load/package_load_worker.ts +17 -5
  39. package/src/package_load/protocol.ts +6 -0
  40. package/src/query_metadata_metrics.ts +49 -0
  41. package/src/server.ts +99 -3
  42. package/src/service/build_plan.spec.ts +125 -0
  43. package/src/service/build_plan.ts +108 -7
  44. package/src/service/connection_config.ts +49 -0
  45. package/src/service/environment.ts +57 -3
  46. package/src/service/materialization_config_validation.spec.ts +99 -0
  47. package/src/service/materialization_config_validation.ts +120 -0
  48. package/src/service/materialization_schedule_surface.spec.ts +124 -0
  49. package/src/service/materialization_service.spec.ts +119 -0
  50. package/src/service/materialization_service.ts +186 -3
  51. package/src/service/materialization_test_fixtures.ts +86 -21
  52. package/src/service/model.spec.ts +45 -1
  53. package/src/service/model.ts +145 -19
  54. package/src/service/package.ts +24 -1
  55. package/src/service/package_manifest.spec.ts +137 -4
  56. package/src/service/package_manifest.ts +140 -5
  57. package/src/service/persist_annotation_validation.spec.ts +12 -0
  58. package/src/service/persist_annotation_validation.ts +9 -4
  59. package/src/service/query_metadata.spec.ts +408 -0
  60. package/src/service/query_metadata.ts +492 -0
  61. package/src/service/query_metadata_identity.spec.ts +149 -0
package/src/server.ts CHANGED
@@ -41,6 +41,7 @@ import {
41
41
  getMemoryGovernorConfig,
42
42
  getPersistCollisionEnforce,
43
43
  getPersistStorageMode,
44
+ getQueryMetadataMode,
44
45
  } from "./config";
45
46
  import { setFilterDeprecationHeaders } from "./filter_deprecation";
46
47
  import { checkHeapConfiguration } from "./heap_check";
@@ -48,6 +49,16 @@ import { queryConcurrency } from "./query_concurrency";
48
49
  import { MaterializationController } from "./controller/materialization.controller";
49
50
  import { ThemeController } from "./controller/theme.controller";
50
51
  import { initializeMcpServer } from "./mcp/server";
52
+ import {
53
+ addCommand,
54
+ ensureMcpConfig,
55
+ logMcpConfigOutcome,
56
+ MCP_CONFIG_FILENAME,
57
+ mcpConfigEnabled,
58
+ mcpEndpoint,
59
+ resolveBoundPort,
60
+ resolveClientHost,
61
+ } from "./mcp_config";
51
62
  import { registerLegacyRoutes } from "./server-old";
52
63
  import { EnvironmentStore } from "./service/environment_store";
53
64
  import { MaterializationScheduler } from "./service/materialization_scheduler";
@@ -95,6 +106,8 @@ function parseArgs() {
95
106
  i++;
96
107
  } else if (arg === "--init") {
97
108
  process.env.INITIALIZE_STORAGE = "true";
109
+ } else if (arg === "--no-mcp-config") {
110
+ process.env.PUBLISHER_NO_MCP_CONFIG = "true";
98
111
  } else if (arg === "--watch-env" && args[i + 1]) {
99
112
  // Append (don't overwrite) so multiple --watch-env flags compose
100
113
  // and so an explicit env var pre-set still wins.
@@ -133,6 +146,9 @@ function parseArgs() {
133
146
  console.log(
134
147
  " --init Wipe persisted storage and re-sync it from the config (default: false)",
135
148
  );
149
+ console.log(
150
+ " --no-mcp-config Do not write .mcp.json into the working directory (default: it is written, so an agent opened here finds this server; skipped when the directory already has one, is your home directory or the filesystem root, is inside a git working tree, or the MCP port bound is not the one requested)",
151
+ );
136
152
  console.log(
137
153
  " --watch-env <name> Enable dev-mode watch for the named environment.",
138
154
  );
@@ -177,9 +193,20 @@ getPersistStorageMode();
177
193
  // than a failed boot.
178
194
  getPersistCollisionEnforce();
179
195
 
196
+ // Same hazard, wider blast radius: getQueryMetadataMode() throws on an invalid
197
+ // value and is read while resolving EVERY statement, so a typo'd off switch
198
+ // ("false", "0", "disabled") would boot clean and then fail every query and
199
+ // every build — the one thing the metadata path promises never to do.
200
+ getQueryMetadataMode();
201
+
180
202
  const PUBLISHER_PORT = Number(process.env.PUBLISHER_PORT || 4000);
181
203
  const PUBLISHER_HOST = process.env.PUBLISHER_HOST || "0.0.0.0";
182
204
  const MCP_PORT = Number(process.env.MCP_PORT || 4040);
205
+ // Resolved here rather than in the listen callback: parseBoolEnv throws on a
206
+ // typo, which is the convention for flags in this server, but a throw inside a
207
+ // listen callback is an uncaughtException that kills a server which has already
208
+ // bound both ports. At module scope it is an ordinary startup failure.
209
+ const MCP_CONFIG_ENABLED = mcpConfigEnabled();
183
210
  const MCP_ENDPOINT = "/mcp";
184
211
  const SHUTDOWN_DRAIN_DURATION_SECONDS = Number(
185
212
  process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0,
@@ -1273,6 +1300,11 @@ app.post(
1273
1300
  req.params.connectionName,
1274
1301
  req.body.sqlStatement as string,
1275
1302
  req.body?.options as string,
1303
+ undefined,
1304
+ {
1305
+ queryMetadata: req.body?.queryMetadata,
1306
+ queryClass: req.body?.queryClass,
1307
+ },
1276
1308
  ),
1277
1309
  );
1278
1310
  } catch (error) {
@@ -1295,6 +1327,10 @@ app.post(
1295
1327
  req.body.sqlStatement as string,
1296
1328
  req.body?.options as string,
1297
1329
  req.params.packageName,
1330
+ {
1331
+ queryMetadata: req.body?.queryMetadata,
1332
+ queryClass: req.body?.queryClass,
1333
+ },
1298
1334
  ),
1299
1335
  );
1300
1336
  } catch (error) {
@@ -1664,6 +1700,11 @@ app.post(
1664
1700
  | undefined,
1665
1701
  req.body.bypassFilters === true ? true : undefined,
1666
1702
  req.body.givens as Record<string, GivenValue> | undefined,
1703
+ {
1704
+ queryMetadata: req.body?.queryMetadata,
1705
+ queryClass: req.body?.queryClass,
1706
+ versionId: req.body?.versionId as string | undefined,
1707
+ },
1667
1708
  );
1668
1709
  setFilterDeprecationHeaders(res, {
1669
1710
  filterParams: req.body.filterParams ?? req.body.sourceFilters,
@@ -1947,9 +1988,64 @@ mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, async () => {
1947
1988
  }
1948
1989
  }
1949
1990
  });
1950
- const mcpServer = mcpApp.listen(MCP_PORT, PUBLISHER_HOST, () => {
1951
- logger.info(`MCP server listening at http://${PUBLISHER_HOST}:${MCP_PORT}`);
1952
- });
1991
+ const mcpServer = mcpApp.listen(
1992
+ MCP_PORT,
1993
+ PUBLISHER_HOST,
1994
+ function (this: import("net").Server) {
1995
+ // Read back rather than reusing MCP_PORT, which is only what was requested.
1996
+ // `--mcp_port 0` asks for any free port, and under bun a non-numeric value
1997
+ // binds an ephemeral one too, so the requested value can be 0 or NaN while
1998
+ // a real port is listening. The listening line uses it as well, which is
1999
+ // why it no longer reads `http://127.0.0.1:0`.
2000
+ const boundPort = resolveBoundPort(this.address(), MCP_PORT);
2001
+ // The BIND address, bracketed when it is an IPv6 literal so the URL
2002
+ // parses. Deliberately not resolveClientHost: create-malloy-package's
2003
+ // README and AGENTS template both tell readers these two listening lines
2004
+ // are "the addresses it really bound", and use them to catch a mistyped
2005
+ // --hostt that silently falls back to 0.0.0.0. Mapping the wildcard to
2006
+ // loopback here would confirm the mistake instead of revealing it. The
2007
+ // dialable form belongs in .mcp.json and in the advice, not here.
2008
+ const bound = this.address();
2009
+ const boundHost =
2010
+ typeof bound === "object" && bound ? bound.address : PUBLISHER_HOST;
2011
+ logger.info(
2012
+ `MCP server listening at http://${boundHost.includes(":") ? `[${boundHost}]` : boundHost}:${boundPort}`,
2013
+ );
2014
+ // Checked before process.cwd(), which can throw: someone who turned the
2015
+ // feature off should not get a warning about it.
2016
+ if (MCP_CONFIG_ENABLED) {
2017
+ // ensureMcpConfig cannot throw, but its arguments can: process.cwd()
2018
+ // raises ENOENT once the working directory has been removed. A throw
2019
+ // here is an uncaught exception inside a listen callback, which would
2020
+ // kill a server that has already bound both ports. Everything the call
2021
+ // needs is built inside the try for that reason, including the
2022
+ // endpoint: it is the newest and least-exercised code in this block.
2023
+ try {
2024
+ // The host an agent should dial, which is NOT `localhost`: that name
2025
+ // resolves to both loopback families while the server binds only one,
2026
+ // so another local process can hold the same port on the other family
2027
+ // and receive the agent's traffic instead.
2028
+ const endpoint = mcpEndpoint(
2029
+ resolveClientHost(this.address(), PUBLISHER_HOST),
2030
+ boundPort,
2031
+ );
2032
+ // cwd, not server_root: the file is for whoever opens an agent here.
2033
+ logMcpConfigOutcome(
2034
+ ensureMcpConfig({
2035
+ dir: process.cwd(),
2036
+ endpoint,
2037
+ requestedPort: MCP_PORT,
2038
+ boundPort,
2039
+ }),
2040
+ );
2041
+ } catch (error) {
2042
+ logger.info(
2043
+ `Could not set up ${MCP_CONFIG_FILENAME} (${error instanceof Error ? error.message : String(error)}). To connect an agent, run: ${addCommand(mcpEndpoint(resolveClientHost(this.address(), PUBLISHER_HOST), boundPort))}`,
2044
+ );
2045
+ }
2046
+ }
2047
+ },
2048
+ );
1953
2049
 
1954
2050
  mcpServer.timeout = 600000;
1955
2051
  mcpServer.keepAliveTimeout = 600000;
@@ -12,6 +12,7 @@ import {
12
12
  iterGraphSources,
13
13
  projectToPublicColumns,
14
14
  resolveFreshness,
15
+ resolveQueryMetadata,
15
16
  resolvePackageConnections,
16
17
  } from "./build_plan";
17
18
  import { MaterializationEligibilityError } from "../errors";
@@ -419,6 +420,130 @@ describe("resolveFreshness", () => {
419
420
  });
420
421
  expect(resolveFreshness(source, null)).toEqual({ window: "1h" });
421
422
  });
423
+
424
+ it("reads the model-file `materialization` envelope", () => {
425
+ const source = fakeSource({
426
+ name: "s",
427
+ sourceEntityId: "bid",
428
+ modelMaterialization: { freshness: { freshness: { window: "12h" } } },
429
+ });
430
+ expect(resolveFreshness(source, null)).toEqual({ window: "12h" });
431
+ });
432
+
433
+ it("prefers the envelope over the deprecated bare model-file form", () => {
434
+ const source = fakeSource({
435
+ name: "s",
436
+ sourceEntityId: "bid",
437
+ modelFreshnessSchedule: { freshness: { window: "48h" } },
438
+ modelMaterialization: { freshness: { freshness: { window: "12h" } } },
439
+ });
440
+ expect(resolveFreshness(source, null)).toEqual({ window: "12h" });
441
+ });
442
+
443
+ it("still reads a bare model-file knob the envelope does not declare", () => {
444
+ // A package published before the envelope existed keeps resolving, and the
445
+ // envelope does not hide the knobs it says nothing about.
446
+ const source = fakeSource({
447
+ name: "s",
448
+ sourceEntityId: "bid",
449
+ modelFreshnessSchedule: { freshness: { fallback: "fail" } },
450
+ modelMaterialization: { freshness: { freshness: { window: "12h" } } },
451
+ });
452
+ expect(resolveFreshness(source, null)).toEqual({
453
+ window: "12h",
454
+ fallback: "fail",
455
+ });
456
+ });
457
+ });
458
+
459
+ describe("resolveQueryMetadata", () => {
460
+ it("returns null when no layer declares anything", () => {
461
+ const source = fakeSource({ name: "s", sourceEntityId: "bid" });
462
+ expect(resolveQueryMetadata(source, null)).toBeNull();
463
+ expect(
464
+ resolveQueryMetadata(source, {
465
+ schedule: null,
466
+ freshness: null,
467
+ queryMetadata: null,
468
+ }),
469
+ ).toBeNull();
470
+ });
471
+
472
+ it("reads the source's `#@ persist queryMetadata.*` properties", () => {
473
+ const source = fakeSource({
474
+ name: "s",
475
+ sourceEntityId: "bid",
476
+ queryMetadata: { team: "finance", workload: "orders" },
477
+ });
478
+ expect(resolveQueryMetadata(source, null)).toEqual({
479
+ team: "finance",
480
+ workload: "orders",
481
+ });
482
+ });
483
+
484
+ it("resolves most-specific-wins PER PROPERTY across all four layers", () => {
485
+ // Package declares team+tier, the model-file envelope overrides tier and
486
+ // adds one of its own, the source overrides only workload: every property
487
+ // nothing more specific overrides has to survive.
488
+ const source = fakeSource({
489
+ name: "s",
490
+ sourceEntityId: "bid",
491
+ queryMetadata: { workload: "orders" },
492
+ modelMaterialization: {
493
+ queryMetadata: { tier: "gold", surface: "marts" },
494
+ },
495
+ });
496
+ expect(
497
+ resolveQueryMetadata(source, {
498
+ schedule: null,
499
+ freshness: null,
500
+ queryMetadata: { team: "finance", tier: "bronze" },
501
+ }),
502
+ ).toEqual({
503
+ team: "finance",
504
+ tier: "gold",
505
+ surface: "marts",
506
+ workload: "orders",
507
+ });
508
+ });
509
+
510
+ it("prefers the model-file envelope over the bare form, per property", () => {
511
+ const source = fakeSource({
512
+ name: "s",
513
+ sourceEntityId: "bid",
514
+ modelQueryMetadata: { tier: "bronze", legacy: "kept" },
515
+ modelMaterialization: { queryMetadata: { tier: "gold" } },
516
+ });
517
+ expect(resolveQueryMetadata(source, null)).toEqual({
518
+ tier: "gold",
519
+ legacy: "kept",
520
+ });
521
+ });
522
+
523
+ it("keeps a contract-violating property for the validator to report", () => {
524
+ const source = fakeSource({
525
+ name: "s",
526
+ sourceEntityId: "bid",
527
+ queryMetadata: { "team.name": "finance" },
528
+ });
529
+ expect(resolveQueryMetadata(source, null)).toEqual({
530
+ "team.name": "finance",
531
+ });
532
+ });
533
+
534
+ it("does not confuse the scalar `#@ persist` fields beside it", () => {
535
+ const source = fakeSource({
536
+ name: "s",
537
+ sourceEntityId: "bid",
538
+ annotationFields: { name: "s_table", refresh: "full" },
539
+ queryMetadata: { team: "finance" },
540
+ });
541
+ expect(resolveQueryMetadata(source, null)).toEqual({ team: "finance" });
542
+ expect(deriveAnnotationFields(source)).toEqual({
543
+ name: "s_table",
544
+ refresh: "full",
545
+ });
546
+ });
422
547
  });
423
548
 
424
549
  describe("deriveBuildPlan freshness", () => {
@@ -27,6 +27,7 @@ type BuildPlan = components["schemas"]["BuildPlan"];
27
27
  type WireFreshness = components["schemas"]["Freshness"];
28
28
  type WirePackageMaterialization =
29
29
  components["schemas"]["PackageMaterializationConfig"];
30
+ type QueryMetadata = components["schemas"]["QueryMetadata"];
30
31
 
31
32
  /** The freshness `fallback` values the publisher recognizes; others are dropped. */
32
33
  const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
@@ -38,9 +39,15 @@ interface FreshnessLayer {
38
39
  fallback?: FreshnessFallback;
39
40
  }
40
41
 
41
- /** Minimal path-reader over a Malloy `Tag` (see `@malloydata/malloy-tag`). */
42
+ /**
43
+ * Minimal reader over a Malloy `Tag` (see `@malloydata/malloy-tag`): scalar
44
+ * reads by path, plus the subtree read that a property collection like
45
+ * `queryMetadata { … }` needs.
46
+ */
42
47
  interface ReadableTag {
43
48
  text(...path: string[]): string | undefined;
49
+ tag(...path: string[]): ReadableTag | undefined;
50
+ entries?(): Iterable<[string, { text(): string | undefined }]>;
44
51
  }
45
52
 
46
53
  /**
@@ -238,6 +245,29 @@ function tagFreshnessLayer(tag: ReadableTag | undefined): FreshnessLayer {
238
245
  return layer;
239
246
  }
240
247
 
248
+ /**
249
+ * The two homes a model-file (`##`) knob can be declared in, most specific
250
+ * first: the `materialization` envelope, then the bare form.
251
+ *
252
+ * The model-file level mirrors the manifest's shape verbatim — `##
253
+ * materialization.freshness.window="24h"` is the manifest's block in tag syntax
254
+ * — so that is where a file-level reader looks. The bare form
255
+ * (`## freshness.window="24h"`), which shipped first, stays readable underneath:
256
+ * the annotation rides the published package, so a package published before the
257
+ * envelope existed keeps resolving until it is republished. Resolution is
258
+ * per-property, so a file may declare one knob in each home without the envelope
259
+ * hiding the other.
260
+ */
261
+ function modelTagLayers(
262
+ tag: ReadableTag | undefined,
263
+ ): (ReadableTag | undefined)[] {
264
+ const envelope =
265
+ tag && typeof tag.tag === "function"
266
+ ? tag.tag("materialization")
267
+ : undefined;
268
+ return [envelope, tag];
269
+ }
270
+
241
271
  /** The package-level `materialization.freshness` as a resolution layer. */
242
272
  function packageFreshnessLayer(
243
273
  cfg: WirePackageMaterialization | null | undefined,
@@ -289,13 +319,16 @@ export function resolveFreshness(
289
319
  source: PersistSource,
290
320
  packageMaterialization: WirePackageMaterialization | null | undefined,
291
321
  ): WireFreshness | null {
292
- const sourceLayer = tagFreshnessLayer(safeSourceTag(source));
293
- const modelLayer = tagFreshnessLayer(safeModelTag(source));
294
- const pkgLayer = packageFreshnessLayer(packageMaterialization);
322
+ const layers: FreshnessLayer[] = [
323
+ // `#@ persist` declares knobs bare — the annotation IS a materialization
324
+ // declaration, so there is no envelope to look under.
325
+ tagFreshnessLayer(safeSourceTag(source)),
326
+ ...modelTagLayers(safeModelTag(source)).map(tagFreshnessLayer),
327
+ packageFreshnessLayer(packageMaterialization),
328
+ ];
295
329
 
296
- const window = sourceLayer.window ?? modelLayer.window ?? pkgLayer.window;
297
- const fallback =
298
- sourceLayer.fallback ?? modelLayer.fallback ?? pkgLayer.fallback;
330
+ const window = layers.map((l) => l.window).find((v) => v !== undefined);
331
+ const fallback = layers.map((l) => l.fallback).find((v) => v !== undefined);
299
332
 
300
333
  if (window === undefined && fallback === undefined) return null;
301
334
  const freshness: WireFreshness = {};
@@ -304,6 +337,69 @@ export function resolveFreshness(
304
337
  return freshness;
305
338
  }
306
339
 
340
+ /**
341
+ * Read the `queryMetadata` property collection from one tag layer. A collection,
342
+ * not a scalar: `queryMetadata { team="finance" env="prod" }` and the equivalent
343
+ * dotted form `queryMetadata.team="finance"` both land here, and neither is
344
+ * captured by the scalar {@link deriveAnnotationFields} loop.
345
+ *
346
+ * Every string-valued property is kept verbatim, including ones that violate
347
+ * Malloy's bag contract — publish reports those as warnings and the runtime
348
+ * clamps them, so an author's typo is visible somewhere instead of vanishing
349
+ * between the annotation and the warehouse.
350
+ */
351
+ function tagQueryMetadataLayer(tag: ReadableTag | undefined): QueryMetadata {
352
+ const subtree =
353
+ tag && typeof tag.tag === "function"
354
+ ? tag.tag("queryMetadata")
355
+ : undefined;
356
+ if (!subtree || typeof subtree.entries !== "function") return {};
357
+ const layer: QueryMetadata = {};
358
+ try {
359
+ for (const [name, value] of subtree.entries()) {
360
+ const text = value.text();
361
+ if (text !== undefined) layer[name] = text;
362
+ }
363
+ } catch {
364
+ // Degrade to {} — mirrors deriveAnnotationFields / deriveColumns.
365
+ return {};
366
+ }
367
+ return layer;
368
+ }
369
+
370
+ /**
371
+ * Resolve a source's EFFECTIVE per-query metadata, most-specific-wins PER
372
+ * PROPERTY: `#@ persist queryMetadata.*` > model-file
373
+ * `## materialization.queryMetadata.*` (bare `## queryMetadata.*` underneath) >
374
+ * package `materialization.queryMetadata`.
375
+ *
376
+ * Per-property rather than per-layer, exactly like {@link resolveFreshness}, so a
377
+ * package-wide `team` property survives a source that only overrides `workload`.
378
+ * Null when no layer declares anything, so absence on the wire always means
379
+ * "declared nowhere" rather than "declared empty".
380
+ *
381
+ * This is the value the publisher attaches (merged under its own context) to
382
+ * every statement it issues while building the source. It is deliberately absent
383
+ * from the source's content address: changing a tag must never re-address a
384
+ * table.
385
+ */
386
+ export function resolveQueryMetadata(
387
+ source: PersistSource,
388
+ packageMaterialization: WirePackageMaterialization | null | undefined,
389
+ ): QueryMetadata | null {
390
+ // Least specific first, so a more specific layer overwrites property by
391
+ // property.
392
+ const layers: QueryMetadata[] = [
393
+ packageMaterialization?.queryMetadata ?? {},
394
+ ...modelTagLayers(safeModelTag(source))
395
+ .map(tagQueryMetadataLayer)
396
+ .reverse(),
397
+ tagQueryMetadataLayer(safeSourceTag(source)),
398
+ ];
399
+ const resolved: QueryMetadata = Object.assign({}, ...layers);
400
+ return Object.keys(resolved).length > 0 ? resolved : null;
401
+ }
402
+
307
403
  /** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
308
404
  export function flattenDependsOn(node: {
309
405
  dependsOn: { sourceID: string }[];
@@ -612,6 +708,11 @@ export function deriveBuildPlan(
612
708
  sql: source.getSQL(),
613
709
  refresh: annotationFields.refresh ?? null,
614
710
  freshness: resolveFreshness(source, packageMaterialization),
711
+ // EFFECTIVE per-source query metadata, resolved per property across the
712
+ // same layer stack as freshness. A property collection rather than a
713
+ // scalar, so it comes from resolveQueryMetadata rather than the
714
+ // annotationFields map.
715
+ queryMetadata: resolveQueryMetadata(source, packageMaterialization),
615
716
  columns: deriveColumns(source),
616
717
  annotationFields,
617
718
  modelPath: sourceModelPaths?.[sourceID],
@@ -2,7 +2,13 @@ import { createPrivateKey } from "crypto";
2
2
  import { existsSync } from "fs";
3
3
  import path from "path";
4
4
  import { components } from "../api";
5
+ import { logger } from "../logger";
5
6
  import { parseHostKeys } from "./proxy";
7
+ import {
8
+ queryMetadataAdvisoryWarnings,
9
+ queryMetadataBudgetWarning,
10
+ queryMetadataViolations,
11
+ } from "./query_metadata";
6
12
 
7
13
  type ApiConnection = components["schemas"]["Connection"];
8
14
  type AttachedDatabase = components["schemas"]["AttachedDatabase"];
@@ -271,6 +277,48 @@ function buildDuckdbEntry(
271
277
  };
272
278
  }
273
279
 
280
+ /**
281
+ * Report a connection default that will not do what it says — a property name
282
+ * the contract rejects, one BigQuery would drop, a bag with no room for the
283
+ * server's own context.
284
+ *
285
+ * Warns rather than throws, unlike everything else in this file: query metadata
286
+ * is observability, and an environment that refuses to load because a tag has a
287
+ * hyphen in it would trade a missing label for an outage. The connection update
288
+ * API rejects the same bag outright (see validateAdminAuthoredConnection) —
289
+ * strict where a human is waiting, lenient where a config is being loaded.
290
+ */
291
+ function warnOnConnectionQueryMetadata(connection: ApiConnection): void {
292
+ let declared = 0;
293
+ for (const field of ["queryMetadata", "queryMetadataEnforced"] as const) {
294
+ const metadata = connection[field];
295
+ if (!metadata) continue;
296
+ declared += Object.keys(metadata).length;
297
+ const problems = [
298
+ ...queryMetadataViolations(metadata),
299
+ ...queryMetadataAdvisoryWarnings(metadata),
300
+ ];
301
+ for (const problem of problems) {
302
+ logger.warn("Connection query metadata will not apply as declared", {
303
+ connectionName: connection.name,
304
+ field,
305
+ problem,
306
+ });
307
+ }
308
+ }
309
+ // The budget is checked over BOTH maps, not each one: they merge into the
310
+ // same bag, so a connection declaring 6 defaults and 6 enforced is over it
311
+ // while neither map is. This is the boundary where the admin who created the
312
+ // squeeze is the one reading the warning.
313
+ const overBudget = queryMetadataBudgetWarning(declared);
314
+ if (overBudget) {
315
+ logger.warn("Connection query metadata will not apply as declared", {
316
+ connectionName: connection.name,
317
+ problem: overBudget,
318
+ });
319
+ }
320
+ }
321
+
274
322
  function validateConnectionShape(connection: ApiConnection): void {
275
323
  if (connection.proxy) {
276
324
  // A connection proxy makes THIS server open an outbound SSH tunnel to a
@@ -606,6 +654,7 @@ export function assembleEnvironmentConnections(
606
654
  processedConnections.add(connection.name);
607
655
  validateDuckdbApiSurface(connection);
608
656
  validateConnectionShape(connection);
657
+ warnOnConnectionQueryMetadata(connection);
609
658
 
610
659
  const apiConnection = cloneApiConnection(connection);
611
660
  apiConnection.attributes = getStaticConnectionAttributes(connection.type);
@@ -1639,6 +1639,60 @@ export class Environment {
1639
1639
  logger.warn(`Could not read manifest for ${packageName}`);
1640
1640
  }
1641
1641
 
1642
+ const onDiskMaterialization =
1643
+ existingManifest.materialization !== null &&
1644
+ typeof existingManifest.materialization === "object" &&
1645
+ !Array.isArray(existingManifest.materialization)
1646
+ ? (existingManifest.materialization as Record<string, unknown>)
1647
+ : undefined;
1648
+
1649
+ // Scope has two homes: `materialization.scope` (canonical) and the
1650
+ // manifest root (deprecated). The server writes BOTH, in sync, for as
1651
+ // long as the root form is supported:
1652
+ //
1653
+ // - writing only the root would author a manifest this build's loader
1654
+ // refuses, since a root that disagrees with an existing envelope is a
1655
+ // conflict (see resolvePackageScope);
1656
+ // - writing only the envelope would silently downgrade a package read
1657
+ // by an older publisher, which knows only the root and would default
1658
+ // to `package` — cross-version table reuse for a package declared
1659
+ // `version`.
1660
+ //
1661
+ // A caller can only express scope through the top-level `scope` field
1662
+ // (the wire materialization block has no `scope`), so an envelope value
1663
+ // already on disk is preserved rather than dropped by a materialization
1664
+ // PATCH that says nothing about it.
1665
+ const resolvedScope =
1666
+ metadata.scope ??
1667
+ (onDiskMaterialization?.scope as ApiPackage["scope"] | undefined) ??
1668
+ (existingManifest.scope as ApiPackage["scope"] | undefined);
1669
+
1670
+ // A materialization PATCH replaces the block wholesale, which is right
1671
+ // for schedule and freshness — they are the policy the caller is
1672
+ // setting, and they are mutually exclusive with each other.
1673
+ // `queryMetadata` is orthogonal to both: a client setting a schedule
1674
+ // has no reason to re-send the package's tags, and dropping them
1675
+ // silently untags every statement the package's builds issue. So it is
1676
+ // preserved on omission, like `scope` above; an explicit null still
1677
+ // clears it, which keeps the block expressible.
1678
+ const preservedQueryMetadata =
1679
+ metadata.materialization !== undefined &&
1680
+ metadata.materialization?.queryMetadata === undefined &&
1681
+ onDiskMaterialization?.queryMetadata !== undefined
1682
+ ? { queryMetadata: onDiskMaterialization.queryMetadata }
1683
+ : {};
1684
+
1685
+ const materializationBase: Record<string, unknown> | undefined =
1686
+ metadata.materialization !== undefined
1687
+ ? { ...metadata.materialization, ...preservedQueryMetadata }
1688
+ : onDiskMaterialization !== undefined
1689
+ ? { ...onDiskMaterialization }
1690
+ : undefined;
1691
+ const materializationBlock =
1692
+ resolvedScope !== undefined
1693
+ ? { ...(materializationBase ?? {}), scope: resolvedScope }
1694
+ : materializationBase;
1695
+
1642
1696
  // Update with new metadata. `explores`/`queryableSources` are only
1643
1697
  // overwritten when the caller explicitly provides them; otherwise the
1644
1698
  // existing on-disk value is preserved via the spread (an undefined here
@@ -1656,9 +1710,9 @@ export class Environment {
1656
1710
  ...(metadata.manifestLocation !== undefined
1657
1711
  ? { manifestLocation: metadata.manifestLocation }
1658
1712
  : {}),
1659
- ...(metadata.scope !== undefined ? { scope: metadata.scope } : {}),
1660
- ...(metadata.materialization !== undefined
1661
- ? { materialization: metadata.materialization }
1713
+ ...(resolvedScope !== undefined ? { scope: resolvedScope } : {}),
1714
+ ...(materializationBlock !== undefined
1715
+ ? { materialization: materializationBlock }
1662
1716
  : {}),
1663
1717
  };
1664
1718