@malloy-publisher/server 0.0.232 → 0.0.234

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.docker.md +1 -0
  2. package/dist/app/api-doc.yaml +269 -10
  3. package/dist/app/assets/{EnvironmentPage-DXEaZIPx.js → EnvironmentPage-DTZQ4Gxc.js} +1 -1
  4. package/dist/app/assets/{HomePage-kofsqpZt.js → HomePage-C5mlDPXK.js} +1 -1
  5. package/dist/app/assets/{LightMode-CNhIlIlJ.js → LightMode-DGNmhG0u.js} +1 -1
  6. package/dist/app/assets/{MainPage-Bgqo8jCy.js → MainPage-CVL_wmP4.js} +1 -1
  7. package/dist/app/assets/{MaterializationsPage-CgBlgGz2.js → MaterializationsPage-DmzMBCpy.js} +1 -1
  8. package/dist/app/assets/{ModelPage-B0TjoDtf.js → ModelPage-Dbvf4QbB.js} +1 -1
  9. package/dist/app/assets/{PackagePage-BL8vnFj1.js → PackagePage-DxdHc2Qs.js} +1 -1
  10. package/dist/app/assets/{RouteError-BzPby0X2.js → RouteError-OJdT4tCd.js} +1 -1
  11. package/dist/app/assets/{ThemeEditorPage-CTEP_9r3.js → ThemeEditorPage-Bk7s0KXY.js} +1 -1
  12. package/dist/app/assets/{WorkbookPage-BwM3BmKw.js → WorkbookPage-j_vCWdN3.js} +1 -1
  13. package/dist/app/assets/{core-CK68iv6w.es-CpRxXBt7.js → core-Rj_4rRnA.es-DoIfLxDJ.js} +1 -1
  14. package/dist/app/assets/{index-B33zGctF.js → index-B_jKMR35.js} +4 -4
  15. package/dist/app/assets/{index-CmkW1MiE.js → index-D-rDyK11.js} +1 -1
  16. package/dist/app/assets/{index-tXJXwdyj.js → index-DWIe_hK0.js} +1 -1
  17. package/dist/app/assets/{index-BkiWKaAF.js → index-hw-xn0X7.js} +1 -1
  18. package/dist/app/index.html +1 -1
  19. package/dist/package_load_worker.mjs +53 -3
  20. package/dist/server.mjs +20277 -925
  21. package/package.json +1 -1
  22. package/src/config.ts +35 -1
  23. package/src/controller/connection.controller.spec.ts +46 -0
  24. package/src/controller/connection.controller.ts +105 -2
  25. package/src/controller/materialization.controller.spec.ts +25 -0
  26. package/src/controller/materialization.controller.ts +60 -0
  27. package/src/controller/model.controller.ts +24 -0
  28. package/src/controller/query.controller.ts +83 -10
  29. package/src/json_utils.spec.ts +51 -0
  30. package/src/json_utils.ts +33 -0
  31. package/src/mcp/handler_utils.ts +10 -2
  32. package/src/mcp/query_envelope.spec.ts +229 -0
  33. package/src/mcp/query_envelope.ts +240 -0
  34. package/src/mcp/server.protocol.spec.ts +128 -16
  35. package/src/mcp/skills/build_skills_bundle.ts +94 -4
  36. package/src/mcp/skills/skills_bundle.json +1 -1
  37. package/src/mcp/skills/skills_bundle.spec.ts +113 -4
  38. package/src/mcp/tool_response.spec.ts +108 -0
  39. package/src/mcp/tool_response.ts +138 -0
  40. package/src/mcp/tools/compile_tool.spec.ts +112 -4
  41. package/src/mcp/tools/compile_tool.ts +61 -30
  42. package/src/mcp/tools/docs_search_tool.ts +6 -16
  43. package/src/mcp/tools/execute_query_tool.spec.ts +154 -3
  44. package/src/mcp/tools/execute_query_tool.ts +131 -155
  45. package/src/mcp/tools/get_context_tool.spec.ts +63 -3
  46. package/src/mcp/tools/get_context_tool.ts +43 -46
  47. package/src/mcp/tools/reload_package_tool.ts +3 -29
  48. package/src/mcp_config.spec.ts +919 -0
  49. package/src/mcp_config.ts +425 -0
  50. package/src/oom_guards.integration.spec.ts +11 -3
  51. package/src/package_load/package_load_pool.ts +2 -0
  52. package/src/package_load/package_load_worker.ts +17 -5
  53. package/src/package_load/protocol.ts +6 -0
  54. package/src/query_metadata_metrics.ts +49 -0
  55. package/src/server.ts +99 -3
  56. package/src/service/build_plan.spec.ts +125 -0
  57. package/src/service/build_plan.ts +108 -7
  58. package/src/service/compile_fragment_techniques.spec.ts +156 -0
  59. package/src/service/connection.spec.ts +371 -1
  60. package/src/service/connection.ts +77 -14
  61. package/src/service/connection_config.spec.ts +60 -0
  62. package/src/service/connection_config.ts +75 -0
  63. package/src/service/duckdb_instance_isolation.spec.ts +137 -0
  64. package/src/service/environment.ts +57 -3
  65. package/src/service/materialization_config_validation.spec.ts +99 -0
  66. package/src/service/materialization_config_validation.ts +120 -0
  67. package/src/service/materialization_schedule_surface.spec.ts +124 -0
  68. package/src/service/materialization_service.spec.ts +119 -0
  69. package/src/service/materialization_service.ts +186 -3
  70. package/src/service/materialization_test_fixtures.ts +86 -21
  71. package/src/service/model.spec.ts +45 -1
  72. package/src/service/model.ts +171 -23
  73. package/src/service/model_limits.spec.ts +28 -0
  74. package/src/service/model_limits.ts +21 -0
  75. package/src/service/package.ts +24 -1
  76. package/src/service/package_manifest.spec.ts +137 -4
  77. package/src/service/package_manifest.ts +140 -5
  78. package/src/service/persist_annotation_validation.spec.ts +12 -0
  79. package/src/service/persist_annotation_validation.ts +9 -4
  80. package/src/service/query_metadata.spec.ts +408 -0
  81. package/src/service/query_metadata.ts +492 -0
  82. package/src/service/query_metadata_identity.spec.ts +149 -0
  83. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +37 -12
package/src/server.ts CHANGED
@@ -41,6 +41,7 @@ import {
41
41
  getMemoryGovernorConfig,
42
42
  getPersistCollisionEnforce,
43
43
  getPersistStorageMode,
44
+ getQueryMetadataMode,
44
45
  } from "./config";
45
46
  import { setFilterDeprecationHeaders } from "./filter_deprecation";
46
47
  import { checkHeapConfiguration } from "./heap_check";
@@ -48,6 +49,16 @@ import { queryConcurrency } from "./query_concurrency";
48
49
  import { MaterializationController } from "./controller/materialization.controller";
49
50
  import { ThemeController } from "./controller/theme.controller";
50
51
  import { initializeMcpServer } from "./mcp/server";
52
+ import {
53
+ addCommand,
54
+ ensureMcpConfig,
55
+ logMcpConfigOutcome,
56
+ MCP_CONFIG_FILENAME,
57
+ mcpConfigEnabled,
58
+ mcpEndpoint,
59
+ resolveBoundPort,
60
+ resolveClientHost,
61
+ } from "./mcp_config";
51
62
  import { registerLegacyRoutes } from "./server-old";
52
63
  import { EnvironmentStore } from "./service/environment_store";
53
64
  import { MaterializationScheduler } from "./service/materialization_scheduler";
@@ -95,6 +106,8 @@ function parseArgs() {
95
106
  i++;
96
107
  } else if (arg === "--init") {
97
108
  process.env.INITIALIZE_STORAGE = "true";
109
+ } else if (arg === "--no-mcp-config") {
110
+ process.env.PUBLISHER_NO_MCP_CONFIG = "true";
98
111
  } else if (arg === "--watch-env" && args[i + 1]) {
99
112
  // Append (don't overwrite) so multiple --watch-env flags compose
100
113
  // and so an explicit env var pre-set still wins.
@@ -133,6 +146,9 @@ function parseArgs() {
133
146
  console.log(
134
147
  " --init Wipe persisted storage and re-sync it from the config (default: false)",
135
148
  );
149
+ console.log(
150
+ " --no-mcp-config Do not write .mcp.json into the working directory (default: it is written, so an agent opened here finds this server; skipped when the directory already has one, is your home directory or the filesystem root, is inside a git working tree, or the MCP port bound is not the one requested)",
151
+ );
136
152
  console.log(
137
153
  " --watch-env <name> Enable dev-mode watch for the named environment.",
138
154
  );
@@ -177,9 +193,20 @@ getPersistStorageMode();
177
193
  // than a failed boot.
178
194
  getPersistCollisionEnforce();
179
195
 
196
+ // Same hazard, wider blast radius: getQueryMetadataMode() throws on an invalid
197
+ // value and is read while resolving EVERY statement, so a typo'd off switch
198
+ // ("false", "0", "disabled") would boot clean and then fail every query and
199
+ // every build — the one thing the metadata path promises never to do.
200
+ getQueryMetadataMode();
201
+
180
202
  const PUBLISHER_PORT = Number(process.env.PUBLISHER_PORT || 4000);
181
203
  const PUBLISHER_HOST = process.env.PUBLISHER_HOST || "0.0.0.0";
182
204
  const MCP_PORT = Number(process.env.MCP_PORT || 4040);
205
+ // Resolved here rather than in the listen callback: parseBoolEnv throws on a
206
+ // typo, which is the convention for flags in this server, but a throw inside a
207
+ // listen callback is an uncaughtException that kills a server which has already
208
+ // bound both ports. At module scope it is an ordinary startup failure.
209
+ const MCP_CONFIG_ENABLED = mcpConfigEnabled();
183
210
  const MCP_ENDPOINT = "/mcp";
184
211
  const SHUTDOWN_DRAIN_DURATION_SECONDS = Number(
185
212
  process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0,
@@ -1273,6 +1300,11 @@ app.post(
1273
1300
  req.params.connectionName,
1274
1301
  req.body.sqlStatement as string,
1275
1302
  req.body?.options as string,
1303
+ undefined,
1304
+ {
1305
+ queryMetadata: req.body?.queryMetadata,
1306
+ queryClass: req.body?.queryClass,
1307
+ },
1276
1308
  ),
1277
1309
  );
1278
1310
  } catch (error) {
@@ -1295,6 +1327,10 @@ app.post(
1295
1327
  req.body.sqlStatement as string,
1296
1328
  req.body?.options as string,
1297
1329
  req.params.packageName,
1330
+ {
1331
+ queryMetadata: req.body?.queryMetadata,
1332
+ queryClass: req.body?.queryClass,
1333
+ },
1298
1334
  ),
1299
1335
  );
1300
1336
  } catch (error) {
@@ -1664,6 +1700,11 @@ app.post(
1664
1700
  | undefined,
1665
1701
  req.body.bypassFilters === true ? true : undefined,
1666
1702
  req.body.givens as Record<string, GivenValue> | undefined,
1703
+ {
1704
+ queryMetadata: req.body?.queryMetadata,
1705
+ queryClass: req.body?.queryClass,
1706
+ versionId: req.body?.versionId as string | undefined,
1707
+ },
1667
1708
  );
1668
1709
  setFilterDeprecationHeaders(res, {
1669
1710
  filterParams: req.body.filterParams ?? req.body.sourceFilters,
@@ -1947,9 +1988,64 @@ mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, async () => {
1947
1988
  }
1948
1989
  }
1949
1990
  });
1950
- const mcpServer = mcpApp.listen(MCP_PORT, PUBLISHER_HOST, () => {
1951
- logger.info(`MCP server listening at http://${PUBLISHER_HOST}:${MCP_PORT}`);
1952
- });
1991
+ const mcpServer = mcpApp.listen(
1992
+ MCP_PORT,
1993
+ PUBLISHER_HOST,
1994
+ function (this: import("net").Server) {
1995
+ // Read back rather than reusing MCP_PORT, which is only what was requested.
1996
+ // `--mcp_port 0` asks for any free port, and under bun a non-numeric value
1997
+ // binds an ephemeral one too, so the requested value can be 0 or NaN while
1998
+ // a real port is listening. The listening line uses it as well, which is
1999
+ // why it no longer reads `http://127.0.0.1:0`.
2000
+ const boundPort = resolveBoundPort(this.address(), MCP_PORT);
2001
+ // The BIND address, bracketed when it is an IPv6 literal so the URL
2002
+ // parses. Deliberately not resolveClientHost: create-malloy-package's
2003
+ // README and AGENTS template both tell readers these two listening lines
2004
+ // are "the addresses it really bound", and use them to catch a mistyped
2005
+ // --hostt that silently falls back to 0.0.0.0. Mapping the wildcard to
2006
+ // loopback here would confirm the mistake instead of revealing it. The
2007
+ // dialable form belongs in .mcp.json and in the advice, not here.
2008
+ const bound = this.address();
2009
+ const boundHost =
2010
+ typeof bound === "object" && bound ? bound.address : PUBLISHER_HOST;
2011
+ logger.info(
2012
+ `MCP server listening at http://${boundHost.includes(":") ? `[${boundHost}]` : boundHost}:${boundPort}`,
2013
+ );
2014
+ // Checked before process.cwd(), which can throw: someone who turned the
2015
+ // feature off should not get a warning about it.
2016
+ if (MCP_CONFIG_ENABLED) {
2017
+ // ensureMcpConfig cannot throw, but its arguments can: process.cwd()
2018
+ // raises ENOENT once the working directory has been removed. A throw
2019
+ // here is an uncaught exception inside a listen callback, which would
2020
+ // kill a server that has already bound both ports. Everything the call
2021
+ // needs is built inside the try for that reason, including the
2022
+ // endpoint: it is the newest and least-exercised code in this block.
2023
+ try {
2024
+ // The host an agent should dial, which is NOT `localhost`: that name
2025
+ // resolves to both loopback families while the server binds only one,
2026
+ // so another local process can hold the same port on the other family
2027
+ // and receive the agent's traffic instead.
2028
+ const endpoint = mcpEndpoint(
2029
+ resolveClientHost(this.address(), PUBLISHER_HOST),
2030
+ boundPort,
2031
+ );
2032
+ // cwd, not server_root: the file is for whoever opens an agent here.
2033
+ logMcpConfigOutcome(
2034
+ ensureMcpConfig({
2035
+ dir: process.cwd(),
2036
+ endpoint,
2037
+ requestedPort: MCP_PORT,
2038
+ boundPort,
2039
+ }),
2040
+ );
2041
+ } catch (error) {
2042
+ logger.info(
2043
+ `Could not set up ${MCP_CONFIG_FILENAME} (${error instanceof Error ? error.message : String(error)}). To connect an agent, run: ${addCommand(mcpEndpoint(resolveClientHost(this.address(), PUBLISHER_HOST), boundPort))}`,
2044
+ );
2045
+ }
2046
+ }
2047
+ },
2048
+ );
1953
2049
 
1954
2050
  mcpServer.timeout = 600000;
1955
2051
  mcpServer.keepAliveTimeout = 600000;
@@ -12,6 +12,7 @@ import {
12
12
  iterGraphSources,
13
13
  projectToPublicColumns,
14
14
  resolveFreshness,
15
+ resolveQueryMetadata,
15
16
  resolvePackageConnections,
16
17
  } from "./build_plan";
17
18
  import { MaterializationEligibilityError } from "../errors";
@@ -419,6 +420,130 @@ describe("resolveFreshness", () => {
419
420
  });
420
421
  expect(resolveFreshness(source, null)).toEqual({ window: "1h" });
421
422
  });
423
+
424
+ it("reads the model-file `materialization` envelope", () => {
425
+ const source = fakeSource({
426
+ name: "s",
427
+ sourceEntityId: "bid",
428
+ modelMaterialization: { freshness: { freshness: { window: "12h" } } },
429
+ });
430
+ expect(resolveFreshness(source, null)).toEqual({ window: "12h" });
431
+ });
432
+
433
+ it("prefers the envelope over the deprecated bare model-file form", () => {
434
+ const source = fakeSource({
435
+ name: "s",
436
+ sourceEntityId: "bid",
437
+ modelFreshnessSchedule: { freshness: { window: "48h" } },
438
+ modelMaterialization: { freshness: { freshness: { window: "12h" } } },
439
+ });
440
+ expect(resolveFreshness(source, null)).toEqual({ window: "12h" });
441
+ });
442
+
443
+ it("still reads a bare model-file knob the envelope does not declare", () => {
444
+ // A package published before the envelope existed keeps resolving, and the
445
+ // envelope does not hide the knobs it says nothing about.
446
+ const source = fakeSource({
447
+ name: "s",
448
+ sourceEntityId: "bid",
449
+ modelFreshnessSchedule: { freshness: { fallback: "fail" } },
450
+ modelMaterialization: { freshness: { freshness: { window: "12h" } } },
451
+ });
452
+ expect(resolveFreshness(source, null)).toEqual({
453
+ window: "12h",
454
+ fallback: "fail",
455
+ });
456
+ });
457
+ });
458
+
459
+ describe("resolveQueryMetadata", () => {
460
+ it("returns null when no layer declares anything", () => {
461
+ const source = fakeSource({ name: "s", sourceEntityId: "bid" });
462
+ expect(resolveQueryMetadata(source, null)).toBeNull();
463
+ expect(
464
+ resolveQueryMetadata(source, {
465
+ schedule: null,
466
+ freshness: null,
467
+ queryMetadata: null,
468
+ }),
469
+ ).toBeNull();
470
+ });
471
+
472
+ it("reads the source's `#@ persist queryMetadata.*` properties", () => {
473
+ const source = fakeSource({
474
+ name: "s",
475
+ sourceEntityId: "bid",
476
+ queryMetadata: { team: "finance", workload: "orders" },
477
+ });
478
+ expect(resolveQueryMetadata(source, null)).toEqual({
479
+ team: "finance",
480
+ workload: "orders",
481
+ });
482
+ });
483
+
484
+ it("resolves most-specific-wins PER PROPERTY across all four layers", () => {
485
+ // Package declares team+tier, the model-file envelope overrides tier and
486
+ // adds one of its own, the source overrides only workload: every property
487
+ // nothing more specific overrides has to survive.
488
+ const source = fakeSource({
489
+ name: "s",
490
+ sourceEntityId: "bid",
491
+ queryMetadata: { workload: "orders" },
492
+ modelMaterialization: {
493
+ queryMetadata: { tier: "gold", surface: "marts" },
494
+ },
495
+ });
496
+ expect(
497
+ resolveQueryMetadata(source, {
498
+ schedule: null,
499
+ freshness: null,
500
+ queryMetadata: { team: "finance", tier: "bronze" },
501
+ }),
502
+ ).toEqual({
503
+ team: "finance",
504
+ tier: "gold",
505
+ surface: "marts",
506
+ workload: "orders",
507
+ });
508
+ });
509
+
510
+ it("prefers the model-file envelope over the bare form, per property", () => {
511
+ const source = fakeSource({
512
+ name: "s",
513
+ sourceEntityId: "bid",
514
+ modelQueryMetadata: { tier: "bronze", legacy: "kept" },
515
+ modelMaterialization: { queryMetadata: { tier: "gold" } },
516
+ });
517
+ expect(resolveQueryMetadata(source, null)).toEqual({
518
+ tier: "gold",
519
+ legacy: "kept",
520
+ });
521
+ });
522
+
523
+ it("keeps a contract-violating property for the validator to report", () => {
524
+ const source = fakeSource({
525
+ name: "s",
526
+ sourceEntityId: "bid",
527
+ queryMetadata: { "team.name": "finance" },
528
+ });
529
+ expect(resolveQueryMetadata(source, null)).toEqual({
530
+ "team.name": "finance",
531
+ });
532
+ });
533
+
534
+ it("does not confuse the scalar `#@ persist` fields beside it", () => {
535
+ const source = fakeSource({
536
+ name: "s",
537
+ sourceEntityId: "bid",
538
+ annotationFields: { name: "s_table", refresh: "full" },
539
+ queryMetadata: { team: "finance" },
540
+ });
541
+ expect(resolveQueryMetadata(source, null)).toEqual({ team: "finance" });
542
+ expect(deriveAnnotationFields(source)).toEqual({
543
+ name: "s_table",
544
+ refresh: "full",
545
+ });
546
+ });
422
547
  });
423
548
 
424
549
  describe("deriveBuildPlan freshness", () => {
@@ -27,6 +27,7 @@ type BuildPlan = components["schemas"]["BuildPlan"];
27
27
  type WireFreshness = components["schemas"]["Freshness"];
28
28
  type WirePackageMaterialization =
29
29
  components["schemas"]["PackageMaterializationConfig"];
30
+ type QueryMetadata = components["schemas"]["QueryMetadata"];
30
31
 
31
32
  /** The freshness `fallback` values the publisher recognizes; others are dropped. */
32
33
  const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
@@ -38,9 +39,15 @@ interface FreshnessLayer {
38
39
  fallback?: FreshnessFallback;
39
40
  }
40
41
 
41
- /** Minimal path-reader over a Malloy `Tag` (see `@malloydata/malloy-tag`). */
42
+ /**
43
+ * Minimal reader over a Malloy `Tag` (see `@malloydata/malloy-tag`): scalar
44
+ * reads by path, plus the subtree read that a property collection like
45
+ * `queryMetadata { … }` needs.
46
+ */
42
47
  interface ReadableTag {
43
48
  text(...path: string[]): string | undefined;
49
+ tag(...path: string[]): ReadableTag | undefined;
50
+ entries?(): Iterable<[string, { text(): string | undefined }]>;
44
51
  }
45
52
 
46
53
  /**
@@ -238,6 +245,29 @@ function tagFreshnessLayer(tag: ReadableTag | undefined): FreshnessLayer {
238
245
  return layer;
239
246
  }
240
247
 
248
+ /**
249
+ * The two homes a model-file (`##`) knob can be declared in, most specific
250
+ * first: the `materialization` envelope, then the bare form.
251
+ *
252
+ * The model-file level mirrors the manifest's shape verbatim — `##
253
+ * materialization.freshness.window="24h"` is the manifest's block in tag syntax
254
+ * — so that is where a file-level reader looks. The bare form
255
+ * (`## freshness.window="24h"`), which shipped first, stays readable underneath:
256
+ * the annotation rides the published package, so a package published before the
257
+ * envelope existed keeps resolving until it is republished. Resolution is
258
+ * per-property, so a file may declare one knob in each home without the envelope
259
+ * hiding the other.
260
+ */
261
+ function modelTagLayers(
262
+ tag: ReadableTag | undefined,
263
+ ): (ReadableTag | undefined)[] {
264
+ const envelope =
265
+ tag && typeof tag.tag === "function"
266
+ ? tag.tag("materialization")
267
+ : undefined;
268
+ return [envelope, tag];
269
+ }
270
+
241
271
  /** The package-level `materialization.freshness` as a resolution layer. */
242
272
  function packageFreshnessLayer(
243
273
  cfg: WirePackageMaterialization | null | undefined,
@@ -289,13 +319,16 @@ export function resolveFreshness(
289
319
  source: PersistSource,
290
320
  packageMaterialization: WirePackageMaterialization | null | undefined,
291
321
  ): WireFreshness | null {
292
- const sourceLayer = tagFreshnessLayer(safeSourceTag(source));
293
- const modelLayer = tagFreshnessLayer(safeModelTag(source));
294
- const pkgLayer = packageFreshnessLayer(packageMaterialization);
322
+ const layers: FreshnessLayer[] = [
323
+ // `#@ persist` declares knobs bare — the annotation IS a materialization
324
+ // declaration, so there is no envelope to look under.
325
+ tagFreshnessLayer(safeSourceTag(source)),
326
+ ...modelTagLayers(safeModelTag(source)).map(tagFreshnessLayer),
327
+ packageFreshnessLayer(packageMaterialization),
328
+ ];
295
329
 
296
- const window = sourceLayer.window ?? modelLayer.window ?? pkgLayer.window;
297
- const fallback =
298
- sourceLayer.fallback ?? modelLayer.fallback ?? pkgLayer.fallback;
330
+ const window = layers.map((l) => l.window).find((v) => v !== undefined);
331
+ const fallback = layers.map((l) => l.fallback).find((v) => v !== undefined);
299
332
 
300
333
  if (window === undefined && fallback === undefined) return null;
301
334
  const freshness: WireFreshness = {};
@@ -304,6 +337,69 @@ export function resolveFreshness(
304
337
  return freshness;
305
338
  }
306
339
 
340
+ /**
341
+ * Read the `queryMetadata` property collection from one tag layer. A collection,
342
+ * not a scalar: `queryMetadata { team="finance" env="prod" }` and the equivalent
343
+ * dotted form `queryMetadata.team="finance"` both land here, and neither is
344
+ * captured by the scalar {@link deriveAnnotationFields} loop.
345
+ *
346
+ * Every string-valued property is kept verbatim, including ones that violate
347
+ * Malloy's bag contract — publish reports those as warnings and the runtime
348
+ * clamps them, so an author's typo is visible somewhere instead of vanishing
349
+ * between the annotation and the warehouse.
350
+ */
351
+ function tagQueryMetadataLayer(tag: ReadableTag | undefined): QueryMetadata {
352
+ const subtree =
353
+ tag && typeof tag.tag === "function"
354
+ ? tag.tag("queryMetadata")
355
+ : undefined;
356
+ if (!subtree || typeof subtree.entries !== "function") return {};
357
+ const layer: QueryMetadata = {};
358
+ try {
359
+ for (const [name, value] of subtree.entries()) {
360
+ const text = value.text();
361
+ if (text !== undefined) layer[name] = text;
362
+ }
363
+ } catch {
364
+ // Degrade to {} — mirrors deriveAnnotationFields / deriveColumns.
365
+ return {};
366
+ }
367
+ return layer;
368
+ }
369
+
370
+ /**
371
+ * Resolve a source's EFFECTIVE per-query metadata, most-specific-wins PER
372
+ * PROPERTY: `#@ persist queryMetadata.*` > model-file
373
+ * `## materialization.queryMetadata.*` (bare `## queryMetadata.*` underneath) >
374
+ * package `materialization.queryMetadata`.
375
+ *
376
+ * Per-property rather than per-layer, exactly like {@link resolveFreshness}, so a
377
+ * package-wide `team` property survives a source that only overrides `workload`.
378
+ * Null when no layer declares anything, so absence on the wire always means
379
+ * "declared nowhere" rather than "declared empty".
380
+ *
381
+ * This is the value the publisher attaches (merged under its own context) to
382
+ * every statement it issues while building the source. It is deliberately absent
383
+ * from the source's content address: changing a tag must never re-address a
384
+ * table.
385
+ */
386
+ export function resolveQueryMetadata(
387
+ source: PersistSource,
388
+ packageMaterialization: WirePackageMaterialization | null | undefined,
389
+ ): QueryMetadata | null {
390
+ // Least specific first, so a more specific layer overwrites property by
391
+ // property.
392
+ const layers: QueryMetadata[] = [
393
+ packageMaterialization?.queryMetadata ?? {},
394
+ ...modelTagLayers(safeModelTag(source))
395
+ .map(tagQueryMetadataLayer)
396
+ .reverse(),
397
+ tagQueryMetadataLayer(safeSourceTag(source)),
398
+ ];
399
+ const resolved: QueryMetadata = Object.assign({}, ...layers);
400
+ return Object.keys(resolved).length > 0 ? resolved : null;
401
+ }
402
+
307
403
  /** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
308
404
  export function flattenDependsOn(node: {
309
405
  dependsOn: { sourceID: string }[];
@@ -612,6 +708,11 @@ export function deriveBuildPlan(
612
708
  sql: source.getSQL(),
613
709
  refresh: annotationFields.refresh ?? null,
614
710
  freshness: resolveFreshness(source, packageMaterialization),
711
+ // EFFECTIVE per-source query metadata, resolved per property across the
712
+ // same layer stack as freshness. A property collection rather than a
713
+ // scalar, so it comes from resolveQueryMetadata rather than the
714
+ // annotationFields map.
715
+ queryMetadata: resolveQueryMetadata(source, packageMaterialization),
615
716
  columns: deriveColumns(source),
616
717
  annotationFields,
617
718
  modelPath: sourceModelPaths?.[sourceID],
@@ -0,0 +1,156 @@
1
+ import { afterEach, beforeEach, describe, expect, it } from "bun:test";
2
+ import fs from "fs/promises";
3
+ import os from "os";
4
+ import path from "path";
5
+ import { Environment } from "./environment";
6
+
7
+ /**
8
+ * The fragment-checking advice in malloy_compile's tool description, pinned
9
+ * against the REAL compiler.
10
+ *
11
+ * That description tells an agent how to validate part of a source: send a view
12
+ * body as a top-level `query:`, or wrap a field in a throwaway `extend`, and do
13
+ * not resubmit the source being edited. None of that is enforced by code, so
14
+ * nothing but a test stops it from quietly becoming false. Asserting it against
15
+ * a hand-written checker would be worthless: only Malloy decides what compiles.
16
+ */
17
+
18
+ const PUBLISHER_JSON = JSON.stringify({
19
+ name: "pkg",
20
+ description: "fragments",
21
+ });
22
+
23
+ // `secret` is private on the base source, so nothing downstream may read it.
24
+ const MODEL = `##! experimental.access_modifiers
25
+ source: base is duckdb.sql("SELECT 1.5 as price, 90 as points, 'shh' as secret") include {
26
+ public: price, points
27
+ private: secret
28
+ }
29
+
30
+ source: sales is base extend {
31
+ measure: record_count is count()
32
+ measure: avg_price is avg(price)
33
+ view: overview is { aggregate: record_count }
34
+ }
35
+ `;
36
+
37
+ describe("malloy_compile: checking part of a source", () => {
38
+ let rootDir: string;
39
+ let env: Environment;
40
+
41
+ beforeEach(async () => {
42
+ rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-frag-"));
43
+ const envPath = path.join(rootDir, "env");
44
+ await fs.mkdir(envPath, { recursive: true });
45
+ env = await Environment.create("testEnv", envPath, []);
46
+ await env.installPackage("pkg", async (stagingPath) => {
47
+ await fs.mkdir(stagingPath, { recursive: true });
48
+ await fs.writeFile(
49
+ path.join(stagingPath, "publisher.json"),
50
+ PUBLISHER_JSON,
51
+ );
52
+ await fs.writeFile(path.join(stagingPath, "model.malloy"), MODEL);
53
+ });
54
+ });
55
+
56
+ afterEach(async () => {
57
+ await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
58
+ });
59
+
60
+ /** Error-severity messages from compiling `source` against the fixture. */
61
+ async function errorsFor(source: string): Promise<string[]> {
62
+ const result = await env.compileSource("pkg", "model.malloy", source);
63
+ return (result.problems ?? [])
64
+ .filter((p) => p.severity === "error")
65
+ .map((p) => p.message);
66
+ }
67
+
68
+ describe("the two failures the description exists to explain", () => {
69
+ it("rejects a bare view: fragment, naming only the symptom", async () => {
70
+ const errors = await errorsFor(
71
+ "view: by_pts is { group_by: points, aggregate: record_count }",
72
+ );
73
+ expect(errors.join(" ")).toContain("view:");
74
+ });
75
+
76
+ it("rejects resubmitting the source being edited", async () => {
77
+ // The natural move when validating an edit, and it fails for a reason
78
+ // that has nothing to do with the edit.
79
+ const errors = await errorsFor(`source: sales is base extend {
80
+ measure: record_count is count()
81
+ view: by_pts is { group_by: points }
82
+ }`);
83
+ expect(errors.join(" ")).toContain("Cannot redefine 'sales'");
84
+ });
85
+ });
86
+
87
+ describe("the top-level query: form", () => {
88
+ it("compiles a view body with no wrapper at all", async () => {
89
+ expect(
90
+ await errorsFor(
91
+ "query: check is sales -> { group_by: points, aggregate: record_count }",
92
+ ),
93
+ ).toEqual([]);
94
+ });
95
+
96
+ it("still resolves measures the source inherits", async () => {
97
+ expect(
98
+ await errorsFor(
99
+ "query: check is sales -> { aggregate: avg_price }",
100
+ ),
101
+ ).toEqual([]);
102
+ });
103
+ });
104
+
105
+ describe("the throwaway extend form", () => {
106
+ const wrap = (fragment: string, into = "sales") =>
107
+ `source: check is ${into} extend {\n${fragment}\n}`;
108
+
109
+ it("compiles bare view, measure, and dimension fragments", async () => {
110
+ expect(
111
+ await errorsFor(
112
+ wrap(
113
+ "view: by_pts is { group_by: points, aggregate: record_count }",
114
+ ),
115
+ ),
116
+ ).toEqual([]);
117
+ expect(
118
+ await errorsFor(wrap("measure: max_price is max(price)")),
119
+ ).toEqual([]);
120
+ expect(
121
+ await errorsFor(wrap("dimension: pricey is price > 1")),
122
+ ).toEqual([]);
123
+ });
124
+
125
+ it("resolves measures the source inherits", async () => {
126
+ expect(
127
+ await errorsFor(wrap("view: v is { aggregate: avg_price }")),
128
+ ).toEqual([]);
129
+ });
130
+
131
+ it("hides a private field exactly as an in-place extension does", async () => {
132
+ // The fidelity claim in the description: the wrapper must not widen or
133
+ // narrow visibility versus the edit it stands in for.
134
+ const wrapped = await errorsFor(wrap("dimension: s is secret"));
135
+ const inPlace = await errorsFor(
136
+ "source: probe is base extend { dimension: s is secret }",
137
+ );
138
+ expect(wrapped.join(" ")).toContain("private");
139
+ expect(inPlace.join(" ")).toContain("private");
140
+ });
141
+
142
+ it("reports a redefinition when the fragment reuses an existing name", async () => {
143
+ // The documented caveat: an extension adds to the namespace rather
144
+ // than overriding it, so checking an EDIT needs the fragment renamed.
145
+ expect(
146
+ await errorsFor(wrap("view: overview is { aggregate: avg_price }")),
147
+ ).toEqual(["Cannot redefine 'overview'"]);
148
+
149
+ expect(
150
+ await errorsFor(
151
+ wrap("view: overview__check is { aggregate: avg_price }"),
152
+ ),
153
+ ).toEqual([]);
154
+ });
155
+ });
156
+ });