@malloy-publisher/server 0.0.232 → 0.0.234

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.docker.md +1 -0
  2. package/dist/app/api-doc.yaml +269 -10
  3. package/dist/app/assets/{EnvironmentPage-DXEaZIPx.js → EnvironmentPage-DTZQ4Gxc.js} +1 -1
  4. package/dist/app/assets/{HomePage-kofsqpZt.js → HomePage-C5mlDPXK.js} +1 -1
  5. package/dist/app/assets/{LightMode-CNhIlIlJ.js → LightMode-DGNmhG0u.js} +1 -1
  6. package/dist/app/assets/{MainPage-Bgqo8jCy.js → MainPage-CVL_wmP4.js} +1 -1
  7. package/dist/app/assets/{MaterializationsPage-CgBlgGz2.js → MaterializationsPage-DmzMBCpy.js} +1 -1
  8. package/dist/app/assets/{ModelPage-B0TjoDtf.js → ModelPage-Dbvf4QbB.js} +1 -1
  9. package/dist/app/assets/{PackagePage-BL8vnFj1.js → PackagePage-DxdHc2Qs.js} +1 -1
  10. package/dist/app/assets/{RouteError-BzPby0X2.js → RouteError-OJdT4tCd.js} +1 -1
  11. package/dist/app/assets/{ThemeEditorPage-CTEP_9r3.js → ThemeEditorPage-Bk7s0KXY.js} +1 -1
  12. package/dist/app/assets/{WorkbookPage-BwM3BmKw.js → WorkbookPage-j_vCWdN3.js} +1 -1
  13. package/dist/app/assets/{core-CK68iv6w.es-CpRxXBt7.js → core-Rj_4rRnA.es-DoIfLxDJ.js} +1 -1
  14. package/dist/app/assets/{index-B33zGctF.js → index-B_jKMR35.js} +4 -4
  15. package/dist/app/assets/{index-CmkW1MiE.js → index-D-rDyK11.js} +1 -1
  16. package/dist/app/assets/{index-tXJXwdyj.js → index-DWIe_hK0.js} +1 -1
  17. package/dist/app/assets/{index-BkiWKaAF.js → index-hw-xn0X7.js} +1 -1
  18. package/dist/app/index.html +1 -1
  19. package/dist/package_load_worker.mjs +53 -3
  20. package/dist/server.mjs +20277 -925
  21. package/package.json +1 -1
  22. package/src/config.ts +35 -1
  23. package/src/controller/connection.controller.spec.ts +46 -0
  24. package/src/controller/connection.controller.ts +105 -2
  25. package/src/controller/materialization.controller.spec.ts +25 -0
  26. package/src/controller/materialization.controller.ts +60 -0
  27. package/src/controller/model.controller.ts +24 -0
  28. package/src/controller/query.controller.ts +83 -10
  29. package/src/json_utils.spec.ts +51 -0
  30. package/src/json_utils.ts +33 -0
  31. package/src/mcp/handler_utils.ts +10 -2
  32. package/src/mcp/query_envelope.spec.ts +229 -0
  33. package/src/mcp/query_envelope.ts +240 -0
  34. package/src/mcp/server.protocol.spec.ts +128 -16
  35. package/src/mcp/skills/build_skills_bundle.ts +94 -4
  36. package/src/mcp/skills/skills_bundle.json +1 -1
  37. package/src/mcp/skills/skills_bundle.spec.ts +113 -4
  38. package/src/mcp/tool_response.spec.ts +108 -0
  39. package/src/mcp/tool_response.ts +138 -0
  40. package/src/mcp/tools/compile_tool.spec.ts +112 -4
  41. package/src/mcp/tools/compile_tool.ts +61 -30
  42. package/src/mcp/tools/docs_search_tool.ts +6 -16
  43. package/src/mcp/tools/execute_query_tool.spec.ts +154 -3
  44. package/src/mcp/tools/execute_query_tool.ts +131 -155
  45. package/src/mcp/tools/get_context_tool.spec.ts +63 -3
  46. package/src/mcp/tools/get_context_tool.ts +43 -46
  47. package/src/mcp/tools/reload_package_tool.ts +3 -29
  48. package/src/mcp_config.spec.ts +919 -0
  49. package/src/mcp_config.ts +425 -0
  50. package/src/oom_guards.integration.spec.ts +11 -3
  51. package/src/package_load/package_load_pool.ts +2 -0
  52. package/src/package_load/package_load_worker.ts +17 -5
  53. package/src/package_load/protocol.ts +6 -0
  54. package/src/query_metadata_metrics.ts +49 -0
  55. package/src/server.ts +99 -3
  56. package/src/service/build_plan.spec.ts +125 -0
  57. package/src/service/build_plan.ts +108 -7
  58. package/src/service/compile_fragment_techniques.spec.ts +156 -0
  59. package/src/service/connection.spec.ts +371 -1
  60. package/src/service/connection.ts +77 -14
  61. package/src/service/connection_config.spec.ts +60 -0
  62. package/src/service/connection_config.ts +75 -0
  63. package/src/service/duckdb_instance_isolation.spec.ts +137 -0
  64. package/src/service/environment.ts +57 -3
  65. package/src/service/materialization_config_validation.spec.ts +99 -0
  66. package/src/service/materialization_config_validation.ts +120 -0
  67. package/src/service/materialization_schedule_surface.spec.ts +124 -0
  68. package/src/service/materialization_service.spec.ts +119 -0
  69. package/src/service/materialization_service.ts +186 -3
  70. package/src/service/materialization_test_fixtures.ts +86 -21
  71. package/src/service/model.spec.ts +45 -1
  72. package/src/service/model.ts +171 -23
  73. package/src/service/model_limits.spec.ts +28 -0
  74. package/src/service/model_limits.ts +21 -0
  75. package/src/service/package.ts +24 -1
  76. package/src/service/package_manifest.spec.ts +137 -4
  77. package/src/service/package_manifest.ts +140 -5
  78. package/src/service/persist_annotation_validation.spec.ts +12 -0
  79. package/src/service/persist_annotation_validation.ts +9 -4
  80. package/src/service/query_metadata.spec.ts +408 -0
  81. package/src/service/query_metadata.ts +492 -0
  82. package/src/service/query_metadata_identity.spec.ts +149 -0
  83. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +37 -12
@@ -2,7 +2,13 @@ import { createPrivateKey } from "crypto";
2
2
  import { existsSync } from "fs";
3
3
  import path from "path";
4
4
  import { components } from "../api";
5
+ import { logger } from "../logger";
5
6
  import { parseHostKeys } from "./proxy";
7
+ import {
8
+ queryMetadataAdvisoryWarnings,
9
+ queryMetadataBudgetWarning,
10
+ queryMetadataViolations,
11
+ } from "./query_metadata";
6
12
 
7
13
  type ApiConnection = components["schemas"]["Connection"];
8
14
  type AttachedDatabase = components["schemas"]["AttachedDatabase"];
@@ -271,6 +277,48 @@ function buildDuckdbEntry(
271
277
  };
272
278
  }
273
279
 
280
+ /**
281
+ * Report a connection default that will not do what it says — a property name
282
+ * the contract rejects, one BigQuery would drop, a bag with no room for the
283
+ * server's own context.
284
+ *
285
+ * Warns rather than throws, unlike everything else in this file: query metadata
286
+ * is observability, and an environment that refuses to load because a tag has a
287
+ * hyphen in it would trade a missing label for an outage. The connection update
288
+ * API rejects the same bag outright (see validateAdminAuthoredConnection) —
289
+ * strict where a human is waiting, lenient where a config is being loaded.
290
+ */
291
+ function warnOnConnectionQueryMetadata(connection: ApiConnection): void {
292
+ let declared = 0;
293
+ for (const field of ["queryMetadata", "queryMetadataEnforced"] as const) {
294
+ const metadata = connection[field];
295
+ if (!metadata) continue;
296
+ declared += Object.keys(metadata).length;
297
+ const problems = [
298
+ ...queryMetadataViolations(metadata),
299
+ ...queryMetadataAdvisoryWarnings(metadata),
300
+ ];
301
+ for (const problem of problems) {
302
+ logger.warn("Connection query metadata will not apply as declared", {
303
+ connectionName: connection.name,
304
+ field,
305
+ problem,
306
+ });
307
+ }
308
+ }
309
+ // The budget is checked over BOTH maps, not each one: they merge into the
310
+ // same bag, so a connection declaring 6 defaults and 6 enforced is over it
311
+ // while neither map is. This is the boundary where the admin who created the
312
+ // squeeze is the one reading the warning.
313
+ const overBudget = queryMetadataBudgetWarning(declared);
314
+ if (overBudget) {
315
+ logger.warn("Connection query metadata will not apply as declared", {
316
+ connectionName: connection.name,
317
+ problem: overBudget,
318
+ });
319
+ }
320
+ }
321
+
274
322
  function validateConnectionShape(connection: ApiConnection): void {
275
323
  if (connection.proxy) {
276
324
  // A connection proxy makes THIS server open an outbound SSH tunnel to a
@@ -444,6 +492,32 @@ function validateConnectionShape(connection: ApiConnection): void {
444
492
  `Storage bucketUrl is required for DuckLake: ${connection.name}`,
445
493
  );
446
494
  }
495
+ // metadataSchema is optional, but when present it reaches the ATTACH as a
496
+ // quoted string literal AND the catalog-format preflight as a quoted
497
+ // identifier. Rather than escape one value for two grammars, restrict it
498
+ // to a plain identifier here — a deterministic config error, caught at
499
+ // load instead of at the connection's first attach.
500
+ //
501
+ // The typeof check is load-bearing, not defensive: the value arrives from
502
+ // untyped JSON, and RegExp.test() coerces its argument, so `true` and `null`
503
+ // both satisfy the pattern as "true"/"null" and would reach escapeSQL's
504
+ // String.replace as a non-string — a TypeError at the first attach, which is
505
+ // exactly the failure this check exists to turn into a config error.
506
+ if (
507
+ connection.ducklakeConnection.catalog.metadataSchema !== undefined
508
+ ) {
509
+ const schema = connection.ducklakeConnection.catalog.metadataSchema;
510
+ if (
511
+ typeof schema !== "string" ||
512
+ !/^[A-Za-z_][A-Za-z0-9_]*$/.test(schema)
513
+ ) {
514
+ throw new Error(
515
+ `DuckLake catalog metadataSchema must be a plain identifier ` +
516
+ `([A-Za-z_][A-Za-z0-9_]*), got '${schema}' for connection: ` +
517
+ `${connection.name}`,
518
+ );
519
+ }
520
+ }
447
521
  break;
448
522
  case "trino":
449
523
  if (!connection.trinoConnection) {
@@ -580,6 +654,7 @@ export function assembleEnvironmentConnections(
580
654
  processedConnections.add(connection.name);
581
655
  validateDuckdbApiSurface(connection);
582
656
  validateConnectionShape(connection);
657
+ warnOnConnectionQueryMetadata(connection);
583
658
 
584
659
  const apiConnection = cloneApiConnection(connection);
585
660
  apiConnection.attributes = getStaticConnectionAttributes(connection.type);
@@ -0,0 +1,137 @@
1
+ import { DuckDBConnection } from "@malloydata/db-duckdb";
2
+ import { afterEach, describe, expect, it } from "bun:test";
3
+ import fs from "fs/promises";
4
+ import os from "os";
5
+ import path from "path";
6
+
7
+ /**
8
+ * Pins the DuckDB instance-isolation contract that same-named connections depend
9
+ * on for multi-tenant safety.
10
+ *
11
+ * `@malloydata/db-duckdb` pools DuckDB instances in a process-global cache keyed
12
+ * by a share key that deliberately EXCLUDES the connection name, and it caches a
13
+ * `:memory:` primary like any other. Meanwhile the DuckLake attach aliases by
14
+ * connection name (`ATTACH OR REPLACE … AS <name>`). Put those together and two
15
+ * same-named connections that land on ONE pooled instance clobber each other's
16
+ * attach: one ends up reading the other's database.
17
+ *
18
+ * `createIsolatedBuildSession` prevents that by giving every build session a
19
+ * unique `workingDirectory`, relying on it participating in the share key. That
20
+ * is an upstream implementation detail, so nothing here asserted it — and the
21
+ * failure mode is silent: no error, just the wrong data. This spec asserts the
22
+ * behaviour instead of the mechanism, so it holds however upstream achieves it.
23
+ *
24
+ * Deliberately NOT asserted: the converse (same name AND same working directory
25
+ * DO share an instance). That is the pooling artifact, not a property we want; if
26
+ * upstream ever made pooling connection-aware, asserting it would fail on an
27
+ * improvement.
28
+ *
29
+ * Each connection creates and populates its OWN attached store rather than
30
+ * reopening a pre-seeded file. Handing a file from one connection to another
31
+ * requires the first to have released its OS lock, and `close()` is refcount-only
32
+ * — which fails outright under Windows' mandatory locking. Attaching a store the
33
+ * connection itself owns avoids the handoff, and matches how a DuckLake
34
+ * destination is actually used.
35
+ */
36
+ describe("DuckDB instance isolation", () => {
37
+ const tempDirs: string[] = [];
38
+ const openConnections: DuckDBConnection[] = [];
39
+
40
+ const tempDir = async (label: string): Promise<string> => {
41
+ const dir = await fs.mkdtemp(
42
+ path.join(os.tmpdir(), `duckdb-iso-${label}-`),
43
+ );
44
+ tempDirs.push(dir);
45
+ return dir;
46
+ };
47
+
48
+ afterEach(async () => {
49
+ for (const connection of openConnections) {
50
+ await connection.close().catch(() => undefined);
51
+ }
52
+ openConnections.length = 0;
53
+ for (const dir of tempDirs) {
54
+ // Best-effort: a still-locked file must not fail the test.
55
+ await fs
56
+ .rm(dir, { recursive: true, force: true })
57
+ .catch(() => undefined);
58
+ }
59
+ tempDirs.length = 0;
60
+ });
61
+
62
+ /**
63
+ * A connection with the given name and its own working directory, which
64
+ * attaches its own private store under `alias` and writes one distinguishing
65
+ * value into it.
66
+ */
67
+ const connectionWithOwnStore = async (
68
+ name: string,
69
+ label: string,
70
+ alias: string,
71
+ value: number,
72
+ ): Promise<DuckDBConnection> => {
73
+ const sessionDir = await tempDir(`session-${label}`);
74
+ const connection = new DuckDBConnection(name, ":memory:", sessionDir);
75
+ openConnections.push(connection);
76
+ const storeDir = await tempDir(`store-${label}`);
77
+ const storeFile = path.join(storeDir, `${label}.duckdb`);
78
+ // OR REPLACE mirrors the DuckLake attach: on a shared instance the second
79
+ // caller silently replaces the first caller's alias.
80
+ await connection.runSQL(`ATTACH OR REPLACE '${storeFile}' AS ${alias};`);
81
+ await connection.runSQL(
82
+ `CREATE OR REPLACE TABLE ${alias}.t AS SELECT ${value} AS x;`,
83
+ );
84
+ return connection;
85
+ };
86
+
87
+ const readValue = async (
88
+ connection: DuckDBConnection,
89
+ alias: string,
90
+ ): Promise<number> => {
91
+ const result = await connection.runSQL(`SELECT x FROM ${alias}.t;`);
92
+ return Number(Object.values(result.rows[0])[0]);
93
+ };
94
+
95
+ it("keeps same-named connections apart when their working directories differ", async () => {
96
+ // Identical connection NAME and identical `:memory:` primary — only the
97
+ // working directory differs. That is exactly the shape a build session
98
+ // uses, and the only thing standing between it and a shared instance.
99
+ //
100
+ // ORDERING IS LOAD-BEARING FOR THIS TEST. The instance cache is consulted
101
+ // inside an async `init()`, so constructing both connections before either
102
+ // is used lets both race past the cache miss and each create a private
103
+ // instance — isolation for the wrong reason, and a test that could never
104
+ // fail. Each helper call fully initializes its connection (the ATTACH
105
+ // forces it) before the next is built, so a shared share key really would
106
+ // hand back the cached instance. Verified: with identical working
107
+ // directories this ordering yields 2 for both connections — the clobber.
108
+ const a = await connectionWithOwnStore("store", "a", "lake", 1);
109
+ const b = await connectionWithOwnStore("store", "b", "lake", 2);
110
+
111
+ expect(await readValue(a, "lake")).toBe(1);
112
+ expect(await readValue(b, "lake")).toBe(2);
113
+ });
114
+
115
+ it("keeps a build-session-shaped connection apart from a long-lived one", async () => {
116
+ // The pairing the build-session comment calls out specifically: a transient
117
+ // build session overlapping a long-lived serve connection. Same name, same
118
+ // alias, both `:memory:` — distinct only by working directory.
119
+ const serve = await connectionWithOwnStore(
120
+ "credible",
121
+ "serve",
122
+ "lake",
123
+ 10,
124
+ );
125
+ const build = await connectionWithOwnStore(
126
+ "credible",
127
+ "build",
128
+ "lake",
129
+ 20,
130
+ );
131
+
132
+ // The serve connection must still see its own store after the build
133
+ // session attaches a different one under the same alias.
134
+ expect(await readValue(serve, "lake")).toBe(10);
135
+ expect(await readValue(build, "lake")).toBe(20);
136
+ });
137
+ });
@@ -1639,6 +1639,60 @@ export class Environment {
1639
1639
  logger.warn(`Could not read manifest for ${packageName}`);
1640
1640
  }
1641
1641
 
1642
+ const onDiskMaterialization =
1643
+ existingManifest.materialization !== null &&
1644
+ typeof existingManifest.materialization === "object" &&
1645
+ !Array.isArray(existingManifest.materialization)
1646
+ ? (existingManifest.materialization as Record<string, unknown>)
1647
+ : undefined;
1648
+
1649
+ // Scope has two homes: `materialization.scope` (canonical) and the
1650
+ // manifest root (deprecated). The server writes BOTH, in sync, for as
1651
+ // long as the root form is supported:
1652
+ //
1653
+ // - writing only the root would author a manifest this build's loader
1654
+ // refuses, since a root that disagrees with an existing envelope is a
1655
+ // conflict (see resolvePackageScope);
1656
+ // - writing only the envelope would silently downgrade a package read
1657
+ // by an older publisher, which knows only the root and would default
1658
+ // to `package` — cross-version table reuse for a package declared
1659
+ // `version`.
1660
+ //
1661
+ // A caller can only express scope through the top-level `scope` field
1662
+ // (the wire materialization block has no `scope`), so an envelope value
1663
+ // already on disk is preserved rather than dropped by a materialization
1664
+ // PATCH that says nothing about it.
1665
+ const resolvedScope =
1666
+ metadata.scope ??
1667
+ (onDiskMaterialization?.scope as ApiPackage["scope"] | undefined) ??
1668
+ (existingManifest.scope as ApiPackage["scope"] | undefined);
1669
+
1670
+ // A materialization PATCH replaces the block wholesale, which is right
1671
+ // for schedule and freshness — they are the policy the caller is
1672
+ // setting, and they are mutually exclusive with each other.
1673
+ // `queryMetadata` is orthogonal to both: a client setting a schedule
1674
+ // has no reason to re-send the package's tags, and dropping them
1675
+ // silently untags every statement the package's builds issue. So it is
1676
+ // preserved on omission, like `scope` above; an explicit null still
1677
+ // clears it, which keeps the block expressible.
1678
+ const preservedQueryMetadata =
1679
+ metadata.materialization !== undefined &&
1680
+ metadata.materialization?.queryMetadata === undefined &&
1681
+ onDiskMaterialization?.queryMetadata !== undefined
1682
+ ? { queryMetadata: onDiskMaterialization.queryMetadata }
1683
+ : {};
1684
+
1685
+ const materializationBase: Record<string, unknown> | undefined =
1686
+ metadata.materialization !== undefined
1687
+ ? { ...metadata.materialization, ...preservedQueryMetadata }
1688
+ : onDiskMaterialization !== undefined
1689
+ ? { ...onDiskMaterialization }
1690
+ : undefined;
1691
+ const materializationBlock =
1692
+ resolvedScope !== undefined
1693
+ ? { ...(materializationBase ?? {}), scope: resolvedScope }
1694
+ : materializationBase;
1695
+
1642
1696
  // Update with new metadata. `explores`/`queryableSources` are only
1643
1697
  // overwritten when the caller explicitly provides them; otherwise the
1644
1698
  // existing on-disk value is preserved via the spread (an undefined here
@@ -1656,9 +1710,9 @@ export class Environment {
1656
1710
  ...(metadata.manifestLocation !== undefined
1657
1711
  ? { manifestLocation: metadata.manifestLocation }
1658
1712
  : {}),
1659
- ...(metadata.scope !== undefined ? { scope: metadata.scope } : {}),
1660
- ...(metadata.materialization !== undefined
1661
- ? { materialization: metadata.materialization }
1713
+ ...(resolvedScope !== undefined ? { scope: resolvedScope } : {}),
1714
+ ...(materializationBlock !== undefined
1715
+ ? { materialization: materializationBlock }
1662
1716
  : {}),
1663
1717
  };
1664
1718
 
@@ -0,0 +1,99 @@
1
+ import { describe, expect, it } from "bun:test";
2
+ import type { components } from "../api";
3
+ import { materializationConfigWarnings } from "./materialization_config_validation";
4
+
5
+ const source = (
6
+ name: string,
7
+ queryMetadata: Record<string, string> | null,
8
+ ): components["schemas"]["PersistSourcePlan"] => ({
9
+ name,
10
+ sourceID: `${name}@m`,
11
+ connectionName: "duckdb",
12
+ sourceEntityId: "bid",
13
+ sql: "SELECT 1",
14
+ columns: [],
15
+ queryMetadata,
16
+ });
17
+
18
+ describe("materializationConfigWarnings", () => {
19
+ it("says nothing about a clean config", () => {
20
+ expect(
21
+ materializationConfigWarnings({
22
+ packageMaterialization: {
23
+ schedule: null,
24
+ freshness: null,
25
+ queryMetadata: { team: "finance" },
26
+ },
27
+ sources: [source("orders", { team: "finance" })],
28
+ }),
29
+ ).toEqual([]);
30
+ });
31
+
32
+ it("passes manifest deprecations through", () => {
33
+ expect(
34
+ materializationConfigWarnings({
35
+ manifestWarnings: ['"scope" at the manifest root is deprecated'],
36
+ }),
37
+ ).toEqual([{ message: '"scope" at the manifest root is deprecated' }]);
38
+ });
39
+
40
+ it("reports a package-level property that violates the contract", () => {
41
+ const warnings = materializationConfigWarnings({
42
+ packageMaterialization: {
43
+ schedule: null,
44
+ freshness: null,
45
+ queryMetadata: { "team.name": "finance" },
46
+ },
47
+ });
48
+ expect(warnings).toHaveLength(1);
49
+ expect(warnings[0].message).toContain("materialization.queryMetadata");
50
+ expect(warnings[0].message).toContain("team.name");
51
+ expect(warnings[0].target).toBeUndefined();
52
+ });
53
+
54
+ it("reports the offending property at the source that resolves it", () => {
55
+ const warnings = materializationConfigWarnings({
56
+ sources: [source("orders", { "team.name": "finance" })],
57
+ });
58
+ expect(warnings).toHaveLength(1);
59
+ expect(warnings[0].target).toBe("orders");
60
+ expect(warnings[0].message).toContain("#@ persist queryMetadata");
61
+ });
62
+
63
+ it("reports a BigQuery-dropped name as a warning, not silence", () => {
64
+ const warnings = materializationConfigWarnings({
65
+ sources: [source("orders", { _team: "finance" })],
66
+ });
67
+ expect(warnings).toHaveLength(1);
68
+ expect(warnings[0].message).toMatch(/BigQuery/);
69
+ });
70
+
71
+ it("reports the problem at every level that resolves it", () => {
72
+ // The package declares it and both sources inherit it. Reporting each
73
+ // level is what lets an author find the one they can edit; identical
74
+ // messages are collapsed, different levels are not.
75
+ const warnings = materializationConfigWarnings({
76
+ packageMaterialization: {
77
+ schedule: null,
78
+ freshness: null,
79
+ queryMetadata: { "team.name": "finance" },
80
+ },
81
+ sources: [
82
+ source("orders", { "team.name": "finance" }),
83
+ source("returns", { "team.name": "finance" }),
84
+ ],
85
+ });
86
+ expect(warnings).toHaveLength(3);
87
+ expect(warnings.map((w) => w.target)).toEqual([
88
+ undefined,
89
+ "orders",
90
+ "returns",
91
+ ]);
92
+ });
93
+
94
+ it("ignores sources with no metadata", () => {
95
+ expect(
96
+ materializationConfigWarnings({ sources: [source("orders", null)] }),
97
+ ).toEqual([]);
98
+ });
99
+ });
@@ -0,0 +1,120 @@
1
+ /**
2
+ * Publish-time validation of a package's materialization config.
3
+ *
4
+ * One module for the rules so they cannot drift between the levels a knob can be
5
+ * declared at. Two principles, both learned from shipped bugs:
6
+ *
7
+ * 1. **Nothing is silently dropped.** A declaration that will not do what it
8
+ * says has to surface somewhere — a publish error, a publish warning, a 400,
9
+ * or a metered runtime drop. A `#@ persist` name that was silently ignored
10
+ * (source published, never materialized, no error anywhere) is why
11
+ * `persist_annotation_validation.ts` exists; the same rule applies here.
12
+ * 2. **Strict at declaration, lenient at execution.** Publish and the API have a
13
+ * human behind them and report problems; the runtime resolve path clamps and
14
+ * meters instead, because metadata must never break a query or a build.
15
+ *
16
+ * Today this owns the `queryMetadata` rules and the manifest-shape deprecations,
17
+ * which are ADVISORY — they surface on the package's operator warnings array.
18
+ * They deliberately do NOT go through `Package.persistencePolicyWarnings`, which
19
+ * is a publish REJECTION gate (and disarms the scheduler): a deprecated manifest
20
+ * shape or a mistyped tag must not stop a package from publishing. The
21
+ * scope/schedule/freshness coherence rules that do reject still live there;
22
+ * moving them here is a mechanical follow-up with no behavior change.
23
+ */
24
+
25
+ import type { components } from "../api";
26
+ import {
27
+ queryMetadataAdvisoryWarnings,
28
+ queryMetadataBudgetWarning,
29
+ queryMetadataViolations,
30
+ type QueryMetadata,
31
+ } from "./query_metadata";
32
+
33
+ type WirePackageMaterialization =
34
+ components["schemas"]["PackageMaterializationConfig"];
35
+ type WirePersistSourcePlan = components["schemas"]["PersistSourcePlan"];
36
+
37
+ export interface MaterializationConfigInput {
38
+ /** The package manifest's `materialization` block, as parsed. */
39
+ packageMaterialization?: WirePackageMaterialization | null;
40
+ /** The compiled plan's sources, carrying each one's RESOLVED metadata. */
41
+ sources?: WirePersistSourcePlan[];
42
+ /**
43
+ * Deprecations the manifest parse tolerated (e.g. a root-level `scope`), which
44
+ * a load keeps working but a publish should report.
45
+ */
46
+ manifestWarnings?: string[];
47
+ }
48
+
49
+ /** One finding, in the shape the wire package's operator warnings array uses. */
50
+ export interface MaterializationConfigWarning {
51
+ /** The persist source the finding belongs to, absent for a package-level one. */
52
+ target?: string;
53
+ message: string;
54
+ }
55
+
56
+ function metadataWarnings(
57
+ level: string,
58
+ metadata: QueryMetadata | null | undefined,
59
+ target?: string,
60
+ ): MaterializationConfigWarning[] {
61
+ if (!metadata) return [];
62
+ const budget = queryMetadataBudgetWarning(Object.keys(metadata).length);
63
+ return [
64
+ ...queryMetadataViolations(metadata),
65
+ ...queryMetadataAdvisoryWarnings(metadata),
66
+ ...(budget ? [budget] : []),
67
+ ].map((message) => ({
68
+ // The level is in the message because a reader needs to know WHICH
69
+ // declaration to edit, and an inherited property has more than one.
70
+ message: `${level}: ${message}`,
71
+ ...(target ? { target } : {}),
72
+ }));
73
+ }
74
+
75
+ /**
76
+ * Everything wrong or inadvisable about a package's materialization config, as
77
+ * actionable messages. Warnings rather than errors: a package published before a
78
+ * rule existed must keep loading, and a bad metadata property degrades
79
+ * attribution rather than corrupting anything.
80
+ *
81
+ * A source's metadata is checked in its RESOLVED form (the effective bag from
82
+ * package → model-file → `#@ persist`), because that is what will actually be
83
+ * attached — a property inherited from the manifest is just as broken at the
84
+ * source as it is at the package, and reporting it at both is how an author finds
85
+ * the one they can edit.
86
+ */
87
+ export function materializationConfigWarnings(
88
+ input: MaterializationConfigInput,
89
+ ): MaterializationConfigWarning[] {
90
+ const warnings: MaterializationConfigWarning[] = (
91
+ input.manifestWarnings ?? []
92
+ ).map((message) => ({ message }));
93
+
94
+ warnings.push(
95
+ ...metadataWarnings(
96
+ "materialization.queryMetadata",
97
+ input.packageMaterialization?.queryMetadata,
98
+ ),
99
+ );
100
+
101
+ for (const source of input.sources ?? []) {
102
+ warnings.push(
103
+ ...metadataWarnings(
104
+ `#@ persist queryMetadata`,
105
+ source.queryMetadata,
106
+ source.name,
107
+ ),
108
+ );
109
+ }
110
+
111
+ // Identical findings collapse (two sources inheriting one bad package
112
+ // property produce one message each, not one per level per source).
113
+ const seen = new Set<string>();
114
+ return warnings.filter((warning) => {
115
+ const key = `${warning.target ?? ""}\u0000${warning.message}`;
116
+ if (seen.has(key)) return false;
117
+ seen.add(key);
118
+ return true;
119
+ });
120
+ }
@@ -54,6 +54,7 @@ describe("materialization schedule surfacing", () => {
54
54
  expect(pkg.getPackageMetadata().materialization).toEqual({
55
55
  schedule: "0 6 * * *",
56
56
  freshness: null,
57
+ queryMetadata: null,
57
58
  });
58
59
  },
59
60
  { timeout: 20000 },
@@ -77,6 +78,7 @@ describe("materialization schedule surfacing", () => {
77
78
  expect(pkg.getPackageMetadata().materialization).toEqual({
78
79
  schedule: null,
79
80
  freshness: { window: "24h", fallback: "stale_ok" },
81
+ queryMetadata: null,
80
82
  });
81
83
  },
82
84
  { timeout: 20000 },
@@ -119,14 +121,136 @@ describe("materialization schedule surfacing", () => {
119
121
  expect(updated.materialization).toEqual({
120
122
  schedule: "0 6 * * *",
121
123
  freshness: null,
124
+ queryMetadata: null,
122
125
  });
123
126
 
124
127
  const pkg = await env.getPackage("pkg", false);
125
128
  expect(pkg.getPackageMetadata().materialization).toEqual({
126
129
  schedule: "0 6 * * *",
127
130
  freshness: null,
131
+ queryMetadata: null,
128
132
  });
129
133
  },
130
134
  { timeout: 20000 },
131
135
  );
136
+
137
+ async function readManifest(): Promise<Record<string, unknown>> {
138
+ return JSON.parse(
139
+ await fs.readFile(path.join(envPath, "pkg", "publisher.json"), "utf8"),
140
+ );
141
+ }
142
+
143
+ it(
144
+ "writes scope to both homes so the manifest it authors still loads",
145
+ async () => {
146
+ // A scope PATCH used to write only the manifest root. With scope also
147
+ // living in the materialization block, that authored a manifest whose
148
+ // two homes disagreed — which the loader refuses, so the package
149
+ // survived the PATCH and then failed on the next restart.
150
+ const env = await Environment.create("testEnv", envPath, []);
151
+ await writePackageDir({
152
+ materialization: {
153
+ scope: "package",
154
+ queryMetadata: { team: "fin" },
155
+ },
156
+ });
157
+ await env.addPackage("pkg");
158
+
159
+ await env.updatePackage("pkg", { name: "pkg", scope: "version" });
160
+
161
+ const manifest = await readManifest();
162
+ expect(manifest.scope).toBe("version");
163
+ expect(manifest.materialization).toMatchObject({
164
+ scope: "version",
165
+ queryMetadata: { team: "fin" },
166
+ });
167
+
168
+ // The real assertion: it loads again.
169
+ const reloaded = await Environment.create("testEnv", envPath, []);
170
+ await reloaded.addPackage("pkg");
171
+ const pkg = await reloaded.getPackage("pkg", false);
172
+ expect(pkg.getPackageMetadata().scope).toBe("version");
173
+ },
174
+ { timeout: 20000 },
175
+ );
176
+
177
+ it(
178
+ "keeps the envelope scope a materialization PATCH cannot express",
179
+ async () => {
180
+ // The wire materialization block has no `scope`, so a PATCH that sends
181
+ // one must not be read as "the author dropped it".
182
+ const env = await Environment.create("testEnv", envPath, []);
183
+ await writePackageDir({ materialization: { scope: "version" } });
184
+ await env.addPackage("pkg");
185
+
186
+ await env.updatePackage("pkg", {
187
+ name: "pkg",
188
+ materialization: { schedule: "0 6 * * *" },
189
+ });
190
+
191
+ const manifest = await readManifest();
192
+ expect(manifest.materialization).toMatchObject({
193
+ scope: "version",
194
+ schedule: "0 6 * * *",
195
+ });
196
+ expect(manifest.scope).toBe("version");
197
+ },
198
+ { timeout: 20000 },
199
+ );
200
+
201
+ it(
202
+ "keeps queryMetadata a schedule PATCH said nothing about",
203
+ async () => {
204
+ // The block is replaced wholesale, which is right for the policy the
205
+ // caller is setting — but queryMetadata is orthogonal to it, so a
206
+ // client that sets a schedule without re-sending the tags would
207
+ // silently untag every statement the package's builds issue. The UI
208
+ // and the control plane are separate clients; preserving it here fixes
209
+ // all of them at once.
210
+ const env = await Environment.create("testEnv", envPath, []);
211
+ await writePackageDir({
212
+ materialization: {
213
+ scope: "version",
214
+ queryMetadata: { team: "finance" },
215
+ },
216
+ });
217
+ await env.addPackage("pkg");
218
+
219
+ await env.updatePackage("pkg", {
220
+ name: "pkg",
221
+ materialization: { schedule: "0 6 * * *" },
222
+ });
223
+
224
+ expect((await readManifest()).materialization).toMatchObject({
225
+ schedule: "0 6 * * *",
226
+ queryMetadata: { team: "finance" },
227
+ });
228
+ },
229
+ { timeout: 20000 },
230
+ );
231
+
232
+ it(
233
+ "still lets an explicit null clear queryMetadata",
234
+ async () => {
235
+ // Preserved-on-omission must not make it unclearable.
236
+ const env = await Environment.create("testEnv", envPath, []);
237
+ await writePackageDir({
238
+ materialization: {
239
+ scope: "version",
240
+ queryMetadata: { team: "finance" },
241
+ },
242
+ });
243
+ await env.addPackage("pkg");
244
+
245
+ await env.updatePackage("pkg", {
246
+ name: "pkg",
247
+ materialization: { schedule: "0 6 * * *", queryMetadata: null },
248
+ });
249
+
250
+ expect(
251
+ (await readManifest()).materialization as Record<string, unknown>,
252
+ ).toMatchObject({ schedule: "0 6 * * *", queryMetadata: null });
253
+ },
254
+ { timeout: 20000 },
255
+ );
132
256
  });