@malloy-publisher/server 0.0.232 → 0.0.234
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.docker.md +1 -0
- package/dist/app/api-doc.yaml +269 -10
- package/dist/app/assets/{EnvironmentPage-DXEaZIPx.js → EnvironmentPage-DTZQ4Gxc.js} +1 -1
- package/dist/app/assets/{HomePage-kofsqpZt.js → HomePage-C5mlDPXK.js} +1 -1
- package/dist/app/assets/{LightMode-CNhIlIlJ.js → LightMode-DGNmhG0u.js} +1 -1
- package/dist/app/assets/{MainPage-Bgqo8jCy.js → MainPage-CVL_wmP4.js} +1 -1
- package/dist/app/assets/{MaterializationsPage-CgBlgGz2.js → MaterializationsPage-DmzMBCpy.js} +1 -1
- package/dist/app/assets/{ModelPage-B0TjoDtf.js → ModelPage-Dbvf4QbB.js} +1 -1
- package/dist/app/assets/{PackagePage-BL8vnFj1.js → PackagePage-DxdHc2Qs.js} +1 -1
- package/dist/app/assets/{RouteError-BzPby0X2.js → RouteError-OJdT4tCd.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-CTEP_9r3.js → ThemeEditorPage-Bk7s0KXY.js} +1 -1
- package/dist/app/assets/{WorkbookPage-BwM3BmKw.js → WorkbookPage-j_vCWdN3.js} +1 -1
- package/dist/app/assets/{core-CK68iv6w.es-CpRxXBt7.js → core-Rj_4rRnA.es-DoIfLxDJ.js} +1 -1
- package/dist/app/assets/{index-B33zGctF.js → index-B_jKMR35.js} +4 -4
- package/dist/app/assets/{index-CmkW1MiE.js → index-D-rDyK11.js} +1 -1
- package/dist/app/assets/{index-tXJXwdyj.js → index-DWIe_hK0.js} +1 -1
- package/dist/app/assets/{index-BkiWKaAF.js → index-hw-xn0X7.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/package_load_worker.mjs +53 -3
- package/dist/server.mjs +20277 -925
- package/package.json +1 -1
- package/src/config.ts +35 -1
- package/src/controller/connection.controller.spec.ts +46 -0
- package/src/controller/connection.controller.ts +105 -2
- package/src/controller/materialization.controller.spec.ts +25 -0
- package/src/controller/materialization.controller.ts +60 -0
- package/src/controller/model.controller.ts +24 -0
- package/src/controller/query.controller.ts +83 -10
- package/src/json_utils.spec.ts +51 -0
- package/src/json_utils.ts +33 -0
- package/src/mcp/handler_utils.ts +10 -2
- package/src/mcp/query_envelope.spec.ts +229 -0
- package/src/mcp/query_envelope.ts +240 -0
- package/src/mcp/server.protocol.spec.ts +128 -16
- package/src/mcp/skills/build_skills_bundle.ts +94 -4
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/skills/skills_bundle.spec.ts +113 -4
- package/src/mcp/tool_response.spec.ts +108 -0
- package/src/mcp/tool_response.ts +138 -0
- package/src/mcp/tools/compile_tool.spec.ts +112 -4
- package/src/mcp/tools/compile_tool.ts +61 -30
- package/src/mcp/tools/docs_search_tool.ts +6 -16
- package/src/mcp/tools/execute_query_tool.spec.ts +154 -3
- package/src/mcp/tools/execute_query_tool.ts +131 -155
- package/src/mcp/tools/get_context_tool.spec.ts +63 -3
- package/src/mcp/tools/get_context_tool.ts +43 -46
- package/src/mcp/tools/reload_package_tool.ts +3 -29
- package/src/mcp_config.spec.ts +919 -0
- package/src/mcp_config.ts +425 -0
- package/src/oom_guards.integration.spec.ts +11 -3
- package/src/package_load/package_load_pool.ts +2 -0
- package/src/package_load/package_load_worker.ts +17 -5
- package/src/package_load/protocol.ts +6 -0
- package/src/query_metadata_metrics.ts +49 -0
- package/src/server.ts +99 -3
- package/src/service/build_plan.spec.ts +125 -0
- package/src/service/build_plan.ts +108 -7
- package/src/service/compile_fragment_techniques.spec.ts +156 -0
- package/src/service/connection.spec.ts +371 -1
- package/src/service/connection.ts +77 -14
- package/src/service/connection_config.spec.ts +60 -0
- package/src/service/connection_config.ts +75 -0
- package/src/service/duckdb_instance_isolation.spec.ts +137 -0
- package/src/service/environment.ts +57 -3
- package/src/service/materialization_config_validation.spec.ts +99 -0
- package/src/service/materialization_config_validation.ts +120 -0
- package/src/service/materialization_schedule_surface.spec.ts +124 -0
- package/src/service/materialization_service.spec.ts +119 -0
- package/src/service/materialization_service.ts +186 -3
- package/src/service/materialization_test_fixtures.ts +86 -21
- package/src/service/model.spec.ts +45 -1
- package/src/service/model.ts +171 -23
- package/src/service/model_limits.spec.ts +28 -0
- package/src/service/model_limits.ts +21 -0
- package/src/service/package.ts +24 -1
- package/src/service/package_manifest.spec.ts +137 -4
- package/src/service/package_manifest.ts +140 -5
- package/src/service/persist_annotation_validation.spec.ts +12 -0
- package/src/service/persist_annotation_validation.ts +9 -4
- package/src/service/query_metadata.spec.ts +408 -0
- package/src/service/query_metadata.ts +492 -0
- package/src/service/query_metadata_identity.spec.ts +149 -0
- package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +37 -12
|
@@ -2,7 +2,13 @@ import { createPrivateKey } from "crypto";
|
|
|
2
2
|
import { existsSync } from "fs";
|
|
3
3
|
import path from "path";
|
|
4
4
|
import { components } from "../api";
|
|
5
|
+
import { logger } from "../logger";
|
|
5
6
|
import { parseHostKeys } from "./proxy";
|
|
7
|
+
import {
|
|
8
|
+
queryMetadataAdvisoryWarnings,
|
|
9
|
+
queryMetadataBudgetWarning,
|
|
10
|
+
queryMetadataViolations,
|
|
11
|
+
} from "./query_metadata";
|
|
6
12
|
|
|
7
13
|
type ApiConnection = components["schemas"]["Connection"];
|
|
8
14
|
type AttachedDatabase = components["schemas"]["AttachedDatabase"];
|
|
@@ -271,6 +277,48 @@ function buildDuckdbEntry(
|
|
|
271
277
|
};
|
|
272
278
|
}
|
|
273
279
|
|
|
280
|
+
/**
|
|
281
|
+
* Report a connection default that will not do what it says — a property name
|
|
282
|
+
* the contract rejects, one BigQuery would drop, a bag with no room for the
|
|
283
|
+
* server's own context.
|
|
284
|
+
*
|
|
285
|
+
* Warns rather than throws, unlike everything else in this file: query metadata
|
|
286
|
+
* is observability, and an environment that refuses to load because a tag has a
|
|
287
|
+
* hyphen in it would trade a missing label for an outage. The connection update
|
|
288
|
+
* API rejects the same bag outright (see validateAdminAuthoredConnection) —
|
|
289
|
+
* strict where a human is waiting, lenient where a config is being loaded.
|
|
290
|
+
*/
|
|
291
|
+
function warnOnConnectionQueryMetadata(connection: ApiConnection): void {
|
|
292
|
+
let declared = 0;
|
|
293
|
+
for (const field of ["queryMetadata", "queryMetadataEnforced"] as const) {
|
|
294
|
+
const metadata = connection[field];
|
|
295
|
+
if (!metadata) continue;
|
|
296
|
+
declared += Object.keys(metadata).length;
|
|
297
|
+
const problems = [
|
|
298
|
+
...queryMetadataViolations(metadata),
|
|
299
|
+
...queryMetadataAdvisoryWarnings(metadata),
|
|
300
|
+
];
|
|
301
|
+
for (const problem of problems) {
|
|
302
|
+
logger.warn("Connection query metadata will not apply as declared", {
|
|
303
|
+
connectionName: connection.name,
|
|
304
|
+
field,
|
|
305
|
+
problem,
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
// The budget is checked over BOTH maps, not each one: they merge into the
|
|
310
|
+
// same bag, so a connection declaring 6 defaults and 6 enforced is over it
|
|
311
|
+
// while neither map is. This is the boundary where the admin who created the
|
|
312
|
+
// squeeze is the one reading the warning.
|
|
313
|
+
const overBudget = queryMetadataBudgetWarning(declared);
|
|
314
|
+
if (overBudget) {
|
|
315
|
+
logger.warn("Connection query metadata will not apply as declared", {
|
|
316
|
+
connectionName: connection.name,
|
|
317
|
+
problem: overBudget,
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
274
322
|
function validateConnectionShape(connection: ApiConnection): void {
|
|
275
323
|
if (connection.proxy) {
|
|
276
324
|
// A connection proxy makes THIS server open an outbound SSH tunnel to a
|
|
@@ -444,6 +492,32 @@ function validateConnectionShape(connection: ApiConnection): void {
|
|
|
444
492
|
`Storage bucketUrl is required for DuckLake: ${connection.name}`,
|
|
445
493
|
);
|
|
446
494
|
}
|
|
495
|
+
// metadataSchema is optional, but when present it reaches the ATTACH as a
|
|
496
|
+
// quoted string literal AND the catalog-format preflight as a quoted
|
|
497
|
+
// identifier. Rather than escape one value for two grammars, restrict it
|
|
498
|
+
// to a plain identifier here — a deterministic config error, caught at
|
|
499
|
+
// load instead of at the connection's first attach.
|
|
500
|
+
//
|
|
501
|
+
// The typeof check is load-bearing, not defensive: the value arrives from
|
|
502
|
+
// untyped JSON, and RegExp.test() coerces its argument, so `true` and `null`
|
|
503
|
+
// both satisfy the pattern as "true"/"null" and would reach escapeSQL's
|
|
504
|
+
// String.replace as a non-string — a TypeError at the first attach, which is
|
|
505
|
+
// exactly the failure this check exists to turn into a config error.
|
|
506
|
+
if (
|
|
507
|
+
connection.ducklakeConnection.catalog.metadataSchema !== undefined
|
|
508
|
+
) {
|
|
509
|
+
const schema = connection.ducklakeConnection.catalog.metadataSchema;
|
|
510
|
+
if (
|
|
511
|
+
typeof schema !== "string" ||
|
|
512
|
+
!/^[A-Za-z_][A-Za-z0-9_]*$/.test(schema)
|
|
513
|
+
) {
|
|
514
|
+
throw new Error(
|
|
515
|
+
`DuckLake catalog metadataSchema must be a plain identifier ` +
|
|
516
|
+
`([A-Za-z_][A-Za-z0-9_]*), got '${schema}' for connection: ` +
|
|
517
|
+
`${connection.name}`,
|
|
518
|
+
);
|
|
519
|
+
}
|
|
520
|
+
}
|
|
447
521
|
break;
|
|
448
522
|
case "trino":
|
|
449
523
|
if (!connection.trinoConnection) {
|
|
@@ -580,6 +654,7 @@ export function assembleEnvironmentConnections(
|
|
|
580
654
|
processedConnections.add(connection.name);
|
|
581
655
|
validateDuckdbApiSurface(connection);
|
|
582
656
|
validateConnectionShape(connection);
|
|
657
|
+
warnOnConnectionQueryMetadata(connection);
|
|
583
658
|
|
|
584
659
|
const apiConnection = cloneApiConnection(connection);
|
|
585
660
|
apiConnection.attributes = getStaticConnectionAttributes(connection.type);
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { DuckDBConnection } from "@malloydata/db-duckdb";
|
|
2
|
+
import { afterEach, describe, expect, it } from "bun:test";
|
|
3
|
+
import fs from "fs/promises";
|
|
4
|
+
import os from "os";
|
|
5
|
+
import path from "path";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Pins the DuckDB instance-isolation contract that same-named connections depend
|
|
9
|
+
* on for multi-tenant safety.
|
|
10
|
+
*
|
|
11
|
+
* `@malloydata/db-duckdb` pools DuckDB instances in a process-global cache keyed
|
|
12
|
+
* by a share key that deliberately EXCLUDES the connection name, and it caches a
|
|
13
|
+
* `:memory:` primary like any other. Meanwhile the DuckLake attach aliases by
|
|
14
|
+
* connection name (`ATTACH OR REPLACE … AS <name>`). Put those together and two
|
|
15
|
+
* same-named connections that land on ONE pooled instance clobber each other's
|
|
16
|
+
* attach: one ends up reading the other's database.
|
|
17
|
+
*
|
|
18
|
+
* `createIsolatedBuildSession` prevents that by giving every build session a
|
|
19
|
+
* unique `workingDirectory`, relying on it participating in the share key. That
|
|
20
|
+
* is an upstream implementation detail, so nothing here asserted it — and the
|
|
21
|
+
* failure mode is silent: no error, just the wrong data. This spec asserts the
|
|
22
|
+
* behaviour instead of the mechanism, so it holds however upstream achieves it.
|
|
23
|
+
*
|
|
24
|
+
* Deliberately NOT asserted: the converse (same name AND same working directory
|
|
25
|
+
* DO share an instance). That is the pooling artifact, not a property we want; if
|
|
26
|
+
* upstream ever made pooling connection-aware, asserting it would fail on an
|
|
27
|
+
* improvement.
|
|
28
|
+
*
|
|
29
|
+
* Each connection creates and populates its OWN attached store rather than
|
|
30
|
+
* reopening a pre-seeded file. Handing a file from one connection to another
|
|
31
|
+
* requires the first to have released its OS lock, and `close()` is refcount-only
|
|
32
|
+
* — which fails outright under Windows' mandatory locking. Attaching a store the
|
|
33
|
+
* connection itself owns avoids the handoff, and matches how a DuckLake
|
|
34
|
+
* destination is actually used.
|
|
35
|
+
*/
|
|
36
|
+
describe("DuckDB instance isolation", () => {
|
|
37
|
+
const tempDirs: string[] = [];
|
|
38
|
+
const openConnections: DuckDBConnection[] = [];
|
|
39
|
+
|
|
40
|
+
const tempDir = async (label: string): Promise<string> => {
|
|
41
|
+
const dir = await fs.mkdtemp(
|
|
42
|
+
path.join(os.tmpdir(), `duckdb-iso-${label}-`),
|
|
43
|
+
);
|
|
44
|
+
tempDirs.push(dir);
|
|
45
|
+
return dir;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
afterEach(async () => {
|
|
49
|
+
for (const connection of openConnections) {
|
|
50
|
+
await connection.close().catch(() => undefined);
|
|
51
|
+
}
|
|
52
|
+
openConnections.length = 0;
|
|
53
|
+
for (const dir of tempDirs) {
|
|
54
|
+
// Best-effort: a still-locked file must not fail the test.
|
|
55
|
+
await fs
|
|
56
|
+
.rm(dir, { recursive: true, force: true })
|
|
57
|
+
.catch(() => undefined);
|
|
58
|
+
}
|
|
59
|
+
tempDirs.length = 0;
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* A connection with the given name and its own working directory, which
|
|
64
|
+
* attaches its own private store under `alias` and writes one distinguishing
|
|
65
|
+
* value into it.
|
|
66
|
+
*/
|
|
67
|
+
const connectionWithOwnStore = async (
|
|
68
|
+
name: string,
|
|
69
|
+
label: string,
|
|
70
|
+
alias: string,
|
|
71
|
+
value: number,
|
|
72
|
+
): Promise<DuckDBConnection> => {
|
|
73
|
+
const sessionDir = await tempDir(`session-${label}`);
|
|
74
|
+
const connection = new DuckDBConnection(name, ":memory:", sessionDir);
|
|
75
|
+
openConnections.push(connection);
|
|
76
|
+
const storeDir = await tempDir(`store-${label}`);
|
|
77
|
+
const storeFile = path.join(storeDir, `${label}.duckdb`);
|
|
78
|
+
// OR REPLACE mirrors the DuckLake attach: on a shared instance the second
|
|
79
|
+
// caller silently replaces the first caller's alias.
|
|
80
|
+
await connection.runSQL(`ATTACH OR REPLACE '${storeFile}' AS ${alias};`);
|
|
81
|
+
await connection.runSQL(
|
|
82
|
+
`CREATE OR REPLACE TABLE ${alias}.t AS SELECT ${value} AS x;`,
|
|
83
|
+
);
|
|
84
|
+
return connection;
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
const readValue = async (
|
|
88
|
+
connection: DuckDBConnection,
|
|
89
|
+
alias: string,
|
|
90
|
+
): Promise<number> => {
|
|
91
|
+
const result = await connection.runSQL(`SELECT x FROM ${alias}.t;`);
|
|
92
|
+
return Number(Object.values(result.rows[0])[0]);
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
it("keeps same-named connections apart when their working directories differ", async () => {
|
|
96
|
+
// Identical connection NAME and identical `:memory:` primary — only the
|
|
97
|
+
// working directory differs. That is exactly the shape a build session
|
|
98
|
+
// uses, and the only thing standing between it and a shared instance.
|
|
99
|
+
//
|
|
100
|
+
// ORDERING IS LOAD-BEARING FOR THIS TEST. The instance cache is consulted
|
|
101
|
+
// inside an async `init()`, so constructing both connections before either
|
|
102
|
+
// is used lets both race past the cache miss and each create a private
|
|
103
|
+
// instance — isolation for the wrong reason, and a test that could never
|
|
104
|
+
// fail. Each helper call fully initializes its connection (the ATTACH
|
|
105
|
+
// forces it) before the next is built, so a shared share key really would
|
|
106
|
+
// hand back the cached instance. Verified: with identical working
|
|
107
|
+
// directories this ordering yields 2 for both connections — the clobber.
|
|
108
|
+
const a = await connectionWithOwnStore("store", "a", "lake", 1);
|
|
109
|
+
const b = await connectionWithOwnStore("store", "b", "lake", 2);
|
|
110
|
+
|
|
111
|
+
expect(await readValue(a, "lake")).toBe(1);
|
|
112
|
+
expect(await readValue(b, "lake")).toBe(2);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it("keeps a build-session-shaped connection apart from a long-lived one", async () => {
|
|
116
|
+
// The pairing the build-session comment calls out specifically: a transient
|
|
117
|
+
// build session overlapping a long-lived serve connection. Same name, same
|
|
118
|
+
// alias, both `:memory:` — distinct only by working directory.
|
|
119
|
+
const serve = await connectionWithOwnStore(
|
|
120
|
+
"credible",
|
|
121
|
+
"serve",
|
|
122
|
+
"lake",
|
|
123
|
+
10,
|
|
124
|
+
);
|
|
125
|
+
const build = await connectionWithOwnStore(
|
|
126
|
+
"credible",
|
|
127
|
+
"build",
|
|
128
|
+
"lake",
|
|
129
|
+
20,
|
|
130
|
+
);
|
|
131
|
+
|
|
132
|
+
// The serve connection must still see its own store after the build
|
|
133
|
+
// session attaches a different one under the same alias.
|
|
134
|
+
expect(await readValue(serve, "lake")).toBe(10);
|
|
135
|
+
expect(await readValue(build, "lake")).toBe(20);
|
|
136
|
+
});
|
|
137
|
+
});
|
|
@@ -1639,6 +1639,60 @@ export class Environment {
|
|
|
1639
1639
|
logger.warn(`Could not read manifest for ${packageName}`);
|
|
1640
1640
|
}
|
|
1641
1641
|
|
|
1642
|
+
const onDiskMaterialization =
|
|
1643
|
+
existingManifest.materialization !== null &&
|
|
1644
|
+
typeof existingManifest.materialization === "object" &&
|
|
1645
|
+
!Array.isArray(existingManifest.materialization)
|
|
1646
|
+
? (existingManifest.materialization as Record<string, unknown>)
|
|
1647
|
+
: undefined;
|
|
1648
|
+
|
|
1649
|
+
// Scope has two homes: `materialization.scope` (canonical) and the
|
|
1650
|
+
// manifest root (deprecated). The server writes BOTH, in sync, for as
|
|
1651
|
+
// long as the root form is supported:
|
|
1652
|
+
//
|
|
1653
|
+
// - writing only the root would author a manifest this build's loader
|
|
1654
|
+
// refuses, since a root that disagrees with an existing envelope is a
|
|
1655
|
+
// conflict (see resolvePackageScope);
|
|
1656
|
+
// - writing only the envelope would silently downgrade a package read
|
|
1657
|
+
// by an older publisher, which knows only the root and would default
|
|
1658
|
+
// to `package` — cross-version table reuse for a package declared
|
|
1659
|
+
// `version`.
|
|
1660
|
+
//
|
|
1661
|
+
// A caller can only express scope through the top-level `scope` field
|
|
1662
|
+
// (the wire materialization block has no `scope`), so an envelope value
|
|
1663
|
+
// already on disk is preserved rather than dropped by a materialization
|
|
1664
|
+
// PATCH that says nothing about it.
|
|
1665
|
+
const resolvedScope =
|
|
1666
|
+
metadata.scope ??
|
|
1667
|
+
(onDiskMaterialization?.scope as ApiPackage["scope"] | undefined) ??
|
|
1668
|
+
(existingManifest.scope as ApiPackage["scope"] | undefined);
|
|
1669
|
+
|
|
1670
|
+
// A materialization PATCH replaces the block wholesale, which is right
|
|
1671
|
+
// for schedule and freshness — they are the policy the caller is
|
|
1672
|
+
// setting, and they are mutually exclusive with each other.
|
|
1673
|
+
// `queryMetadata` is orthogonal to both: a client setting a schedule
|
|
1674
|
+
// has no reason to re-send the package's tags, and dropping them
|
|
1675
|
+
// silently untags every statement the package's builds issue. So it is
|
|
1676
|
+
// preserved on omission, like `scope` above; an explicit null still
|
|
1677
|
+
// clears it, which keeps the block expressible.
|
|
1678
|
+
const preservedQueryMetadata =
|
|
1679
|
+
metadata.materialization !== undefined &&
|
|
1680
|
+
metadata.materialization?.queryMetadata === undefined &&
|
|
1681
|
+
onDiskMaterialization?.queryMetadata !== undefined
|
|
1682
|
+
? { queryMetadata: onDiskMaterialization.queryMetadata }
|
|
1683
|
+
: {};
|
|
1684
|
+
|
|
1685
|
+
const materializationBase: Record<string, unknown> | undefined =
|
|
1686
|
+
metadata.materialization !== undefined
|
|
1687
|
+
? { ...metadata.materialization, ...preservedQueryMetadata }
|
|
1688
|
+
: onDiskMaterialization !== undefined
|
|
1689
|
+
? { ...onDiskMaterialization }
|
|
1690
|
+
: undefined;
|
|
1691
|
+
const materializationBlock =
|
|
1692
|
+
resolvedScope !== undefined
|
|
1693
|
+
? { ...(materializationBase ?? {}), scope: resolvedScope }
|
|
1694
|
+
: materializationBase;
|
|
1695
|
+
|
|
1642
1696
|
// Update with new metadata. `explores`/`queryableSources` are only
|
|
1643
1697
|
// overwritten when the caller explicitly provides them; otherwise the
|
|
1644
1698
|
// existing on-disk value is preserved via the spread (an undefined here
|
|
@@ -1656,9 +1710,9 @@ export class Environment {
|
|
|
1656
1710
|
...(metadata.manifestLocation !== undefined
|
|
1657
1711
|
? { manifestLocation: metadata.manifestLocation }
|
|
1658
1712
|
: {}),
|
|
1659
|
-
...(
|
|
1660
|
-
...(
|
|
1661
|
-
? { materialization:
|
|
1713
|
+
...(resolvedScope !== undefined ? { scope: resolvedScope } : {}),
|
|
1714
|
+
...(materializationBlock !== undefined
|
|
1715
|
+
? { materialization: materializationBlock }
|
|
1662
1716
|
: {}),
|
|
1663
1717
|
};
|
|
1664
1718
|
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
import type { components } from "../api";
|
|
3
|
+
import { materializationConfigWarnings } from "./materialization_config_validation";
|
|
4
|
+
|
|
5
|
+
const source = (
|
|
6
|
+
name: string,
|
|
7
|
+
queryMetadata: Record<string, string> | null,
|
|
8
|
+
): components["schemas"]["PersistSourcePlan"] => ({
|
|
9
|
+
name,
|
|
10
|
+
sourceID: `${name}@m`,
|
|
11
|
+
connectionName: "duckdb",
|
|
12
|
+
sourceEntityId: "bid",
|
|
13
|
+
sql: "SELECT 1",
|
|
14
|
+
columns: [],
|
|
15
|
+
queryMetadata,
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
describe("materializationConfigWarnings", () => {
|
|
19
|
+
it("says nothing about a clean config", () => {
|
|
20
|
+
expect(
|
|
21
|
+
materializationConfigWarnings({
|
|
22
|
+
packageMaterialization: {
|
|
23
|
+
schedule: null,
|
|
24
|
+
freshness: null,
|
|
25
|
+
queryMetadata: { team: "finance" },
|
|
26
|
+
},
|
|
27
|
+
sources: [source("orders", { team: "finance" })],
|
|
28
|
+
}),
|
|
29
|
+
).toEqual([]);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
it("passes manifest deprecations through", () => {
|
|
33
|
+
expect(
|
|
34
|
+
materializationConfigWarnings({
|
|
35
|
+
manifestWarnings: ['"scope" at the manifest root is deprecated'],
|
|
36
|
+
}),
|
|
37
|
+
).toEqual([{ message: '"scope" at the manifest root is deprecated' }]);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
it("reports a package-level property that violates the contract", () => {
|
|
41
|
+
const warnings = materializationConfigWarnings({
|
|
42
|
+
packageMaterialization: {
|
|
43
|
+
schedule: null,
|
|
44
|
+
freshness: null,
|
|
45
|
+
queryMetadata: { "team.name": "finance" },
|
|
46
|
+
},
|
|
47
|
+
});
|
|
48
|
+
expect(warnings).toHaveLength(1);
|
|
49
|
+
expect(warnings[0].message).toContain("materialization.queryMetadata");
|
|
50
|
+
expect(warnings[0].message).toContain("team.name");
|
|
51
|
+
expect(warnings[0].target).toBeUndefined();
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it("reports the offending property at the source that resolves it", () => {
|
|
55
|
+
const warnings = materializationConfigWarnings({
|
|
56
|
+
sources: [source("orders", { "team.name": "finance" })],
|
|
57
|
+
});
|
|
58
|
+
expect(warnings).toHaveLength(1);
|
|
59
|
+
expect(warnings[0].target).toBe("orders");
|
|
60
|
+
expect(warnings[0].message).toContain("#@ persist queryMetadata");
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
it("reports a BigQuery-dropped name as a warning, not silence", () => {
|
|
64
|
+
const warnings = materializationConfigWarnings({
|
|
65
|
+
sources: [source("orders", { _team: "finance" })],
|
|
66
|
+
});
|
|
67
|
+
expect(warnings).toHaveLength(1);
|
|
68
|
+
expect(warnings[0].message).toMatch(/BigQuery/);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it("reports the problem at every level that resolves it", () => {
|
|
72
|
+
// The package declares it and both sources inherit it. Reporting each
|
|
73
|
+
// level is what lets an author find the one they can edit; identical
|
|
74
|
+
// messages are collapsed, different levels are not.
|
|
75
|
+
const warnings = materializationConfigWarnings({
|
|
76
|
+
packageMaterialization: {
|
|
77
|
+
schedule: null,
|
|
78
|
+
freshness: null,
|
|
79
|
+
queryMetadata: { "team.name": "finance" },
|
|
80
|
+
},
|
|
81
|
+
sources: [
|
|
82
|
+
source("orders", { "team.name": "finance" }),
|
|
83
|
+
source("returns", { "team.name": "finance" }),
|
|
84
|
+
],
|
|
85
|
+
});
|
|
86
|
+
expect(warnings).toHaveLength(3);
|
|
87
|
+
expect(warnings.map((w) => w.target)).toEqual([
|
|
88
|
+
undefined,
|
|
89
|
+
"orders",
|
|
90
|
+
"returns",
|
|
91
|
+
]);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
it("ignores sources with no metadata", () => {
|
|
95
|
+
expect(
|
|
96
|
+
materializationConfigWarnings({ sources: [source("orders", null)] }),
|
|
97
|
+
).toEqual([]);
|
|
98
|
+
});
|
|
99
|
+
});
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Publish-time validation of a package's materialization config.
|
|
3
|
+
*
|
|
4
|
+
* One module for the rules so they cannot drift between the levels a knob can be
|
|
5
|
+
* declared at. Two principles, both learned from shipped bugs:
|
|
6
|
+
*
|
|
7
|
+
* 1. **Nothing is silently dropped.** A declaration that will not do what it
|
|
8
|
+
* says has to surface somewhere — a publish error, a publish warning, a 400,
|
|
9
|
+
* or a metered runtime drop. A `#@ persist` name that was silently ignored
|
|
10
|
+
* (source published, never materialized, no error anywhere) is why
|
|
11
|
+
* `persist_annotation_validation.ts` exists; the same rule applies here.
|
|
12
|
+
* 2. **Strict at declaration, lenient at execution.** Publish and the API have a
|
|
13
|
+
* human behind them and report problems; the runtime resolve path clamps and
|
|
14
|
+
* meters instead, because metadata must never break a query or a build.
|
|
15
|
+
*
|
|
16
|
+
* Today this owns the `queryMetadata` rules and the manifest-shape deprecations,
|
|
17
|
+
* which are ADVISORY — they surface on the package's operator warnings array.
|
|
18
|
+
* They deliberately do NOT go through `Package.persistencePolicyWarnings`, which
|
|
19
|
+
* is a publish REJECTION gate (and disarms the scheduler): a deprecated manifest
|
|
20
|
+
* shape or a mistyped tag must not stop a package from publishing. The
|
|
21
|
+
* scope/schedule/freshness coherence rules that do reject still live there;
|
|
22
|
+
* moving them here is a mechanical follow-up with no behavior change.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import type { components } from "../api";
|
|
26
|
+
import {
|
|
27
|
+
queryMetadataAdvisoryWarnings,
|
|
28
|
+
queryMetadataBudgetWarning,
|
|
29
|
+
queryMetadataViolations,
|
|
30
|
+
type QueryMetadata,
|
|
31
|
+
} from "./query_metadata";
|
|
32
|
+
|
|
33
|
+
type WirePackageMaterialization =
|
|
34
|
+
components["schemas"]["PackageMaterializationConfig"];
|
|
35
|
+
type WirePersistSourcePlan = components["schemas"]["PersistSourcePlan"];
|
|
36
|
+
|
|
37
|
+
export interface MaterializationConfigInput {
|
|
38
|
+
/** The package manifest's `materialization` block, as parsed. */
|
|
39
|
+
packageMaterialization?: WirePackageMaterialization | null;
|
|
40
|
+
/** The compiled plan's sources, carrying each one's RESOLVED metadata. */
|
|
41
|
+
sources?: WirePersistSourcePlan[];
|
|
42
|
+
/**
|
|
43
|
+
* Deprecations the manifest parse tolerated (e.g. a root-level `scope`), which
|
|
44
|
+
* a load keeps working but a publish should report.
|
|
45
|
+
*/
|
|
46
|
+
manifestWarnings?: string[];
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** One finding, in the shape the wire package's operator warnings array uses. */
|
|
50
|
+
export interface MaterializationConfigWarning {
|
|
51
|
+
/** The persist source the finding belongs to, absent for a package-level one. */
|
|
52
|
+
target?: string;
|
|
53
|
+
message: string;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function metadataWarnings(
|
|
57
|
+
level: string,
|
|
58
|
+
metadata: QueryMetadata | null | undefined,
|
|
59
|
+
target?: string,
|
|
60
|
+
): MaterializationConfigWarning[] {
|
|
61
|
+
if (!metadata) return [];
|
|
62
|
+
const budget = queryMetadataBudgetWarning(Object.keys(metadata).length);
|
|
63
|
+
return [
|
|
64
|
+
...queryMetadataViolations(metadata),
|
|
65
|
+
...queryMetadataAdvisoryWarnings(metadata),
|
|
66
|
+
...(budget ? [budget] : []),
|
|
67
|
+
].map((message) => ({
|
|
68
|
+
// The level is in the message because a reader needs to know WHICH
|
|
69
|
+
// declaration to edit, and an inherited property has more than one.
|
|
70
|
+
message: `${level}: ${message}`,
|
|
71
|
+
...(target ? { target } : {}),
|
|
72
|
+
}));
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Everything wrong or inadvisable about a package's materialization config, as
|
|
77
|
+
* actionable messages. Warnings rather than errors: a package published before a
|
|
78
|
+
* rule existed must keep loading, and a bad metadata property degrades
|
|
79
|
+
* attribution rather than corrupting anything.
|
|
80
|
+
*
|
|
81
|
+
* A source's metadata is checked in its RESOLVED form (the effective bag from
|
|
82
|
+
* package → model-file → `#@ persist`), because that is what will actually be
|
|
83
|
+
* attached — a property inherited from the manifest is just as broken at the
|
|
84
|
+
* source as it is at the package, and reporting it at both is how an author finds
|
|
85
|
+
* the one they can edit.
|
|
86
|
+
*/
|
|
87
|
+
export function materializationConfigWarnings(
|
|
88
|
+
input: MaterializationConfigInput,
|
|
89
|
+
): MaterializationConfigWarning[] {
|
|
90
|
+
const warnings: MaterializationConfigWarning[] = (
|
|
91
|
+
input.manifestWarnings ?? []
|
|
92
|
+
).map((message) => ({ message }));
|
|
93
|
+
|
|
94
|
+
warnings.push(
|
|
95
|
+
...metadataWarnings(
|
|
96
|
+
"materialization.queryMetadata",
|
|
97
|
+
input.packageMaterialization?.queryMetadata,
|
|
98
|
+
),
|
|
99
|
+
);
|
|
100
|
+
|
|
101
|
+
for (const source of input.sources ?? []) {
|
|
102
|
+
warnings.push(
|
|
103
|
+
...metadataWarnings(
|
|
104
|
+
`#@ persist queryMetadata`,
|
|
105
|
+
source.queryMetadata,
|
|
106
|
+
source.name,
|
|
107
|
+
),
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// Identical findings collapse (two sources inheriting one bad package
|
|
112
|
+
// property produce one message each, not one per level per source).
|
|
113
|
+
const seen = new Set<string>();
|
|
114
|
+
return warnings.filter((warning) => {
|
|
115
|
+
const key = `${warning.target ?? ""}\u0000${warning.message}`;
|
|
116
|
+
if (seen.has(key)) return false;
|
|
117
|
+
seen.add(key);
|
|
118
|
+
return true;
|
|
119
|
+
});
|
|
120
|
+
}
|
|
@@ -54,6 +54,7 @@ describe("materialization schedule surfacing", () => {
|
|
|
54
54
|
expect(pkg.getPackageMetadata().materialization).toEqual({
|
|
55
55
|
schedule: "0 6 * * *",
|
|
56
56
|
freshness: null,
|
|
57
|
+
queryMetadata: null,
|
|
57
58
|
});
|
|
58
59
|
},
|
|
59
60
|
{ timeout: 20000 },
|
|
@@ -77,6 +78,7 @@ describe("materialization schedule surfacing", () => {
|
|
|
77
78
|
expect(pkg.getPackageMetadata().materialization).toEqual({
|
|
78
79
|
schedule: null,
|
|
79
80
|
freshness: { window: "24h", fallback: "stale_ok" },
|
|
81
|
+
queryMetadata: null,
|
|
80
82
|
});
|
|
81
83
|
},
|
|
82
84
|
{ timeout: 20000 },
|
|
@@ -119,14 +121,136 @@ describe("materialization schedule surfacing", () => {
|
|
|
119
121
|
expect(updated.materialization).toEqual({
|
|
120
122
|
schedule: "0 6 * * *",
|
|
121
123
|
freshness: null,
|
|
124
|
+
queryMetadata: null,
|
|
122
125
|
});
|
|
123
126
|
|
|
124
127
|
const pkg = await env.getPackage("pkg", false);
|
|
125
128
|
expect(pkg.getPackageMetadata().materialization).toEqual({
|
|
126
129
|
schedule: "0 6 * * *",
|
|
127
130
|
freshness: null,
|
|
131
|
+
queryMetadata: null,
|
|
128
132
|
});
|
|
129
133
|
},
|
|
130
134
|
{ timeout: 20000 },
|
|
131
135
|
);
|
|
136
|
+
|
|
137
|
+
async function readManifest(): Promise<Record<string, unknown>> {
|
|
138
|
+
return JSON.parse(
|
|
139
|
+
await fs.readFile(path.join(envPath, "pkg", "publisher.json"), "utf8"),
|
|
140
|
+
);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
it(
|
|
144
|
+
"writes scope to both homes so the manifest it authors still loads",
|
|
145
|
+
async () => {
|
|
146
|
+
// A scope PATCH used to write only the manifest root. With scope also
|
|
147
|
+
// living in the materialization block, that authored a manifest whose
|
|
148
|
+
// two homes disagreed — which the loader refuses, so the package
|
|
149
|
+
// survived the PATCH and then failed on the next restart.
|
|
150
|
+
const env = await Environment.create("testEnv", envPath, []);
|
|
151
|
+
await writePackageDir({
|
|
152
|
+
materialization: {
|
|
153
|
+
scope: "package",
|
|
154
|
+
queryMetadata: { team: "fin" },
|
|
155
|
+
},
|
|
156
|
+
});
|
|
157
|
+
await env.addPackage("pkg");
|
|
158
|
+
|
|
159
|
+
await env.updatePackage("pkg", { name: "pkg", scope: "version" });
|
|
160
|
+
|
|
161
|
+
const manifest = await readManifest();
|
|
162
|
+
expect(manifest.scope).toBe("version");
|
|
163
|
+
expect(manifest.materialization).toMatchObject({
|
|
164
|
+
scope: "version",
|
|
165
|
+
queryMetadata: { team: "fin" },
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
// The real assertion: it loads again.
|
|
169
|
+
const reloaded = await Environment.create("testEnv", envPath, []);
|
|
170
|
+
await reloaded.addPackage("pkg");
|
|
171
|
+
const pkg = await reloaded.getPackage("pkg", false);
|
|
172
|
+
expect(pkg.getPackageMetadata().scope).toBe("version");
|
|
173
|
+
},
|
|
174
|
+
{ timeout: 20000 },
|
|
175
|
+
);
|
|
176
|
+
|
|
177
|
+
it(
|
|
178
|
+
"keeps the envelope scope a materialization PATCH cannot express",
|
|
179
|
+
async () => {
|
|
180
|
+
// The wire materialization block has no `scope`, so a PATCH that sends
|
|
181
|
+
// one must not be read as "the author dropped it".
|
|
182
|
+
const env = await Environment.create("testEnv", envPath, []);
|
|
183
|
+
await writePackageDir({ materialization: { scope: "version" } });
|
|
184
|
+
await env.addPackage("pkg");
|
|
185
|
+
|
|
186
|
+
await env.updatePackage("pkg", {
|
|
187
|
+
name: "pkg",
|
|
188
|
+
materialization: { schedule: "0 6 * * *" },
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
const manifest = await readManifest();
|
|
192
|
+
expect(manifest.materialization).toMatchObject({
|
|
193
|
+
scope: "version",
|
|
194
|
+
schedule: "0 6 * * *",
|
|
195
|
+
});
|
|
196
|
+
expect(manifest.scope).toBe("version");
|
|
197
|
+
},
|
|
198
|
+
{ timeout: 20000 },
|
|
199
|
+
);
|
|
200
|
+
|
|
201
|
+
it(
|
|
202
|
+
"keeps queryMetadata a schedule PATCH said nothing about",
|
|
203
|
+
async () => {
|
|
204
|
+
// The block is replaced wholesale, which is right for the policy the
|
|
205
|
+
// caller is setting — but queryMetadata is orthogonal to it, so a
|
|
206
|
+
// client that sets a schedule without re-sending the tags would
|
|
207
|
+
// silently untag every statement the package's builds issue. The UI
|
|
208
|
+
// and the control plane are separate clients; preserving it here fixes
|
|
209
|
+
// all of them at once.
|
|
210
|
+
const env = await Environment.create("testEnv", envPath, []);
|
|
211
|
+
await writePackageDir({
|
|
212
|
+
materialization: {
|
|
213
|
+
scope: "version",
|
|
214
|
+
queryMetadata: { team: "finance" },
|
|
215
|
+
},
|
|
216
|
+
});
|
|
217
|
+
await env.addPackage("pkg");
|
|
218
|
+
|
|
219
|
+
await env.updatePackage("pkg", {
|
|
220
|
+
name: "pkg",
|
|
221
|
+
materialization: { schedule: "0 6 * * *" },
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
expect((await readManifest()).materialization).toMatchObject({
|
|
225
|
+
schedule: "0 6 * * *",
|
|
226
|
+
queryMetadata: { team: "finance" },
|
|
227
|
+
});
|
|
228
|
+
},
|
|
229
|
+
{ timeout: 20000 },
|
|
230
|
+
);
|
|
231
|
+
|
|
232
|
+
it(
|
|
233
|
+
"still lets an explicit null clear queryMetadata",
|
|
234
|
+
async () => {
|
|
235
|
+
// Preserved-on-omission must not make it unclearable.
|
|
236
|
+
const env = await Environment.create("testEnv", envPath, []);
|
|
237
|
+
await writePackageDir({
|
|
238
|
+
materialization: {
|
|
239
|
+
scope: "version",
|
|
240
|
+
queryMetadata: { team: "finance" },
|
|
241
|
+
},
|
|
242
|
+
});
|
|
243
|
+
await env.addPackage("pkg");
|
|
244
|
+
|
|
245
|
+
await env.updatePackage("pkg", {
|
|
246
|
+
name: "pkg",
|
|
247
|
+
materialization: { schedule: "0 6 * * *", queryMetadata: null },
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
expect(
|
|
251
|
+
(await readManifest()).materialization as Record<string, unknown>,
|
|
252
|
+
).toMatchObject({ schedule: "0 6 * * *", queryMetadata: null });
|
|
253
|
+
},
|
|
254
|
+
{ timeout: 20000 },
|
|
255
|
+
);
|
|
132
256
|
});
|