@malloy-publisher/server 0.0.233 → 0.0.235
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.docker.md +1 -0
- package/dist/app/api-doc.yaml +208 -0
- package/dist/app/assets/{EnvironmentPage-DutP7T8h.js → EnvironmentPage-BsAnavYN.js} +1 -1
- package/dist/app/assets/{HomePage-BcxDrBfl.js → HomePage-CADE138j.js} +1 -1
- package/dist/app/assets/{LightMode-BJukGxgz.js → LightMode-Cfh7KzN8.js} +1 -1
- package/dist/app/assets/{MainPage-DXbwlMeF.js → MainPage-CO3pRlnV.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-BBQksmTU.js → MaterializationsPage-p9YjkRXZ.js} +1 -1
- package/dist/app/assets/{ModelPage-C6tK51uU.js → ModelPage-C1OSTv-x.js} +1 -1
- package/dist/app/assets/{PackagePage-Bo3cwwZE.js → PackagePage-e4kN75YR.js} +1 -1
- package/dist/app/assets/{RouteError-BufkcAKE.js → RouteError-CzbfOkng.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-DICvvKpa.js → ThemeEditorPage-CciagFTq.js} +1 -1
- package/dist/app/assets/{WorkbookPage-Dkwt75Nj.js → WorkbookPage-DNWmkCXa.js} +1 -1
- package/dist/app/assets/{core-C0nunIQT.es-DlMLKZBK.js → core-Rj_4rRnA.es-BZyvITuO.js} +1 -1
- package/dist/app/assets/{index-CmEVVe-8.js → index-CH2AcDzc.js} +4 -4
- package/dist/app/assets/{index-qnhU9CGo.js → index-DQa463gC.js} +2 -2
- package/dist/app/assets/{index-Cs4WVm2z.js → index-DQpV7MyA.js} +1 -1
- package/dist/app/assets/{index-BabP-V-S.js → index-DzaYbhnD.js} +1 -1
- package/dist/app/assets/{index-BusxL5Pt.js → index-VBbcc8s6.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/package_load_worker.mjs +53 -3
- package/dist/server.mjs +835 -97
- package/package.json +12 -12
- package/src/config.ts +35 -1
- package/src/controller/connection.controller.spec.ts +46 -0
- package/src/controller/connection.controller.ts +105 -2
- package/src/controller/materialization.controller.spec.ts +25 -0
- package/src/controller/materialization.controller.ts +60 -0
- package/src/controller/model.controller.ts +24 -0
- package/src/controller/query.controller.ts +83 -15
- package/src/mcp/handler_utils.ts +10 -2
- package/src/mcp/query_envelope.ts +10 -0
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/tools/execute_query_tool.spec.ts +131 -0
- package/src/mcp/tools/execute_query_tool.ts +62 -25
- package/src/mcp_config.spec.ts +919 -0
- package/src/mcp_config.ts +425 -0
- package/src/oom_guards.integration.spec.ts +11 -3
- package/src/package_load/package_load_pool.ts +2 -0
- package/src/package_load/package_load_worker.ts +17 -5
- package/src/package_load/protocol.ts +6 -0
- package/src/query_metadata_metrics.ts +49 -0
- package/src/server.ts +99 -3
- package/src/service/build_plan.spec.ts +125 -0
- package/src/service/build_plan.ts +108 -7
- package/src/service/connection_config.ts +49 -0
- package/src/service/environment.ts +57 -3
- package/src/service/materialization_config_validation.spec.ts +99 -0
- package/src/service/materialization_config_validation.ts +120 -0
- package/src/service/materialization_schedule_surface.spec.ts +124 -0
- package/src/service/materialization_service.spec.ts +119 -0
- package/src/service/materialization_service.ts +186 -3
- package/src/service/materialization_test_fixtures.ts +86 -21
- package/src/service/model.spec.ts +45 -1
- package/src/service/model.ts +145 -19
- package/src/service/package.ts +24 -1
- package/src/service/package_manifest.spec.ts +137 -4
- package/src/service/package_manifest.ts +140 -5
- package/src/service/persist_annotation_validation.spec.ts +12 -0
- package/src/service/persist_annotation_validation.ts +9 -4
- package/src/service/query_metadata.spec.ts +408 -0
- package/src/service/query_metadata.ts +492 -0
- package/src/service/query_metadata_identity.spec.ts +149 -0
package/src/server.ts
CHANGED
|
@@ -41,6 +41,7 @@ import {
|
|
|
41
41
|
getMemoryGovernorConfig,
|
|
42
42
|
getPersistCollisionEnforce,
|
|
43
43
|
getPersistStorageMode,
|
|
44
|
+
getQueryMetadataMode,
|
|
44
45
|
} from "./config";
|
|
45
46
|
import { setFilterDeprecationHeaders } from "./filter_deprecation";
|
|
46
47
|
import { checkHeapConfiguration } from "./heap_check";
|
|
@@ -48,6 +49,16 @@ import { queryConcurrency } from "./query_concurrency";
|
|
|
48
49
|
import { MaterializationController } from "./controller/materialization.controller";
|
|
49
50
|
import { ThemeController } from "./controller/theme.controller";
|
|
50
51
|
import { initializeMcpServer } from "./mcp/server";
|
|
52
|
+
import {
|
|
53
|
+
addCommand,
|
|
54
|
+
ensureMcpConfig,
|
|
55
|
+
logMcpConfigOutcome,
|
|
56
|
+
MCP_CONFIG_FILENAME,
|
|
57
|
+
mcpConfigEnabled,
|
|
58
|
+
mcpEndpoint,
|
|
59
|
+
resolveBoundPort,
|
|
60
|
+
resolveClientHost,
|
|
61
|
+
} from "./mcp_config";
|
|
51
62
|
import { registerLegacyRoutes } from "./server-old";
|
|
52
63
|
import { EnvironmentStore } from "./service/environment_store";
|
|
53
64
|
import { MaterializationScheduler } from "./service/materialization_scheduler";
|
|
@@ -95,6 +106,8 @@ function parseArgs() {
|
|
|
95
106
|
i++;
|
|
96
107
|
} else if (arg === "--init") {
|
|
97
108
|
process.env.INITIALIZE_STORAGE = "true";
|
|
109
|
+
} else if (arg === "--no-mcp-config") {
|
|
110
|
+
process.env.PUBLISHER_NO_MCP_CONFIG = "true";
|
|
98
111
|
} else if (arg === "--watch-env" && args[i + 1]) {
|
|
99
112
|
// Append (don't overwrite) so multiple --watch-env flags compose
|
|
100
113
|
// and so an explicit env var pre-set still wins.
|
|
@@ -133,6 +146,9 @@ function parseArgs() {
|
|
|
133
146
|
console.log(
|
|
134
147
|
" --init Wipe persisted storage and re-sync it from the config (default: false)",
|
|
135
148
|
);
|
|
149
|
+
console.log(
|
|
150
|
+
" --no-mcp-config Do not write .mcp.json into the working directory (default: it is written, so an agent opened here finds this server; skipped when the directory already has one, is your home directory or the filesystem root, is inside a git working tree, or the MCP port bound is not the one requested)",
|
|
151
|
+
);
|
|
136
152
|
console.log(
|
|
137
153
|
" --watch-env <name> Enable dev-mode watch for the named environment.",
|
|
138
154
|
);
|
|
@@ -177,9 +193,20 @@ getPersistStorageMode();
|
|
|
177
193
|
// than a failed boot.
|
|
178
194
|
getPersistCollisionEnforce();
|
|
179
195
|
|
|
196
|
+
// Same hazard, wider blast radius: getQueryMetadataMode() throws on an invalid
|
|
197
|
+
// value and is read while resolving EVERY statement, so a typo'd off switch
|
|
198
|
+
// ("false", "0", "disabled") would boot clean and then fail every query and
|
|
199
|
+
// every build — the one thing the metadata path promises never to do.
|
|
200
|
+
getQueryMetadataMode();
|
|
201
|
+
|
|
180
202
|
const PUBLISHER_PORT = Number(process.env.PUBLISHER_PORT || 4000);
|
|
181
203
|
const PUBLISHER_HOST = process.env.PUBLISHER_HOST || "0.0.0.0";
|
|
182
204
|
const MCP_PORT = Number(process.env.MCP_PORT || 4040);
|
|
205
|
+
// Resolved here rather than in the listen callback: parseBoolEnv throws on a
|
|
206
|
+
// typo, which is the convention for flags in this server, but a throw inside a
|
|
207
|
+
// listen callback is an uncaughtException that kills a server which has already
|
|
208
|
+
// bound both ports. At module scope it is an ordinary startup failure.
|
|
209
|
+
const MCP_CONFIG_ENABLED = mcpConfigEnabled();
|
|
183
210
|
const MCP_ENDPOINT = "/mcp";
|
|
184
211
|
const SHUTDOWN_DRAIN_DURATION_SECONDS = Number(
|
|
185
212
|
process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0,
|
|
@@ -1273,6 +1300,11 @@ app.post(
|
|
|
1273
1300
|
req.params.connectionName,
|
|
1274
1301
|
req.body.sqlStatement as string,
|
|
1275
1302
|
req.body?.options as string,
|
|
1303
|
+
undefined,
|
|
1304
|
+
{
|
|
1305
|
+
queryMetadata: req.body?.queryMetadata,
|
|
1306
|
+
queryClass: req.body?.queryClass,
|
|
1307
|
+
},
|
|
1276
1308
|
),
|
|
1277
1309
|
);
|
|
1278
1310
|
} catch (error) {
|
|
@@ -1295,6 +1327,10 @@ app.post(
|
|
|
1295
1327
|
req.body.sqlStatement as string,
|
|
1296
1328
|
req.body?.options as string,
|
|
1297
1329
|
req.params.packageName,
|
|
1330
|
+
{
|
|
1331
|
+
queryMetadata: req.body?.queryMetadata,
|
|
1332
|
+
queryClass: req.body?.queryClass,
|
|
1333
|
+
},
|
|
1298
1334
|
),
|
|
1299
1335
|
);
|
|
1300
1336
|
} catch (error) {
|
|
@@ -1664,6 +1700,11 @@ app.post(
|
|
|
1664
1700
|
| undefined,
|
|
1665
1701
|
req.body.bypassFilters === true ? true : undefined,
|
|
1666
1702
|
req.body.givens as Record<string, GivenValue> | undefined,
|
|
1703
|
+
{
|
|
1704
|
+
queryMetadata: req.body?.queryMetadata,
|
|
1705
|
+
queryClass: req.body?.queryClass,
|
|
1706
|
+
versionId: req.body?.versionId as string | undefined,
|
|
1707
|
+
},
|
|
1667
1708
|
);
|
|
1668
1709
|
setFilterDeprecationHeaders(res, {
|
|
1669
1710
|
filterParams: req.body.filterParams ?? req.body.sourceFilters,
|
|
@@ -1947,9 +1988,64 @@ mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, async () => {
|
|
|
1947
1988
|
}
|
|
1948
1989
|
}
|
|
1949
1990
|
});
|
|
1950
|
-
const mcpServer = mcpApp.listen(
|
|
1951
|
-
|
|
1952
|
-
|
|
1991
|
+
const mcpServer = mcpApp.listen(
|
|
1992
|
+
MCP_PORT,
|
|
1993
|
+
PUBLISHER_HOST,
|
|
1994
|
+
function (this: import("net").Server) {
|
|
1995
|
+
// Read back rather than reusing MCP_PORT, which is only what was requested.
|
|
1996
|
+
// `--mcp_port 0` asks for any free port, and under bun a non-numeric value
|
|
1997
|
+
// binds an ephemeral one too, so the requested value can be 0 or NaN while
|
|
1998
|
+
// a real port is listening. The listening line uses it as well, which is
|
|
1999
|
+
// why it no longer reads `http://127.0.0.1:0`.
|
|
2000
|
+
const boundPort = resolveBoundPort(this.address(), MCP_PORT);
|
|
2001
|
+
// The BIND address, bracketed when it is an IPv6 literal so the URL
|
|
2002
|
+
// parses. Deliberately not resolveClientHost: create-malloy-package's
|
|
2003
|
+
// README and AGENTS template both tell readers these two listening lines
|
|
2004
|
+
// are "the addresses it really bound", and use them to catch a mistyped
|
|
2005
|
+
// --hostt that silently falls back to 0.0.0.0. Mapping the wildcard to
|
|
2006
|
+
// loopback here would confirm the mistake instead of revealing it. The
|
|
2007
|
+
// dialable form belongs in .mcp.json and in the advice, not here.
|
|
2008
|
+
const bound = this.address();
|
|
2009
|
+
const boundHost =
|
|
2010
|
+
typeof bound === "object" && bound ? bound.address : PUBLISHER_HOST;
|
|
2011
|
+
logger.info(
|
|
2012
|
+
`MCP server listening at http://${boundHost.includes(":") ? `[${boundHost}]` : boundHost}:${boundPort}`,
|
|
2013
|
+
);
|
|
2014
|
+
// Checked before process.cwd(), which can throw: someone who turned the
|
|
2015
|
+
// feature off should not get a warning about it.
|
|
2016
|
+
if (MCP_CONFIG_ENABLED) {
|
|
2017
|
+
// ensureMcpConfig cannot throw, but its arguments can: process.cwd()
|
|
2018
|
+
// raises ENOENT once the working directory has been removed. A throw
|
|
2019
|
+
// here is an uncaught exception inside a listen callback, which would
|
|
2020
|
+
// kill a server that has already bound both ports. Everything the call
|
|
2021
|
+
// needs is built inside the try for that reason, including the
|
|
2022
|
+
// endpoint: it is the newest and least-exercised code in this block.
|
|
2023
|
+
try {
|
|
2024
|
+
// The host an agent should dial, which is NOT `localhost`: that name
|
|
2025
|
+
// resolves to both loopback families while the server binds only one,
|
|
2026
|
+
// so another local process can hold the same port on the other family
|
|
2027
|
+
// and receive the agent's traffic instead.
|
|
2028
|
+
const endpoint = mcpEndpoint(
|
|
2029
|
+
resolveClientHost(this.address(), PUBLISHER_HOST),
|
|
2030
|
+
boundPort,
|
|
2031
|
+
);
|
|
2032
|
+
// cwd, not server_root: the file is for whoever opens an agent here.
|
|
2033
|
+
logMcpConfigOutcome(
|
|
2034
|
+
ensureMcpConfig({
|
|
2035
|
+
dir: process.cwd(),
|
|
2036
|
+
endpoint,
|
|
2037
|
+
requestedPort: MCP_PORT,
|
|
2038
|
+
boundPort,
|
|
2039
|
+
}),
|
|
2040
|
+
);
|
|
2041
|
+
} catch (error) {
|
|
2042
|
+
logger.info(
|
|
2043
|
+
`Could not set up ${MCP_CONFIG_FILENAME} (${error instanceof Error ? error.message : String(error)}). To connect an agent, run: ${addCommand(mcpEndpoint(resolveClientHost(this.address(), PUBLISHER_HOST), boundPort))}`,
|
|
2044
|
+
);
|
|
2045
|
+
}
|
|
2046
|
+
}
|
|
2047
|
+
},
|
|
2048
|
+
);
|
|
1953
2049
|
|
|
1954
2050
|
mcpServer.timeout = 600000;
|
|
1955
2051
|
mcpServer.keepAliveTimeout = 600000;
|
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
iterGraphSources,
|
|
13
13
|
projectToPublicColumns,
|
|
14
14
|
resolveFreshness,
|
|
15
|
+
resolveQueryMetadata,
|
|
15
16
|
resolvePackageConnections,
|
|
16
17
|
} from "./build_plan";
|
|
17
18
|
import { MaterializationEligibilityError } from "../errors";
|
|
@@ -419,6 +420,130 @@ describe("resolveFreshness", () => {
|
|
|
419
420
|
});
|
|
420
421
|
expect(resolveFreshness(source, null)).toEqual({ window: "1h" });
|
|
421
422
|
});
|
|
423
|
+
|
|
424
|
+
it("reads the model-file `materialization` envelope", () => {
|
|
425
|
+
const source = fakeSource({
|
|
426
|
+
name: "s",
|
|
427
|
+
sourceEntityId: "bid",
|
|
428
|
+
modelMaterialization: { freshness: { freshness: { window: "12h" } } },
|
|
429
|
+
});
|
|
430
|
+
expect(resolveFreshness(source, null)).toEqual({ window: "12h" });
|
|
431
|
+
});
|
|
432
|
+
|
|
433
|
+
it("prefers the envelope over the deprecated bare model-file form", () => {
|
|
434
|
+
const source = fakeSource({
|
|
435
|
+
name: "s",
|
|
436
|
+
sourceEntityId: "bid",
|
|
437
|
+
modelFreshnessSchedule: { freshness: { window: "48h" } },
|
|
438
|
+
modelMaterialization: { freshness: { freshness: { window: "12h" } } },
|
|
439
|
+
});
|
|
440
|
+
expect(resolveFreshness(source, null)).toEqual({ window: "12h" });
|
|
441
|
+
});
|
|
442
|
+
|
|
443
|
+
it("still reads a bare model-file knob the envelope does not declare", () => {
|
|
444
|
+
// A package published before the envelope existed keeps resolving, and the
|
|
445
|
+
// envelope does not hide the knobs it says nothing about.
|
|
446
|
+
const source = fakeSource({
|
|
447
|
+
name: "s",
|
|
448
|
+
sourceEntityId: "bid",
|
|
449
|
+
modelFreshnessSchedule: { freshness: { fallback: "fail" } },
|
|
450
|
+
modelMaterialization: { freshness: { freshness: { window: "12h" } } },
|
|
451
|
+
});
|
|
452
|
+
expect(resolveFreshness(source, null)).toEqual({
|
|
453
|
+
window: "12h",
|
|
454
|
+
fallback: "fail",
|
|
455
|
+
});
|
|
456
|
+
});
|
|
457
|
+
});
|
|
458
|
+
|
|
459
|
+
describe("resolveQueryMetadata", () => {
|
|
460
|
+
it("returns null when no layer declares anything", () => {
|
|
461
|
+
const source = fakeSource({ name: "s", sourceEntityId: "bid" });
|
|
462
|
+
expect(resolveQueryMetadata(source, null)).toBeNull();
|
|
463
|
+
expect(
|
|
464
|
+
resolveQueryMetadata(source, {
|
|
465
|
+
schedule: null,
|
|
466
|
+
freshness: null,
|
|
467
|
+
queryMetadata: null,
|
|
468
|
+
}),
|
|
469
|
+
).toBeNull();
|
|
470
|
+
});
|
|
471
|
+
|
|
472
|
+
it("reads the source's `#@ persist queryMetadata.*` properties", () => {
|
|
473
|
+
const source = fakeSource({
|
|
474
|
+
name: "s",
|
|
475
|
+
sourceEntityId: "bid",
|
|
476
|
+
queryMetadata: { team: "finance", workload: "orders" },
|
|
477
|
+
});
|
|
478
|
+
expect(resolveQueryMetadata(source, null)).toEqual({
|
|
479
|
+
team: "finance",
|
|
480
|
+
workload: "orders",
|
|
481
|
+
});
|
|
482
|
+
});
|
|
483
|
+
|
|
484
|
+
it("resolves most-specific-wins PER PROPERTY across all four layers", () => {
|
|
485
|
+
// Package declares team+tier, the model-file envelope overrides tier and
|
|
486
|
+
// adds one of its own, the source overrides only workload: every property
|
|
487
|
+
// nothing more specific overrides has to survive.
|
|
488
|
+
const source = fakeSource({
|
|
489
|
+
name: "s",
|
|
490
|
+
sourceEntityId: "bid",
|
|
491
|
+
queryMetadata: { workload: "orders" },
|
|
492
|
+
modelMaterialization: {
|
|
493
|
+
queryMetadata: { tier: "gold", surface: "marts" },
|
|
494
|
+
},
|
|
495
|
+
});
|
|
496
|
+
expect(
|
|
497
|
+
resolveQueryMetadata(source, {
|
|
498
|
+
schedule: null,
|
|
499
|
+
freshness: null,
|
|
500
|
+
queryMetadata: { team: "finance", tier: "bronze" },
|
|
501
|
+
}),
|
|
502
|
+
).toEqual({
|
|
503
|
+
team: "finance",
|
|
504
|
+
tier: "gold",
|
|
505
|
+
surface: "marts",
|
|
506
|
+
workload: "orders",
|
|
507
|
+
});
|
|
508
|
+
});
|
|
509
|
+
|
|
510
|
+
it("prefers the model-file envelope over the bare form, per property", () => {
|
|
511
|
+
const source = fakeSource({
|
|
512
|
+
name: "s",
|
|
513
|
+
sourceEntityId: "bid",
|
|
514
|
+
modelQueryMetadata: { tier: "bronze", legacy: "kept" },
|
|
515
|
+
modelMaterialization: { queryMetadata: { tier: "gold" } },
|
|
516
|
+
});
|
|
517
|
+
expect(resolveQueryMetadata(source, null)).toEqual({
|
|
518
|
+
tier: "gold",
|
|
519
|
+
legacy: "kept",
|
|
520
|
+
});
|
|
521
|
+
});
|
|
522
|
+
|
|
523
|
+
it("keeps a contract-violating property for the validator to report", () => {
|
|
524
|
+
const source = fakeSource({
|
|
525
|
+
name: "s",
|
|
526
|
+
sourceEntityId: "bid",
|
|
527
|
+
queryMetadata: { "team.name": "finance" },
|
|
528
|
+
});
|
|
529
|
+
expect(resolveQueryMetadata(source, null)).toEqual({
|
|
530
|
+
"team.name": "finance",
|
|
531
|
+
});
|
|
532
|
+
});
|
|
533
|
+
|
|
534
|
+
it("does not confuse the scalar `#@ persist` fields beside it", () => {
|
|
535
|
+
const source = fakeSource({
|
|
536
|
+
name: "s",
|
|
537
|
+
sourceEntityId: "bid",
|
|
538
|
+
annotationFields: { name: "s_table", refresh: "full" },
|
|
539
|
+
queryMetadata: { team: "finance" },
|
|
540
|
+
});
|
|
541
|
+
expect(resolveQueryMetadata(source, null)).toEqual({ team: "finance" });
|
|
542
|
+
expect(deriveAnnotationFields(source)).toEqual({
|
|
543
|
+
name: "s_table",
|
|
544
|
+
refresh: "full",
|
|
545
|
+
});
|
|
546
|
+
});
|
|
422
547
|
});
|
|
423
548
|
|
|
424
549
|
describe("deriveBuildPlan freshness", () => {
|
|
@@ -27,6 +27,7 @@ type BuildPlan = components["schemas"]["BuildPlan"];
|
|
|
27
27
|
type WireFreshness = components["schemas"]["Freshness"];
|
|
28
28
|
type WirePackageMaterialization =
|
|
29
29
|
components["schemas"]["PackageMaterializationConfig"];
|
|
30
|
+
type QueryMetadata = components["schemas"]["QueryMetadata"];
|
|
30
31
|
|
|
31
32
|
/** The freshness `fallback` values the publisher recognizes; others are dropped. */
|
|
32
33
|
const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
|
|
@@ -38,9 +39,15 @@ interface FreshnessLayer {
|
|
|
38
39
|
fallback?: FreshnessFallback;
|
|
39
40
|
}
|
|
40
41
|
|
|
41
|
-
/**
|
|
42
|
+
/**
|
|
43
|
+
* Minimal reader over a Malloy `Tag` (see `@malloydata/malloy-tag`): scalar
|
|
44
|
+
* reads by path, plus the subtree read that a property collection like
|
|
45
|
+
* `queryMetadata { … }` needs.
|
|
46
|
+
*/
|
|
42
47
|
interface ReadableTag {
|
|
43
48
|
text(...path: string[]): string | undefined;
|
|
49
|
+
tag(...path: string[]): ReadableTag | undefined;
|
|
50
|
+
entries?(): Iterable<[string, { text(): string | undefined }]>;
|
|
44
51
|
}
|
|
45
52
|
|
|
46
53
|
/**
|
|
@@ -238,6 +245,29 @@ function tagFreshnessLayer(tag: ReadableTag | undefined): FreshnessLayer {
|
|
|
238
245
|
return layer;
|
|
239
246
|
}
|
|
240
247
|
|
|
248
|
+
/**
|
|
249
|
+
* The two homes a model-file (`##`) knob can be declared in, most specific
|
|
250
|
+
* first: the `materialization` envelope, then the bare form.
|
|
251
|
+
*
|
|
252
|
+
* The model-file level mirrors the manifest's shape verbatim — `##
|
|
253
|
+
* materialization.freshness.window="24h"` is the manifest's block in tag syntax
|
|
254
|
+
* — so that is where a file-level reader looks. The bare form
|
|
255
|
+
* (`## freshness.window="24h"`), which shipped first, stays readable underneath:
|
|
256
|
+
* the annotation rides the published package, so a package published before the
|
|
257
|
+
* envelope existed keeps resolving until it is republished. Resolution is
|
|
258
|
+
* per-property, so a file may declare one knob in each home without the envelope
|
|
259
|
+
* hiding the other.
|
|
260
|
+
*/
|
|
261
|
+
function modelTagLayers(
|
|
262
|
+
tag: ReadableTag | undefined,
|
|
263
|
+
): (ReadableTag | undefined)[] {
|
|
264
|
+
const envelope =
|
|
265
|
+
tag && typeof tag.tag === "function"
|
|
266
|
+
? tag.tag("materialization")
|
|
267
|
+
: undefined;
|
|
268
|
+
return [envelope, tag];
|
|
269
|
+
}
|
|
270
|
+
|
|
241
271
|
/** The package-level `materialization.freshness` as a resolution layer. */
|
|
242
272
|
function packageFreshnessLayer(
|
|
243
273
|
cfg: WirePackageMaterialization | null | undefined,
|
|
@@ -289,13 +319,16 @@ export function resolveFreshness(
|
|
|
289
319
|
source: PersistSource,
|
|
290
320
|
packageMaterialization: WirePackageMaterialization | null | undefined,
|
|
291
321
|
): WireFreshness | null {
|
|
292
|
-
const
|
|
293
|
-
|
|
294
|
-
|
|
322
|
+
const layers: FreshnessLayer[] = [
|
|
323
|
+
// `#@ persist` declares knobs bare — the annotation IS a materialization
|
|
324
|
+
// declaration, so there is no envelope to look under.
|
|
325
|
+
tagFreshnessLayer(safeSourceTag(source)),
|
|
326
|
+
...modelTagLayers(safeModelTag(source)).map(tagFreshnessLayer),
|
|
327
|
+
packageFreshnessLayer(packageMaterialization),
|
|
328
|
+
];
|
|
295
329
|
|
|
296
|
-
const window =
|
|
297
|
-
const fallback =
|
|
298
|
-
sourceLayer.fallback ?? modelLayer.fallback ?? pkgLayer.fallback;
|
|
330
|
+
const window = layers.map((l) => l.window).find((v) => v !== undefined);
|
|
331
|
+
const fallback = layers.map((l) => l.fallback).find((v) => v !== undefined);
|
|
299
332
|
|
|
300
333
|
if (window === undefined && fallback === undefined) return null;
|
|
301
334
|
const freshness: WireFreshness = {};
|
|
@@ -304,6 +337,69 @@ export function resolveFreshness(
|
|
|
304
337
|
return freshness;
|
|
305
338
|
}
|
|
306
339
|
|
|
340
|
+
/**
|
|
341
|
+
* Read the `queryMetadata` property collection from one tag layer. A collection,
|
|
342
|
+
* not a scalar: `queryMetadata { team="finance" env="prod" }` and the equivalent
|
|
343
|
+
* dotted form `queryMetadata.team="finance"` both land here, and neither is
|
|
344
|
+
* captured by the scalar {@link deriveAnnotationFields} loop.
|
|
345
|
+
*
|
|
346
|
+
* Every string-valued property is kept verbatim, including ones that violate
|
|
347
|
+
* Malloy's bag contract — publish reports those as warnings and the runtime
|
|
348
|
+
* clamps them, so an author's typo is visible somewhere instead of vanishing
|
|
349
|
+
* between the annotation and the warehouse.
|
|
350
|
+
*/
|
|
351
|
+
function tagQueryMetadataLayer(tag: ReadableTag | undefined): QueryMetadata {
|
|
352
|
+
const subtree =
|
|
353
|
+
tag && typeof tag.tag === "function"
|
|
354
|
+
? tag.tag("queryMetadata")
|
|
355
|
+
: undefined;
|
|
356
|
+
if (!subtree || typeof subtree.entries !== "function") return {};
|
|
357
|
+
const layer: QueryMetadata = {};
|
|
358
|
+
try {
|
|
359
|
+
for (const [name, value] of subtree.entries()) {
|
|
360
|
+
const text = value.text();
|
|
361
|
+
if (text !== undefined) layer[name] = text;
|
|
362
|
+
}
|
|
363
|
+
} catch {
|
|
364
|
+
// Degrade to {} — mirrors deriveAnnotationFields / deriveColumns.
|
|
365
|
+
return {};
|
|
366
|
+
}
|
|
367
|
+
return layer;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/**
|
|
371
|
+
* Resolve a source's EFFECTIVE per-query metadata, most-specific-wins PER
|
|
372
|
+
* PROPERTY: `#@ persist queryMetadata.*` > model-file
|
|
373
|
+
* `## materialization.queryMetadata.*` (bare `## queryMetadata.*` underneath) >
|
|
374
|
+
* package `materialization.queryMetadata`.
|
|
375
|
+
*
|
|
376
|
+
* Per-property rather than per-layer, exactly like {@link resolveFreshness}, so a
|
|
377
|
+
* package-wide `team` property survives a source that only overrides `workload`.
|
|
378
|
+
* Null when no layer declares anything, so absence on the wire always means
|
|
379
|
+
* "declared nowhere" rather than "declared empty".
|
|
380
|
+
*
|
|
381
|
+
* This is the value the publisher attaches (merged under its own context) to
|
|
382
|
+
* every statement it issues while building the source. It is deliberately absent
|
|
383
|
+
* from the source's content address: changing a tag must never re-address a
|
|
384
|
+
* table.
|
|
385
|
+
*/
|
|
386
|
+
export function resolveQueryMetadata(
|
|
387
|
+
source: PersistSource,
|
|
388
|
+
packageMaterialization: WirePackageMaterialization | null | undefined,
|
|
389
|
+
): QueryMetadata | null {
|
|
390
|
+
// Least specific first, so a more specific layer overwrites property by
|
|
391
|
+
// property.
|
|
392
|
+
const layers: QueryMetadata[] = [
|
|
393
|
+
packageMaterialization?.queryMetadata ?? {},
|
|
394
|
+
...modelTagLayers(safeModelTag(source))
|
|
395
|
+
.map(tagQueryMetadataLayer)
|
|
396
|
+
.reverse(),
|
|
397
|
+
tagQueryMetadataLayer(safeSourceTag(source)),
|
|
398
|
+
];
|
|
399
|
+
const resolved: QueryMetadata = Object.assign({}, ...layers);
|
|
400
|
+
return Object.keys(resolved).length > 0 ? resolved : null;
|
|
401
|
+
}
|
|
402
|
+
|
|
307
403
|
/** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
|
|
308
404
|
export function flattenDependsOn(node: {
|
|
309
405
|
dependsOn: { sourceID: string }[];
|
|
@@ -612,6 +708,11 @@ export function deriveBuildPlan(
|
|
|
612
708
|
sql: source.getSQL(),
|
|
613
709
|
refresh: annotationFields.refresh ?? null,
|
|
614
710
|
freshness: resolveFreshness(source, packageMaterialization),
|
|
711
|
+
// EFFECTIVE per-source query metadata, resolved per property across the
|
|
712
|
+
// same layer stack as freshness. A property collection rather than a
|
|
713
|
+
// scalar, so it comes from resolveQueryMetadata rather than the
|
|
714
|
+
// annotationFields map.
|
|
715
|
+
queryMetadata: resolveQueryMetadata(source, packageMaterialization),
|
|
615
716
|
columns: deriveColumns(source),
|
|
616
717
|
annotationFields,
|
|
617
718
|
modelPath: sourceModelPaths?.[sourceID],
|
|
@@ -2,7 +2,13 @@ import { createPrivateKey } from "crypto";
|
|
|
2
2
|
import { existsSync } from "fs";
|
|
3
3
|
import path from "path";
|
|
4
4
|
import { components } from "../api";
|
|
5
|
+
import { logger } from "../logger";
|
|
5
6
|
import { parseHostKeys } from "./proxy";
|
|
7
|
+
import {
|
|
8
|
+
queryMetadataAdvisoryWarnings,
|
|
9
|
+
queryMetadataBudgetWarning,
|
|
10
|
+
queryMetadataViolations,
|
|
11
|
+
} from "./query_metadata";
|
|
6
12
|
|
|
7
13
|
type ApiConnection = components["schemas"]["Connection"];
|
|
8
14
|
type AttachedDatabase = components["schemas"]["AttachedDatabase"];
|
|
@@ -271,6 +277,48 @@ function buildDuckdbEntry(
|
|
|
271
277
|
};
|
|
272
278
|
}
|
|
273
279
|
|
|
280
|
+
/**
|
|
281
|
+
* Report a connection default that will not do what it says — a property name
|
|
282
|
+
* the contract rejects, one BigQuery would drop, a bag with no room for the
|
|
283
|
+
* server's own context.
|
|
284
|
+
*
|
|
285
|
+
* Warns rather than throws, unlike everything else in this file: query metadata
|
|
286
|
+
* is observability, and an environment that refuses to load because a tag has a
|
|
287
|
+
* hyphen in it would trade a missing label for an outage. The connection update
|
|
288
|
+
* API rejects the same bag outright (see validateAdminAuthoredConnection) —
|
|
289
|
+
* strict where a human is waiting, lenient where a config is being loaded.
|
|
290
|
+
*/
|
|
291
|
+
function warnOnConnectionQueryMetadata(connection: ApiConnection): void {
|
|
292
|
+
let declared = 0;
|
|
293
|
+
for (const field of ["queryMetadata", "queryMetadataEnforced"] as const) {
|
|
294
|
+
const metadata = connection[field];
|
|
295
|
+
if (!metadata) continue;
|
|
296
|
+
declared += Object.keys(metadata).length;
|
|
297
|
+
const problems = [
|
|
298
|
+
...queryMetadataViolations(metadata),
|
|
299
|
+
...queryMetadataAdvisoryWarnings(metadata),
|
|
300
|
+
];
|
|
301
|
+
for (const problem of problems) {
|
|
302
|
+
logger.warn("Connection query metadata will not apply as declared", {
|
|
303
|
+
connectionName: connection.name,
|
|
304
|
+
field,
|
|
305
|
+
problem,
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
// The budget is checked over BOTH maps, not each one: they merge into the
|
|
310
|
+
// same bag, so a connection declaring 6 defaults and 6 enforced is over it
|
|
311
|
+
// while neither map is. This is the boundary where the admin who created the
|
|
312
|
+
// squeeze is the one reading the warning.
|
|
313
|
+
const overBudget = queryMetadataBudgetWarning(declared);
|
|
314
|
+
if (overBudget) {
|
|
315
|
+
logger.warn("Connection query metadata will not apply as declared", {
|
|
316
|
+
connectionName: connection.name,
|
|
317
|
+
problem: overBudget,
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
274
322
|
function validateConnectionShape(connection: ApiConnection): void {
|
|
275
323
|
if (connection.proxy) {
|
|
276
324
|
// A connection proxy makes THIS server open an outbound SSH tunnel to a
|
|
@@ -606,6 +654,7 @@ export function assembleEnvironmentConnections(
|
|
|
606
654
|
processedConnections.add(connection.name);
|
|
607
655
|
validateDuckdbApiSurface(connection);
|
|
608
656
|
validateConnectionShape(connection);
|
|
657
|
+
warnOnConnectionQueryMetadata(connection);
|
|
609
658
|
|
|
610
659
|
const apiConnection = cloneApiConnection(connection);
|
|
611
660
|
apiConnection.attributes = getStaticConnectionAttributes(connection.type);
|
|
@@ -1639,6 +1639,60 @@ export class Environment {
|
|
|
1639
1639
|
logger.warn(`Could not read manifest for ${packageName}`);
|
|
1640
1640
|
}
|
|
1641
1641
|
|
|
1642
|
+
const onDiskMaterialization =
|
|
1643
|
+
existingManifest.materialization !== null &&
|
|
1644
|
+
typeof existingManifest.materialization === "object" &&
|
|
1645
|
+
!Array.isArray(existingManifest.materialization)
|
|
1646
|
+
? (existingManifest.materialization as Record<string, unknown>)
|
|
1647
|
+
: undefined;
|
|
1648
|
+
|
|
1649
|
+
// Scope has two homes: `materialization.scope` (canonical) and the
|
|
1650
|
+
// manifest root (deprecated). The server writes BOTH, in sync, for as
|
|
1651
|
+
// long as the root form is supported:
|
|
1652
|
+
//
|
|
1653
|
+
// - writing only the root would author a manifest this build's loader
|
|
1654
|
+
// refuses, since a root that disagrees with an existing envelope is a
|
|
1655
|
+
// conflict (see resolvePackageScope);
|
|
1656
|
+
// - writing only the envelope would silently downgrade a package read
|
|
1657
|
+
// by an older publisher, which knows only the root and would default
|
|
1658
|
+
// to `package` — cross-version table reuse for a package declared
|
|
1659
|
+
// `version`.
|
|
1660
|
+
//
|
|
1661
|
+
// A caller can only express scope through the top-level `scope` field
|
|
1662
|
+
// (the wire materialization block has no `scope`), so an envelope value
|
|
1663
|
+
// already on disk is preserved rather than dropped by a materialization
|
|
1664
|
+
// PATCH that says nothing about it.
|
|
1665
|
+
const resolvedScope =
|
|
1666
|
+
metadata.scope ??
|
|
1667
|
+
(onDiskMaterialization?.scope as ApiPackage["scope"] | undefined) ??
|
|
1668
|
+
(existingManifest.scope as ApiPackage["scope"] | undefined);
|
|
1669
|
+
|
|
1670
|
+
// A materialization PATCH replaces the block wholesale, which is right
|
|
1671
|
+
// for schedule and freshness — they are the policy the caller is
|
|
1672
|
+
// setting, and they are mutually exclusive with each other.
|
|
1673
|
+
// `queryMetadata` is orthogonal to both: a client setting a schedule
|
|
1674
|
+
// has no reason to re-send the package's tags, and dropping them
|
|
1675
|
+
// silently untags every statement the package's builds issue. So it is
|
|
1676
|
+
// preserved on omission, like `scope` above; an explicit null still
|
|
1677
|
+
// clears it, which keeps the block expressible.
|
|
1678
|
+
const preservedQueryMetadata =
|
|
1679
|
+
metadata.materialization !== undefined &&
|
|
1680
|
+
metadata.materialization?.queryMetadata === undefined &&
|
|
1681
|
+
onDiskMaterialization?.queryMetadata !== undefined
|
|
1682
|
+
? { queryMetadata: onDiskMaterialization.queryMetadata }
|
|
1683
|
+
: {};
|
|
1684
|
+
|
|
1685
|
+
const materializationBase: Record<string, unknown> | undefined =
|
|
1686
|
+
metadata.materialization !== undefined
|
|
1687
|
+
? { ...metadata.materialization, ...preservedQueryMetadata }
|
|
1688
|
+
: onDiskMaterialization !== undefined
|
|
1689
|
+
? { ...onDiskMaterialization }
|
|
1690
|
+
: undefined;
|
|
1691
|
+
const materializationBlock =
|
|
1692
|
+
resolvedScope !== undefined
|
|
1693
|
+
? { ...(materializationBase ?? {}), scope: resolvedScope }
|
|
1694
|
+
: materializationBase;
|
|
1695
|
+
|
|
1642
1696
|
// Update with new metadata. `explores`/`queryableSources` are only
|
|
1643
1697
|
// overwritten when the caller explicitly provides them; otherwise the
|
|
1644
1698
|
// existing on-disk value is preserved via the spread (an undefined here
|
|
@@ -1656,9 +1710,9 @@ export class Environment {
|
|
|
1656
1710
|
...(metadata.manifestLocation !== undefined
|
|
1657
1711
|
? { manifestLocation: metadata.manifestLocation }
|
|
1658
1712
|
: {}),
|
|
1659
|
-
...(
|
|
1660
|
-
...(
|
|
1661
|
-
? { materialization:
|
|
1713
|
+
...(resolvedScope !== undefined ? { scope: resolvedScope } : {}),
|
|
1714
|
+
...(materializationBlock !== undefined
|
|
1715
|
+
? { materialization: materializationBlock }
|
|
1662
1716
|
: {}),
|
|
1663
1717
|
};
|
|
1664
1718
|
|