@malloy-publisher/server 0.0.232 → 0.0.234
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.docker.md +1 -0
- package/dist/app/api-doc.yaml +269 -10
- package/dist/app/assets/{EnvironmentPage-DXEaZIPx.js → EnvironmentPage-DTZQ4Gxc.js} +1 -1
- package/dist/app/assets/{HomePage-kofsqpZt.js → HomePage-C5mlDPXK.js} +1 -1
- package/dist/app/assets/{LightMode-CNhIlIlJ.js → LightMode-DGNmhG0u.js} +1 -1
- package/dist/app/assets/{MainPage-Bgqo8jCy.js → MainPage-CVL_wmP4.js} +1 -1
- package/dist/app/assets/{MaterializationsPage-CgBlgGz2.js → MaterializationsPage-DmzMBCpy.js} +1 -1
- package/dist/app/assets/{ModelPage-B0TjoDtf.js → ModelPage-Dbvf4QbB.js} +1 -1
- package/dist/app/assets/{PackagePage-BL8vnFj1.js → PackagePage-DxdHc2Qs.js} +1 -1
- package/dist/app/assets/{RouteError-BzPby0X2.js → RouteError-OJdT4tCd.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-CTEP_9r3.js → ThemeEditorPage-Bk7s0KXY.js} +1 -1
- package/dist/app/assets/{WorkbookPage-BwM3BmKw.js → WorkbookPage-j_vCWdN3.js} +1 -1
- package/dist/app/assets/{core-CK68iv6w.es-CpRxXBt7.js → core-Rj_4rRnA.es-DoIfLxDJ.js} +1 -1
- package/dist/app/assets/{index-B33zGctF.js → index-B_jKMR35.js} +4 -4
- package/dist/app/assets/{index-CmkW1MiE.js → index-D-rDyK11.js} +1 -1
- package/dist/app/assets/{index-tXJXwdyj.js → index-DWIe_hK0.js} +1 -1
- package/dist/app/assets/{index-BkiWKaAF.js → index-hw-xn0X7.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/package_load_worker.mjs +53 -3
- package/dist/server.mjs +20277 -925
- package/package.json +1 -1
- package/src/config.ts +35 -1
- package/src/controller/connection.controller.spec.ts +46 -0
- package/src/controller/connection.controller.ts +105 -2
- package/src/controller/materialization.controller.spec.ts +25 -0
- package/src/controller/materialization.controller.ts +60 -0
- package/src/controller/model.controller.ts +24 -0
- package/src/controller/query.controller.ts +83 -10
- package/src/json_utils.spec.ts +51 -0
- package/src/json_utils.ts +33 -0
- package/src/mcp/handler_utils.ts +10 -2
- package/src/mcp/query_envelope.spec.ts +229 -0
- package/src/mcp/query_envelope.ts +240 -0
- package/src/mcp/server.protocol.spec.ts +128 -16
- package/src/mcp/skills/build_skills_bundle.ts +94 -4
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/skills/skills_bundle.spec.ts +113 -4
- package/src/mcp/tool_response.spec.ts +108 -0
- package/src/mcp/tool_response.ts +138 -0
- package/src/mcp/tools/compile_tool.spec.ts +112 -4
- package/src/mcp/tools/compile_tool.ts +61 -30
- package/src/mcp/tools/docs_search_tool.ts +6 -16
- package/src/mcp/tools/execute_query_tool.spec.ts +154 -3
- package/src/mcp/tools/execute_query_tool.ts +131 -155
- package/src/mcp/tools/get_context_tool.spec.ts +63 -3
- package/src/mcp/tools/get_context_tool.ts +43 -46
- package/src/mcp/tools/reload_package_tool.ts +3 -29
- package/src/mcp_config.spec.ts +919 -0
- package/src/mcp_config.ts +425 -0
- package/src/oom_guards.integration.spec.ts +11 -3
- package/src/package_load/package_load_pool.ts +2 -0
- package/src/package_load/package_load_worker.ts +17 -5
- package/src/package_load/protocol.ts +6 -0
- package/src/query_metadata_metrics.ts +49 -0
- package/src/server.ts +99 -3
- package/src/service/build_plan.spec.ts +125 -0
- package/src/service/build_plan.ts +108 -7
- package/src/service/compile_fragment_techniques.spec.ts +156 -0
- package/src/service/connection.spec.ts +371 -1
- package/src/service/connection.ts +77 -14
- package/src/service/connection_config.spec.ts +60 -0
- package/src/service/connection_config.ts +75 -0
- package/src/service/duckdb_instance_isolation.spec.ts +137 -0
- package/src/service/environment.ts +57 -3
- package/src/service/materialization_config_validation.spec.ts +99 -0
- package/src/service/materialization_config_validation.ts +120 -0
- package/src/service/materialization_schedule_surface.spec.ts +124 -0
- package/src/service/materialization_service.spec.ts +119 -0
- package/src/service/materialization_service.ts +186 -3
- package/src/service/materialization_test_fixtures.ts +86 -21
- package/src/service/model.spec.ts +45 -1
- package/src/service/model.ts +171 -23
- package/src/service/model_limits.spec.ts +28 -0
- package/src/service/model_limits.ts +21 -0
- package/src/service/package.ts +24 -1
- package/src/service/package_manifest.spec.ts +137 -4
- package/src/service/package_manifest.ts +140 -5
- package/src/service/persist_annotation_validation.spec.ts +12 -0
- package/src/service/persist_annotation_validation.ts +9 -4
- package/src/service/query_metadata.spec.ts +408 -0
- package/src/service/query_metadata.ts +492 -0
- package/src/service/query_metadata_identity.spec.ts +149 -0
- package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +37 -12
package/src/server.ts
CHANGED
|
@@ -41,6 +41,7 @@ import {
|
|
|
41
41
|
getMemoryGovernorConfig,
|
|
42
42
|
getPersistCollisionEnforce,
|
|
43
43
|
getPersistStorageMode,
|
|
44
|
+
getQueryMetadataMode,
|
|
44
45
|
} from "./config";
|
|
45
46
|
import { setFilterDeprecationHeaders } from "./filter_deprecation";
|
|
46
47
|
import { checkHeapConfiguration } from "./heap_check";
|
|
@@ -48,6 +49,16 @@ import { queryConcurrency } from "./query_concurrency";
|
|
|
48
49
|
import { MaterializationController } from "./controller/materialization.controller";
|
|
49
50
|
import { ThemeController } from "./controller/theme.controller";
|
|
50
51
|
import { initializeMcpServer } from "./mcp/server";
|
|
52
|
+
import {
|
|
53
|
+
addCommand,
|
|
54
|
+
ensureMcpConfig,
|
|
55
|
+
logMcpConfigOutcome,
|
|
56
|
+
MCP_CONFIG_FILENAME,
|
|
57
|
+
mcpConfigEnabled,
|
|
58
|
+
mcpEndpoint,
|
|
59
|
+
resolveBoundPort,
|
|
60
|
+
resolveClientHost,
|
|
61
|
+
} from "./mcp_config";
|
|
51
62
|
import { registerLegacyRoutes } from "./server-old";
|
|
52
63
|
import { EnvironmentStore } from "./service/environment_store";
|
|
53
64
|
import { MaterializationScheduler } from "./service/materialization_scheduler";
|
|
@@ -95,6 +106,8 @@ function parseArgs() {
|
|
|
95
106
|
i++;
|
|
96
107
|
} else if (arg === "--init") {
|
|
97
108
|
process.env.INITIALIZE_STORAGE = "true";
|
|
109
|
+
} else if (arg === "--no-mcp-config") {
|
|
110
|
+
process.env.PUBLISHER_NO_MCP_CONFIG = "true";
|
|
98
111
|
} else if (arg === "--watch-env" && args[i + 1]) {
|
|
99
112
|
// Append (don't overwrite) so multiple --watch-env flags compose
|
|
100
113
|
// and so an explicit env var pre-set still wins.
|
|
@@ -133,6 +146,9 @@ function parseArgs() {
|
|
|
133
146
|
console.log(
|
|
134
147
|
" --init Wipe persisted storage and re-sync it from the config (default: false)",
|
|
135
148
|
);
|
|
149
|
+
console.log(
|
|
150
|
+
" --no-mcp-config Do not write .mcp.json into the working directory (default: it is written, so an agent opened here finds this server; skipped when the directory already has one, is your home directory or the filesystem root, is inside a git working tree, or the MCP port bound is not the one requested)",
|
|
151
|
+
);
|
|
136
152
|
console.log(
|
|
137
153
|
" --watch-env <name> Enable dev-mode watch for the named environment.",
|
|
138
154
|
);
|
|
@@ -177,9 +193,20 @@ getPersistStorageMode();
|
|
|
177
193
|
// than a failed boot.
|
|
178
194
|
getPersistCollisionEnforce();
|
|
179
195
|
|
|
196
|
+
// Same hazard, wider blast radius: getQueryMetadataMode() throws on an invalid
|
|
197
|
+
// value and is read while resolving EVERY statement, so a typo'd off switch
|
|
198
|
+
// ("false", "0", "disabled") would boot clean and then fail every query and
|
|
199
|
+
// every build — the one thing the metadata path promises never to do.
|
|
200
|
+
getQueryMetadataMode();
|
|
201
|
+
|
|
180
202
|
const PUBLISHER_PORT = Number(process.env.PUBLISHER_PORT || 4000);
|
|
181
203
|
const PUBLISHER_HOST = process.env.PUBLISHER_HOST || "0.0.0.0";
|
|
182
204
|
const MCP_PORT = Number(process.env.MCP_PORT || 4040);
|
|
205
|
+
// Resolved here rather than in the listen callback: parseBoolEnv throws on a
|
|
206
|
+
// typo, which is the convention for flags in this server, but a throw inside a
|
|
207
|
+
// listen callback is an uncaughtException that kills a server which has already
|
|
208
|
+
// bound both ports. At module scope it is an ordinary startup failure.
|
|
209
|
+
const MCP_CONFIG_ENABLED = mcpConfigEnabled();
|
|
183
210
|
const MCP_ENDPOINT = "/mcp";
|
|
184
211
|
const SHUTDOWN_DRAIN_DURATION_SECONDS = Number(
|
|
185
212
|
process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0,
|
|
@@ -1273,6 +1300,11 @@ app.post(
|
|
|
1273
1300
|
req.params.connectionName,
|
|
1274
1301
|
req.body.sqlStatement as string,
|
|
1275
1302
|
req.body?.options as string,
|
|
1303
|
+
undefined,
|
|
1304
|
+
{
|
|
1305
|
+
queryMetadata: req.body?.queryMetadata,
|
|
1306
|
+
queryClass: req.body?.queryClass,
|
|
1307
|
+
},
|
|
1276
1308
|
),
|
|
1277
1309
|
);
|
|
1278
1310
|
} catch (error) {
|
|
@@ -1295,6 +1327,10 @@ app.post(
|
|
|
1295
1327
|
req.body.sqlStatement as string,
|
|
1296
1328
|
req.body?.options as string,
|
|
1297
1329
|
req.params.packageName,
|
|
1330
|
+
{
|
|
1331
|
+
queryMetadata: req.body?.queryMetadata,
|
|
1332
|
+
queryClass: req.body?.queryClass,
|
|
1333
|
+
},
|
|
1298
1334
|
),
|
|
1299
1335
|
);
|
|
1300
1336
|
} catch (error) {
|
|
@@ -1664,6 +1700,11 @@ app.post(
|
|
|
1664
1700
|
| undefined,
|
|
1665
1701
|
req.body.bypassFilters === true ? true : undefined,
|
|
1666
1702
|
req.body.givens as Record<string, GivenValue> | undefined,
|
|
1703
|
+
{
|
|
1704
|
+
queryMetadata: req.body?.queryMetadata,
|
|
1705
|
+
queryClass: req.body?.queryClass,
|
|
1706
|
+
versionId: req.body?.versionId as string | undefined,
|
|
1707
|
+
},
|
|
1667
1708
|
);
|
|
1668
1709
|
setFilterDeprecationHeaders(res, {
|
|
1669
1710
|
filterParams: req.body.filterParams ?? req.body.sourceFilters,
|
|
@@ -1947,9 +1988,64 @@ mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, async () => {
|
|
|
1947
1988
|
}
|
|
1948
1989
|
}
|
|
1949
1990
|
});
|
|
1950
|
-
const mcpServer = mcpApp.listen(
|
|
1951
|
-
|
|
1952
|
-
|
|
1991
|
+
const mcpServer = mcpApp.listen(
|
|
1992
|
+
MCP_PORT,
|
|
1993
|
+
PUBLISHER_HOST,
|
|
1994
|
+
function (this: import("net").Server) {
|
|
1995
|
+
// Read back rather than reusing MCP_PORT, which is only what was requested.
|
|
1996
|
+
// `--mcp_port 0` asks for any free port, and under bun a non-numeric value
|
|
1997
|
+
// binds an ephemeral one too, so the requested value can be 0 or NaN while
|
|
1998
|
+
// a real port is listening. The listening line uses it as well, which is
|
|
1999
|
+
// why it no longer reads `http://127.0.0.1:0`.
|
|
2000
|
+
const boundPort = resolveBoundPort(this.address(), MCP_PORT);
|
|
2001
|
+
// The BIND address, bracketed when it is an IPv6 literal so the URL
|
|
2002
|
+
// parses. Deliberately not resolveClientHost: create-malloy-package's
|
|
2003
|
+
// README and AGENTS template both tell readers these two listening lines
|
|
2004
|
+
// are "the addresses it really bound", and use them to catch a mistyped
|
|
2005
|
+
// --hostt that silently falls back to 0.0.0.0. Mapping the wildcard to
|
|
2006
|
+
// loopback here would confirm the mistake instead of revealing it. The
|
|
2007
|
+
// dialable form belongs in .mcp.json and in the advice, not here.
|
|
2008
|
+
const bound = this.address();
|
|
2009
|
+
const boundHost =
|
|
2010
|
+
typeof bound === "object" && bound ? bound.address : PUBLISHER_HOST;
|
|
2011
|
+
logger.info(
|
|
2012
|
+
`MCP server listening at http://${boundHost.includes(":") ? `[${boundHost}]` : boundHost}:${boundPort}`,
|
|
2013
|
+
);
|
|
2014
|
+
// Checked before process.cwd(), which can throw: someone who turned the
|
|
2015
|
+
// feature off should not get a warning about it.
|
|
2016
|
+
if (MCP_CONFIG_ENABLED) {
|
|
2017
|
+
// ensureMcpConfig cannot throw, but its arguments can: process.cwd()
|
|
2018
|
+
// raises ENOENT once the working directory has been removed. A throw
|
|
2019
|
+
// here is an uncaught exception inside a listen callback, which would
|
|
2020
|
+
// kill a server that has already bound both ports. Everything the call
|
|
2021
|
+
// needs is built inside the try for that reason, including the
|
|
2022
|
+
// endpoint: it is the newest and least-exercised code in this block.
|
|
2023
|
+
try {
|
|
2024
|
+
// The host an agent should dial, which is NOT `localhost`: that name
|
|
2025
|
+
// resolves to both loopback families while the server binds only one,
|
|
2026
|
+
// so another local process can hold the same port on the other family
|
|
2027
|
+
// and receive the agent's traffic instead.
|
|
2028
|
+
const endpoint = mcpEndpoint(
|
|
2029
|
+
resolveClientHost(this.address(), PUBLISHER_HOST),
|
|
2030
|
+
boundPort,
|
|
2031
|
+
);
|
|
2032
|
+
// cwd, not server_root: the file is for whoever opens an agent here.
|
|
2033
|
+
logMcpConfigOutcome(
|
|
2034
|
+
ensureMcpConfig({
|
|
2035
|
+
dir: process.cwd(),
|
|
2036
|
+
endpoint,
|
|
2037
|
+
requestedPort: MCP_PORT,
|
|
2038
|
+
boundPort,
|
|
2039
|
+
}),
|
|
2040
|
+
);
|
|
2041
|
+
} catch (error) {
|
|
2042
|
+
logger.info(
|
|
2043
|
+
`Could not set up ${MCP_CONFIG_FILENAME} (${error instanceof Error ? error.message : String(error)}). To connect an agent, run: ${addCommand(mcpEndpoint(resolveClientHost(this.address(), PUBLISHER_HOST), boundPort))}`,
|
|
2044
|
+
);
|
|
2045
|
+
}
|
|
2046
|
+
}
|
|
2047
|
+
},
|
|
2048
|
+
);
|
|
1953
2049
|
|
|
1954
2050
|
mcpServer.timeout = 600000;
|
|
1955
2051
|
mcpServer.keepAliveTimeout = 600000;
|
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
iterGraphSources,
|
|
13
13
|
projectToPublicColumns,
|
|
14
14
|
resolveFreshness,
|
|
15
|
+
resolveQueryMetadata,
|
|
15
16
|
resolvePackageConnections,
|
|
16
17
|
} from "./build_plan";
|
|
17
18
|
import { MaterializationEligibilityError } from "../errors";
|
|
@@ -419,6 +420,130 @@ describe("resolveFreshness", () => {
|
|
|
419
420
|
});
|
|
420
421
|
expect(resolveFreshness(source, null)).toEqual({ window: "1h" });
|
|
421
422
|
});
|
|
423
|
+
|
|
424
|
+
it("reads the model-file `materialization` envelope", () => {
|
|
425
|
+
const source = fakeSource({
|
|
426
|
+
name: "s",
|
|
427
|
+
sourceEntityId: "bid",
|
|
428
|
+
modelMaterialization: { freshness: { freshness: { window: "12h" } } },
|
|
429
|
+
});
|
|
430
|
+
expect(resolveFreshness(source, null)).toEqual({ window: "12h" });
|
|
431
|
+
});
|
|
432
|
+
|
|
433
|
+
it("prefers the envelope over the deprecated bare model-file form", () => {
|
|
434
|
+
const source = fakeSource({
|
|
435
|
+
name: "s",
|
|
436
|
+
sourceEntityId: "bid",
|
|
437
|
+
modelFreshnessSchedule: { freshness: { window: "48h" } },
|
|
438
|
+
modelMaterialization: { freshness: { freshness: { window: "12h" } } },
|
|
439
|
+
});
|
|
440
|
+
expect(resolveFreshness(source, null)).toEqual({ window: "12h" });
|
|
441
|
+
});
|
|
442
|
+
|
|
443
|
+
it("still reads a bare model-file knob the envelope does not declare", () => {
|
|
444
|
+
// A package published before the envelope existed keeps resolving, and the
|
|
445
|
+
// envelope does not hide the knobs it says nothing about.
|
|
446
|
+
const source = fakeSource({
|
|
447
|
+
name: "s",
|
|
448
|
+
sourceEntityId: "bid",
|
|
449
|
+
modelFreshnessSchedule: { freshness: { fallback: "fail" } },
|
|
450
|
+
modelMaterialization: { freshness: { freshness: { window: "12h" } } },
|
|
451
|
+
});
|
|
452
|
+
expect(resolveFreshness(source, null)).toEqual({
|
|
453
|
+
window: "12h",
|
|
454
|
+
fallback: "fail",
|
|
455
|
+
});
|
|
456
|
+
});
|
|
457
|
+
});
|
|
458
|
+
|
|
459
|
+
describe("resolveQueryMetadata", () => {
|
|
460
|
+
it("returns null when no layer declares anything", () => {
|
|
461
|
+
const source = fakeSource({ name: "s", sourceEntityId: "bid" });
|
|
462
|
+
expect(resolveQueryMetadata(source, null)).toBeNull();
|
|
463
|
+
expect(
|
|
464
|
+
resolveQueryMetadata(source, {
|
|
465
|
+
schedule: null,
|
|
466
|
+
freshness: null,
|
|
467
|
+
queryMetadata: null,
|
|
468
|
+
}),
|
|
469
|
+
).toBeNull();
|
|
470
|
+
});
|
|
471
|
+
|
|
472
|
+
it("reads the source's `#@ persist queryMetadata.*` properties", () => {
|
|
473
|
+
const source = fakeSource({
|
|
474
|
+
name: "s",
|
|
475
|
+
sourceEntityId: "bid",
|
|
476
|
+
queryMetadata: { team: "finance", workload: "orders" },
|
|
477
|
+
});
|
|
478
|
+
expect(resolveQueryMetadata(source, null)).toEqual({
|
|
479
|
+
team: "finance",
|
|
480
|
+
workload: "orders",
|
|
481
|
+
});
|
|
482
|
+
});
|
|
483
|
+
|
|
484
|
+
it("resolves most-specific-wins PER PROPERTY across all four layers", () => {
|
|
485
|
+
// Package declares team+tier, the model-file envelope overrides tier and
|
|
486
|
+
// adds one of its own, the source overrides only workload: every property
|
|
487
|
+
// nothing more specific overrides has to survive.
|
|
488
|
+
const source = fakeSource({
|
|
489
|
+
name: "s",
|
|
490
|
+
sourceEntityId: "bid",
|
|
491
|
+
queryMetadata: { workload: "orders" },
|
|
492
|
+
modelMaterialization: {
|
|
493
|
+
queryMetadata: { tier: "gold", surface: "marts" },
|
|
494
|
+
},
|
|
495
|
+
});
|
|
496
|
+
expect(
|
|
497
|
+
resolveQueryMetadata(source, {
|
|
498
|
+
schedule: null,
|
|
499
|
+
freshness: null,
|
|
500
|
+
queryMetadata: { team: "finance", tier: "bronze" },
|
|
501
|
+
}),
|
|
502
|
+
).toEqual({
|
|
503
|
+
team: "finance",
|
|
504
|
+
tier: "gold",
|
|
505
|
+
surface: "marts",
|
|
506
|
+
workload: "orders",
|
|
507
|
+
});
|
|
508
|
+
});
|
|
509
|
+
|
|
510
|
+
it("prefers the model-file envelope over the bare form, per property", () => {
|
|
511
|
+
const source = fakeSource({
|
|
512
|
+
name: "s",
|
|
513
|
+
sourceEntityId: "bid",
|
|
514
|
+
modelQueryMetadata: { tier: "bronze", legacy: "kept" },
|
|
515
|
+
modelMaterialization: { queryMetadata: { tier: "gold" } },
|
|
516
|
+
});
|
|
517
|
+
expect(resolveQueryMetadata(source, null)).toEqual({
|
|
518
|
+
tier: "gold",
|
|
519
|
+
legacy: "kept",
|
|
520
|
+
});
|
|
521
|
+
});
|
|
522
|
+
|
|
523
|
+
it("keeps a contract-violating property for the validator to report", () => {
|
|
524
|
+
const source = fakeSource({
|
|
525
|
+
name: "s",
|
|
526
|
+
sourceEntityId: "bid",
|
|
527
|
+
queryMetadata: { "team.name": "finance" },
|
|
528
|
+
});
|
|
529
|
+
expect(resolveQueryMetadata(source, null)).toEqual({
|
|
530
|
+
"team.name": "finance",
|
|
531
|
+
});
|
|
532
|
+
});
|
|
533
|
+
|
|
534
|
+
it("does not confuse the scalar `#@ persist` fields beside it", () => {
|
|
535
|
+
const source = fakeSource({
|
|
536
|
+
name: "s",
|
|
537
|
+
sourceEntityId: "bid",
|
|
538
|
+
annotationFields: { name: "s_table", refresh: "full" },
|
|
539
|
+
queryMetadata: { team: "finance" },
|
|
540
|
+
});
|
|
541
|
+
expect(resolveQueryMetadata(source, null)).toEqual({ team: "finance" });
|
|
542
|
+
expect(deriveAnnotationFields(source)).toEqual({
|
|
543
|
+
name: "s_table",
|
|
544
|
+
refresh: "full",
|
|
545
|
+
});
|
|
546
|
+
});
|
|
422
547
|
});
|
|
423
548
|
|
|
424
549
|
describe("deriveBuildPlan freshness", () => {
|
|
@@ -27,6 +27,7 @@ type BuildPlan = components["schemas"]["BuildPlan"];
|
|
|
27
27
|
type WireFreshness = components["schemas"]["Freshness"];
|
|
28
28
|
type WirePackageMaterialization =
|
|
29
29
|
components["schemas"]["PackageMaterializationConfig"];
|
|
30
|
+
type QueryMetadata = components["schemas"]["QueryMetadata"];
|
|
30
31
|
|
|
31
32
|
/** The freshness `fallback` values the publisher recognizes; others are dropped. */
|
|
32
33
|
const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
|
|
@@ -38,9 +39,15 @@ interface FreshnessLayer {
|
|
|
38
39
|
fallback?: FreshnessFallback;
|
|
39
40
|
}
|
|
40
41
|
|
|
41
|
-
/**
|
|
42
|
+
/**
|
|
43
|
+
* Minimal reader over a Malloy `Tag` (see `@malloydata/malloy-tag`): scalar
|
|
44
|
+
* reads by path, plus the subtree read that a property collection like
|
|
45
|
+
* `queryMetadata { … }` needs.
|
|
46
|
+
*/
|
|
42
47
|
interface ReadableTag {
|
|
43
48
|
text(...path: string[]): string | undefined;
|
|
49
|
+
tag(...path: string[]): ReadableTag | undefined;
|
|
50
|
+
entries?(): Iterable<[string, { text(): string | undefined }]>;
|
|
44
51
|
}
|
|
45
52
|
|
|
46
53
|
/**
|
|
@@ -238,6 +245,29 @@ function tagFreshnessLayer(tag: ReadableTag | undefined): FreshnessLayer {
|
|
|
238
245
|
return layer;
|
|
239
246
|
}
|
|
240
247
|
|
|
248
|
+
/**
|
|
249
|
+
* The two homes a model-file (`##`) knob can be declared in, most specific
|
|
250
|
+
* first: the `materialization` envelope, then the bare form.
|
|
251
|
+
*
|
|
252
|
+
* The model-file level mirrors the manifest's shape verbatim — `##
|
|
253
|
+
* materialization.freshness.window="24h"` is the manifest's block in tag syntax
|
|
254
|
+
* — so that is where a file-level reader looks. The bare form
|
|
255
|
+
* (`## freshness.window="24h"`), which shipped first, stays readable underneath:
|
|
256
|
+
* the annotation rides the published package, so a package published before the
|
|
257
|
+
* envelope existed keeps resolving until it is republished. Resolution is
|
|
258
|
+
* per-property, so a file may declare one knob in each home without the envelope
|
|
259
|
+
* hiding the other.
|
|
260
|
+
*/
|
|
261
|
+
function modelTagLayers(
|
|
262
|
+
tag: ReadableTag | undefined,
|
|
263
|
+
): (ReadableTag | undefined)[] {
|
|
264
|
+
const envelope =
|
|
265
|
+
tag && typeof tag.tag === "function"
|
|
266
|
+
? tag.tag("materialization")
|
|
267
|
+
: undefined;
|
|
268
|
+
return [envelope, tag];
|
|
269
|
+
}
|
|
270
|
+
|
|
241
271
|
/** The package-level `materialization.freshness` as a resolution layer. */
|
|
242
272
|
function packageFreshnessLayer(
|
|
243
273
|
cfg: WirePackageMaterialization | null | undefined,
|
|
@@ -289,13 +319,16 @@ export function resolveFreshness(
|
|
|
289
319
|
source: PersistSource,
|
|
290
320
|
packageMaterialization: WirePackageMaterialization | null | undefined,
|
|
291
321
|
): WireFreshness | null {
|
|
292
|
-
const
|
|
293
|
-
|
|
294
|
-
|
|
322
|
+
const layers: FreshnessLayer[] = [
|
|
323
|
+
// `#@ persist` declares knobs bare — the annotation IS a materialization
|
|
324
|
+
// declaration, so there is no envelope to look under.
|
|
325
|
+
tagFreshnessLayer(safeSourceTag(source)),
|
|
326
|
+
...modelTagLayers(safeModelTag(source)).map(tagFreshnessLayer),
|
|
327
|
+
packageFreshnessLayer(packageMaterialization),
|
|
328
|
+
];
|
|
295
329
|
|
|
296
|
-
const window =
|
|
297
|
-
const fallback =
|
|
298
|
-
sourceLayer.fallback ?? modelLayer.fallback ?? pkgLayer.fallback;
|
|
330
|
+
const window = layers.map((l) => l.window).find((v) => v !== undefined);
|
|
331
|
+
const fallback = layers.map((l) => l.fallback).find((v) => v !== undefined);
|
|
299
332
|
|
|
300
333
|
if (window === undefined && fallback === undefined) return null;
|
|
301
334
|
const freshness: WireFreshness = {};
|
|
@@ -304,6 +337,69 @@ export function resolveFreshness(
|
|
|
304
337
|
return freshness;
|
|
305
338
|
}
|
|
306
339
|
|
|
340
|
+
/**
|
|
341
|
+
* Read the `queryMetadata` property collection from one tag layer. A collection,
|
|
342
|
+
* not a scalar: `queryMetadata { team="finance" env="prod" }` and the equivalent
|
|
343
|
+
* dotted form `queryMetadata.team="finance"` both land here, and neither is
|
|
344
|
+
* captured by the scalar {@link deriveAnnotationFields} loop.
|
|
345
|
+
*
|
|
346
|
+
* Every string-valued property is kept verbatim, including ones that violate
|
|
347
|
+
* Malloy's bag contract — publish reports those as warnings and the runtime
|
|
348
|
+
* clamps them, so an author's typo is visible somewhere instead of vanishing
|
|
349
|
+
* between the annotation and the warehouse.
|
|
350
|
+
*/
|
|
351
|
+
function tagQueryMetadataLayer(tag: ReadableTag | undefined): QueryMetadata {
|
|
352
|
+
const subtree =
|
|
353
|
+
tag && typeof tag.tag === "function"
|
|
354
|
+
? tag.tag("queryMetadata")
|
|
355
|
+
: undefined;
|
|
356
|
+
if (!subtree || typeof subtree.entries !== "function") return {};
|
|
357
|
+
const layer: QueryMetadata = {};
|
|
358
|
+
try {
|
|
359
|
+
for (const [name, value] of subtree.entries()) {
|
|
360
|
+
const text = value.text();
|
|
361
|
+
if (text !== undefined) layer[name] = text;
|
|
362
|
+
}
|
|
363
|
+
} catch {
|
|
364
|
+
// Degrade to {} — mirrors deriveAnnotationFields / deriveColumns.
|
|
365
|
+
return {};
|
|
366
|
+
}
|
|
367
|
+
return layer;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/**
|
|
371
|
+
* Resolve a source's EFFECTIVE per-query metadata, most-specific-wins PER
|
|
372
|
+
* PROPERTY: `#@ persist queryMetadata.*` > model-file
|
|
373
|
+
* `## materialization.queryMetadata.*` (bare `## queryMetadata.*` underneath) >
|
|
374
|
+
* package `materialization.queryMetadata`.
|
|
375
|
+
*
|
|
376
|
+
* Per-property rather than per-layer, exactly like {@link resolveFreshness}, so a
|
|
377
|
+
* package-wide `team` property survives a source that only overrides `workload`.
|
|
378
|
+
* Null when no layer declares anything, so absence on the wire always means
|
|
379
|
+
* "declared nowhere" rather than "declared empty".
|
|
380
|
+
*
|
|
381
|
+
* This is the value the publisher attaches (merged under its own context) to
|
|
382
|
+
* every statement it issues while building the source. It is deliberately absent
|
|
383
|
+
* from the source's content address: changing a tag must never re-address a
|
|
384
|
+
* table.
|
|
385
|
+
*/
|
|
386
|
+
export function resolveQueryMetadata(
|
|
387
|
+
source: PersistSource,
|
|
388
|
+
packageMaterialization: WirePackageMaterialization | null | undefined,
|
|
389
|
+
): QueryMetadata | null {
|
|
390
|
+
// Least specific first, so a more specific layer overwrites property by
|
|
391
|
+
// property.
|
|
392
|
+
const layers: QueryMetadata[] = [
|
|
393
|
+
packageMaterialization?.queryMetadata ?? {},
|
|
394
|
+
...modelTagLayers(safeModelTag(source))
|
|
395
|
+
.map(tagQueryMetadataLayer)
|
|
396
|
+
.reverse(),
|
|
397
|
+
tagQueryMetadataLayer(safeSourceTag(source)),
|
|
398
|
+
];
|
|
399
|
+
const resolved: QueryMetadata = Object.assign({}, ...layers);
|
|
400
|
+
return Object.keys(resolved).length > 0 ? resolved : null;
|
|
401
|
+
}
|
|
402
|
+
|
|
307
403
|
/** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
|
|
308
404
|
export function flattenDependsOn(node: {
|
|
309
405
|
dependsOn: { sourceID: string }[];
|
|
@@ -612,6 +708,11 @@ export function deriveBuildPlan(
|
|
|
612
708
|
sql: source.getSQL(),
|
|
613
709
|
refresh: annotationFields.refresh ?? null,
|
|
614
710
|
freshness: resolveFreshness(source, packageMaterialization),
|
|
711
|
+
// EFFECTIVE per-source query metadata, resolved per property across the
|
|
712
|
+
// same layer stack as freshness. A property collection rather than a
|
|
713
|
+
// scalar, so it comes from resolveQueryMetadata rather than the
|
|
714
|
+
// annotationFields map.
|
|
715
|
+
queryMetadata: resolveQueryMetadata(source, packageMaterialization),
|
|
615
716
|
columns: deriveColumns(source),
|
|
616
717
|
annotationFields,
|
|
617
718
|
modelPath: sourceModelPaths?.[sourceID],
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
|
2
|
+
import fs from "fs/promises";
|
|
3
|
+
import os from "os";
|
|
4
|
+
import path from "path";
|
|
5
|
+
import { Environment } from "./environment";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* The fragment-checking advice in malloy_compile's tool description, pinned
|
|
9
|
+
* against the REAL compiler.
|
|
10
|
+
*
|
|
11
|
+
* That description tells an agent how to validate part of a source: send a view
|
|
12
|
+
* body as a top-level `query:`, or wrap a field in a throwaway `extend`, and do
|
|
13
|
+
* not resubmit the source being edited. None of that is enforced by code, so
|
|
14
|
+
* nothing but a test stops it from quietly becoming false. Asserting it against
|
|
15
|
+
* a hand-written checker would be worthless: only Malloy decides what compiles.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
const PUBLISHER_JSON = JSON.stringify({
|
|
19
|
+
name: "pkg",
|
|
20
|
+
description: "fragments",
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
// `secret` is private on the base source, so nothing downstream may read it.
|
|
24
|
+
const MODEL = `##! experimental.access_modifiers
|
|
25
|
+
source: base is duckdb.sql("SELECT 1.5 as price, 90 as points, 'shh' as secret") include {
|
|
26
|
+
public: price, points
|
|
27
|
+
private: secret
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
source: sales is base extend {
|
|
31
|
+
measure: record_count is count()
|
|
32
|
+
measure: avg_price is avg(price)
|
|
33
|
+
view: overview is { aggregate: record_count }
|
|
34
|
+
}
|
|
35
|
+
`;
|
|
36
|
+
|
|
37
|
+
describe("malloy_compile: checking part of a source", () => {
|
|
38
|
+
let rootDir: string;
|
|
39
|
+
let env: Environment;
|
|
40
|
+
|
|
41
|
+
beforeEach(async () => {
|
|
42
|
+
rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-frag-"));
|
|
43
|
+
const envPath = path.join(rootDir, "env");
|
|
44
|
+
await fs.mkdir(envPath, { recursive: true });
|
|
45
|
+
env = await Environment.create("testEnv", envPath, []);
|
|
46
|
+
await env.installPackage("pkg", async (stagingPath) => {
|
|
47
|
+
await fs.mkdir(stagingPath, { recursive: true });
|
|
48
|
+
await fs.writeFile(
|
|
49
|
+
path.join(stagingPath, "publisher.json"),
|
|
50
|
+
PUBLISHER_JSON,
|
|
51
|
+
);
|
|
52
|
+
await fs.writeFile(path.join(stagingPath, "model.malloy"), MODEL);
|
|
53
|
+
});
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
afterEach(async () => {
|
|
57
|
+
await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
/** Error-severity messages from compiling `source` against the fixture. */
|
|
61
|
+
async function errorsFor(source: string): Promise<string[]> {
|
|
62
|
+
const result = await env.compileSource("pkg", "model.malloy", source);
|
|
63
|
+
return (result.problems ?? [])
|
|
64
|
+
.filter((p) => p.severity === "error")
|
|
65
|
+
.map((p) => p.message);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
describe("the two failures the description exists to explain", () => {
|
|
69
|
+
it("rejects a bare view: fragment, naming only the symptom", async () => {
|
|
70
|
+
const errors = await errorsFor(
|
|
71
|
+
"view: by_pts is { group_by: points, aggregate: record_count }",
|
|
72
|
+
);
|
|
73
|
+
expect(errors.join(" ")).toContain("view:");
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it("rejects resubmitting the source being edited", async () => {
|
|
77
|
+
// The natural move when validating an edit, and it fails for a reason
|
|
78
|
+
// that has nothing to do with the edit.
|
|
79
|
+
const errors = await errorsFor(`source: sales is base extend {
|
|
80
|
+
measure: record_count is count()
|
|
81
|
+
view: by_pts is { group_by: points }
|
|
82
|
+
}`);
|
|
83
|
+
expect(errors.join(" ")).toContain("Cannot redefine 'sales'");
|
|
84
|
+
});
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
describe("the top-level query: form", () => {
|
|
88
|
+
it("compiles a view body with no wrapper at all", async () => {
|
|
89
|
+
expect(
|
|
90
|
+
await errorsFor(
|
|
91
|
+
"query: check is sales -> { group_by: points, aggregate: record_count }",
|
|
92
|
+
),
|
|
93
|
+
).toEqual([]);
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
it("still resolves measures the source inherits", async () => {
|
|
97
|
+
expect(
|
|
98
|
+
await errorsFor(
|
|
99
|
+
"query: check is sales -> { aggregate: avg_price }",
|
|
100
|
+
),
|
|
101
|
+
).toEqual([]);
|
|
102
|
+
});
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
describe("the throwaway extend form", () => {
|
|
106
|
+
const wrap = (fragment: string, into = "sales") =>
|
|
107
|
+
`source: check is ${into} extend {\n${fragment}\n}`;
|
|
108
|
+
|
|
109
|
+
it("compiles bare view, measure, and dimension fragments", async () => {
|
|
110
|
+
expect(
|
|
111
|
+
await errorsFor(
|
|
112
|
+
wrap(
|
|
113
|
+
"view: by_pts is { group_by: points, aggregate: record_count }",
|
|
114
|
+
),
|
|
115
|
+
),
|
|
116
|
+
).toEqual([]);
|
|
117
|
+
expect(
|
|
118
|
+
await errorsFor(wrap("measure: max_price is max(price)")),
|
|
119
|
+
).toEqual([]);
|
|
120
|
+
expect(
|
|
121
|
+
await errorsFor(wrap("dimension: pricey is price > 1")),
|
|
122
|
+
).toEqual([]);
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
it("resolves measures the source inherits", async () => {
|
|
126
|
+
expect(
|
|
127
|
+
await errorsFor(wrap("view: v is { aggregate: avg_price }")),
|
|
128
|
+
).toEqual([]);
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
it("hides a private field exactly as an in-place extension does", async () => {
|
|
132
|
+
// The fidelity claim in the description: the wrapper must not widen or
|
|
133
|
+
// narrow visibility versus the edit it stands in for.
|
|
134
|
+
const wrapped = await errorsFor(wrap("dimension: s is secret"));
|
|
135
|
+
const inPlace = await errorsFor(
|
|
136
|
+
"source: probe is base extend { dimension: s is secret }",
|
|
137
|
+
);
|
|
138
|
+
expect(wrapped.join(" ")).toContain("private");
|
|
139
|
+
expect(inPlace.join(" ")).toContain("private");
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
it("reports a redefinition when the fragment reuses an existing name", async () => {
|
|
143
|
+
// The documented caveat: an extension adds to the namespace rather
|
|
144
|
+
// than overriding it, so checking an EDIT needs the fragment renamed.
|
|
145
|
+
expect(
|
|
146
|
+
await errorsFor(wrap("view: overview is { aggregate: avg_price }")),
|
|
147
|
+
).toEqual(["Cannot redefine 'overview'"]);
|
|
148
|
+
|
|
149
|
+
expect(
|
|
150
|
+
await errorsFor(
|
|
151
|
+
wrap("view: overview__check is { aggregate: avg_price }"),
|
|
152
|
+
),
|
|
153
|
+
).toEqual([]);
|
|
154
|
+
});
|
|
155
|
+
});
|
|
156
|
+
});
|