@malloy-publisher/server 0.0.233 → 0.0.235
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.docker.md +1 -0
- package/dist/app/api-doc.yaml +208 -0
- package/dist/app/assets/{EnvironmentPage-DutP7T8h.js → EnvironmentPage-BsAnavYN.js} +1 -1
- package/dist/app/assets/{HomePage-BcxDrBfl.js → HomePage-CADE138j.js} +1 -1
- package/dist/app/assets/{LightMode-BJukGxgz.js → LightMode-Cfh7KzN8.js} +1 -1
- package/dist/app/assets/{MainPage-DXbwlMeF.js → MainPage-CO3pRlnV.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-BBQksmTU.js → MaterializationsPage-p9YjkRXZ.js} +1 -1
- package/dist/app/assets/{ModelPage-C6tK51uU.js → ModelPage-C1OSTv-x.js} +1 -1
- package/dist/app/assets/{PackagePage-Bo3cwwZE.js → PackagePage-e4kN75YR.js} +1 -1
- package/dist/app/assets/{RouteError-BufkcAKE.js → RouteError-CzbfOkng.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-DICvvKpa.js → ThemeEditorPage-CciagFTq.js} +1 -1
- package/dist/app/assets/{WorkbookPage-Dkwt75Nj.js → WorkbookPage-DNWmkCXa.js} +1 -1
- package/dist/app/assets/{core-C0nunIQT.es-DlMLKZBK.js → core-Rj_4rRnA.es-BZyvITuO.js} +1 -1
- package/dist/app/assets/{index-CmEVVe-8.js → index-CH2AcDzc.js} +4 -4
- package/dist/app/assets/{index-qnhU9CGo.js → index-DQa463gC.js} +2 -2
- package/dist/app/assets/{index-Cs4WVm2z.js → index-DQpV7MyA.js} +1 -1
- package/dist/app/assets/{index-BabP-V-S.js → index-DzaYbhnD.js} +1 -1
- package/dist/app/assets/{index-BusxL5Pt.js → index-VBbcc8s6.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/package_load_worker.mjs +53 -3
- package/dist/server.mjs +835 -97
- package/package.json +12 -12
- package/src/config.ts +35 -1
- package/src/controller/connection.controller.spec.ts +46 -0
- package/src/controller/connection.controller.ts +105 -2
- package/src/controller/materialization.controller.spec.ts +25 -0
- package/src/controller/materialization.controller.ts +60 -0
- package/src/controller/model.controller.ts +24 -0
- package/src/controller/query.controller.ts +83 -15
- package/src/mcp/handler_utils.ts +10 -2
- package/src/mcp/query_envelope.ts +10 -0
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/tools/execute_query_tool.spec.ts +131 -0
- package/src/mcp/tools/execute_query_tool.ts +62 -25
- package/src/mcp_config.spec.ts +919 -0
- package/src/mcp_config.ts +425 -0
- package/src/oom_guards.integration.spec.ts +11 -3
- package/src/package_load/package_load_pool.ts +2 -0
- package/src/package_load/package_load_worker.ts +17 -5
- package/src/package_load/protocol.ts +6 -0
- package/src/query_metadata_metrics.ts +49 -0
- package/src/server.ts +99 -3
- package/src/service/build_plan.spec.ts +125 -0
- package/src/service/build_plan.ts +108 -7
- package/src/service/connection_config.ts +49 -0
- package/src/service/environment.ts +57 -3
- package/src/service/materialization_config_validation.spec.ts +99 -0
- package/src/service/materialization_config_validation.ts +120 -0
- package/src/service/materialization_schedule_surface.spec.ts +124 -0
- package/src/service/materialization_service.spec.ts +119 -0
- package/src/service/materialization_service.ts +186 -3
- package/src/service/materialization_test_fixtures.ts +86 -21
- package/src/service/model.spec.ts +45 -1
- package/src/service/model.ts +145 -19
- package/src/service/package.ts +24 -1
- package/src/service/package_manifest.spec.ts +137 -4
- package/src/service/package_manifest.ts +140 -5
- package/src/service/persist_annotation_validation.spec.ts +12 -0
- package/src/service/persist_annotation_validation.ts +9 -4
- package/src/service/query_metadata.spec.ts +408 -0
- package/src/service/query_metadata.ts +492 -0
- package/src/service/query_metadata_identity.spec.ts +149 -0
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@malloy-publisher/server",
|
|
3
3
|
"description": "Malloy Publisher Server",
|
|
4
|
-
"version": "0.0.
|
|
4
|
+
"version": "0.0.235",
|
|
5
5
|
"main": "dist/server.mjs",
|
|
6
6
|
"bin": {
|
|
7
7
|
"malloy-publisher": "dist/server.mjs"
|
|
@@ -36,17 +36,17 @@
|
|
|
36
36
|
"@azure/storage-blob": "^12.26.0",
|
|
37
37
|
"@duckdb/node-api": "1.5.3-r.2",
|
|
38
38
|
"@google-cloud/storage": "^7.16.0",
|
|
39
|
-
"@malloydata/db-bigquery": "^0.0.
|
|
40
|
-
"@malloydata/db-databricks": "^0.0.
|
|
41
|
-
"@malloydata/db-duckdb": "^0.0.
|
|
42
|
-
"@malloydata/db-mysql": "^0.0.
|
|
43
|
-
"@malloydata/db-postgres": "^0.0.
|
|
44
|
-
"@malloydata/db-publisher": "^0.0.
|
|
45
|
-
"@malloydata/db-snowflake": "^0.0.
|
|
46
|
-
"@malloydata/db-trino": "^0.0.
|
|
47
|
-
"@malloydata/malloy": "^0.0.
|
|
48
|
-
"@malloydata/malloy-sql": "^0.0.
|
|
49
|
-
"@malloydata/render-validator": "^0.0.
|
|
39
|
+
"@malloydata/db-bigquery": "^0.0.427",
|
|
40
|
+
"@malloydata/db-databricks": "^0.0.427",
|
|
41
|
+
"@malloydata/db-duckdb": "^0.0.427",
|
|
42
|
+
"@malloydata/db-mysql": "^0.0.427",
|
|
43
|
+
"@malloydata/db-postgres": "^0.0.427",
|
|
44
|
+
"@malloydata/db-publisher": "^0.0.427",
|
|
45
|
+
"@malloydata/db-snowflake": "^0.0.427",
|
|
46
|
+
"@malloydata/db-trino": "^0.0.427",
|
|
47
|
+
"@malloydata/malloy": "^0.0.427",
|
|
48
|
+
"@malloydata/malloy-sql": "^0.0.427",
|
|
49
|
+
"@malloydata/render-validator": "^0.0.427",
|
|
50
50
|
"@modelcontextprotocol/sdk": "^1.13.2",
|
|
51
51
|
"@opentelemetry/api": "^1.9.0",
|
|
52
52
|
"@opentelemetry/auto-instrumentations-node": "^0.57.0",
|
package/src/config.ts
CHANGED
|
@@ -189,7 +189,7 @@ function parseFloatEnv(name: string): number | undefined {
|
|
|
189
189
|
return value;
|
|
190
190
|
}
|
|
191
191
|
|
|
192
|
-
function parseBoolEnv(name: string): boolean | undefined {
|
|
192
|
+
export function parseBoolEnv(name: string): boolean | undefined {
|
|
193
193
|
const raw = process.env[name];
|
|
194
194
|
if (raw === undefined || raw.trim() === "") return undefined;
|
|
195
195
|
const normalised = raw.trim().toLowerCase();
|
|
@@ -615,6 +615,40 @@ export const getPersistCollisionEnforce = (): boolean =>
|
|
|
615
615
|
// prevent. A typo throws at startup, like every other flag here.
|
|
616
616
|
parseBoolEnv("PERSIST_COLLISION_ENFORCE") ?? false;
|
|
617
617
|
|
|
618
|
+
/**
|
|
619
|
+
* Whether the publisher attaches per-query metadata at all, from
|
|
620
|
+
* `PUBLISHER_QUERY_METADATA` (default `off`).
|
|
621
|
+
*
|
|
622
|
+
* Ships dark for a release, like `PERSIST_STORAGE_MODE` before it, and for the
|
|
623
|
+
* same reason: this is the rare feature that touches EVERY statement the server
|
|
624
|
+
* sends. On a backend with no native tag facility the bag rides as a leading SQL
|
|
625
|
+
* comment, so `on` changes the text of the statement (never its meaning or its
|
|
626
|
+
* results) and puts the bag in query logs and `pg_stat_activity`.
|
|
627
|
+
*
|
|
628
|
+
* The risk that decides the default is upstream, not here. Malloy validates the
|
|
629
|
+
* bag at dispatch and THROWS on one it cannot render, and the contract it
|
|
630
|
+
* validates against is mirrored in `service/query_metadata.ts` against a pinned
|
|
631
|
+
* version. Every mitigation on this path — clamping, shedding, never throwing —
|
|
632
|
+
* is downstream of that mirror being right, so a tightened upstream limit would
|
|
633
|
+
* surface as failing customer queries on a path nobody opted into. `off` for a
|
|
634
|
+
* release means a deployment turns attribution on deliberately, having read
|
|
635
|
+
* what it does to its statements.
|
|
636
|
+
*
|
|
637
|
+
* Case-insensitive; loud-fails on an unrecognized value, so a typo cannot
|
|
638
|
+
* silently leave a deployment that asked for attribution without it.
|
|
639
|
+
*/
|
|
640
|
+
export type QueryMetadataMode = "on" | "off";
|
|
641
|
+
|
|
642
|
+
export const getQueryMetadataMode = (): QueryMetadataMode => {
|
|
643
|
+
const raw = process.env.PUBLISHER_QUERY_METADATA;
|
|
644
|
+
if (raw === undefined || raw.trim() === "") return "off";
|
|
645
|
+
const value = raw.trim().toLowerCase();
|
|
646
|
+
if (value === "on" || value === "off") return value;
|
|
647
|
+
throw new Error(
|
|
648
|
+
`PUBLISHER_QUERY_METADATA must be on | off (got ${JSON.stringify(raw)})`,
|
|
649
|
+
);
|
|
650
|
+
};
|
|
651
|
+
|
|
618
652
|
function substituteEnvVars(value: string): string {
|
|
619
653
|
const envVarPattern = /\$\{([A-Z_][A-Z0-9_]*)\}/g;
|
|
620
654
|
|
|
@@ -60,6 +60,52 @@ function buildController(
|
|
|
60
60
|
return { controller, runSQL, assertCanAdmitQuery };
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
+
describe("ConnectionController connection queryMetadata", () => {
|
|
64
|
+
afterEach(() => sinon.restore());
|
|
65
|
+
|
|
66
|
+
it("rejects a connection default the contract cannot render", async () => {
|
|
67
|
+
// The connection is the one metadata layer whose author is right here, so
|
|
68
|
+
// it is the one that can be told. `cost-centre` is the natural spelling
|
|
69
|
+
// and would otherwise be dropped at dispatch, leaving every statement on
|
|
70
|
+
// the connection missing the property its operator configured.
|
|
71
|
+
const { controller } = buildController(sinon.stub().resolves());
|
|
72
|
+
await expect(
|
|
73
|
+
controller.addConnection("env", "warehouse", {
|
|
74
|
+
name: "warehouse",
|
|
75
|
+
type: "postgres",
|
|
76
|
+
queryMetadata: { "cost-centre": "eng" },
|
|
77
|
+
}),
|
|
78
|
+
).rejects.toThrow(/queryMetadata is invalid/);
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it("rejects an enforced bag the contract cannot render", async () => {
|
|
82
|
+
const { controller } = buildController(sinon.stub().resolves());
|
|
83
|
+
await expect(
|
|
84
|
+
controller.addConnection("env", "warehouse", {
|
|
85
|
+
name: "warehouse",
|
|
86
|
+
type: "postgres",
|
|
87
|
+
queryMetadataEnforced: { "org.id": "acme" },
|
|
88
|
+
}),
|
|
89
|
+
).rejects.toThrow(/queryMetadataEnforced is invalid/);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it("accepts a conforming connection default", async () => {
|
|
93
|
+
const { controller } = buildController(sinon.stub().resolves());
|
|
94
|
+
const addConnection = sinon.stub().resolves();
|
|
95
|
+
(
|
|
96
|
+
controller as unknown as {
|
|
97
|
+
connectionService: { addConnection: sinon.SinonStub };
|
|
98
|
+
}
|
|
99
|
+
).connectionService = { addConnection } as never;
|
|
100
|
+
await controller.addConnection("env", "warehouse", {
|
|
101
|
+
name: "warehouse",
|
|
102
|
+
type: "postgres",
|
|
103
|
+
queryMetadata: { cost_centre: "eng" },
|
|
104
|
+
});
|
|
105
|
+
expect(addConnection.calledOnce).toBe(true);
|
|
106
|
+
});
|
|
107
|
+
});
|
|
108
|
+
|
|
63
109
|
describe("ConnectionController.getConnectionQueryData row cap", () => {
|
|
64
110
|
const originalEnv = process.env.PUBLISHER_MAX_QUERY_ROWS;
|
|
65
111
|
|
|
@@ -21,6 +21,15 @@ import {
|
|
|
21
21
|
getSchemasForConnection,
|
|
22
22
|
listTablesForSchema,
|
|
23
23
|
} from "../service/db_utils";
|
|
24
|
+
import {
|
|
25
|
+
mergeQueryMetadata,
|
|
26
|
+
mintCorrelationId,
|
|
27
|
+
parseQueryClass,
|
|
28
|
+
parseSuppliedQueryMetadata,
|
|
29
|
+
queryMetadataViolations,
|
|
30
|
+
type QueryClass,
|
|
31
|
+
type QueryMetadata,
|
|
32
|
+
} from "../service/query_metadata";
|
|
24
33
|
import type { Environment } from "../service/environment";
|
|
25
34
|
import { EnvironmentStore } from "../service/environment_store";
|
|
26
35
|
import { isStreamingConnection, streamSqlWithBudget } from "../stream_helpers";
|
|
@@ -112,6 +121,22 @@ function validateAdminAuthoredConnection(
|
|
|
112
121
|
} catch (error) {
|
|
113
122
|
throw new BadRequestError((error as Error).message);
|
|
114
123
|
}
|
|
124
|
+
|
|
125
|
+
// The connection is the one metadata layer whose author is right here, so it
|
|
126
|
+
// is the one that can be told. A property the contract rejects (a hyphen in
|
|
127
|
+
// `cost-centre` is the natural spelling and the first thing anyone tries)
|
|
128
|
+
// would otherwise be dropped at dispatch and only ever surface as a metric —
|
|
129
|
+
// every statement on the connection missing the property its operator
|
|
130
|
+
// believes they configured. Config LOAD warns instead of throwing (see
|
|
131
|
+
// assembleEnvironmentConnections): a tag must never fail an environment.
|
|
132
|
+
for (const field of ["queryMetadata", "queryMetadataEnforced"] as const) {
|
|
133
|
+
const violations = queryMetadataViolations(connectionConfig[field]);
|
|
134
|
+
if (violations.length > 0) {
|
|
135
|
+
throw new BadRequestError(
|
|
136
|
+
`Connection "${connectionName}" ${field} is invalid: ${violations.join("; ")}`,
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
115
140
|
}
|
|
116
141
|
|
|
117
142
|
export class ConnectionController {
|
|
@@ -140,6 +165,43 @@ export class ConnectionController {
|
|
|
140
165
|
return environment.getApiConnection(connectionName);
|
|
141
166
|
}
|
|
142
167
|
|
|
168
|
+
/**
|
|
169
|
+
* A connection's default per-query metadata, or null when it declares none.
|
|
170
|
+
* Fails open: metadata is observability, so a connection whose config can't be
|
|
171
|
+
* read contributes no default rather than failing the query the caller asked
|
|
172
|
+
* for.
|
|
173
|
+
*/
|
|
174
|
+
private async connectionQueryMetadata(
|
|
175
|
+
environmentName: string,
|
|
176
|
+
connectionName: string,
|
|
177
|
+
): Promise<{
|
|
178
|
+
default: QueryMetadata | null;
|
|
179
|
+
enforced: QueryMetadata | null;
|
|
180
|
+
}> {
|
|
181
|
+
try {
|
|
182
|
+
const environment = await this.environmentStore.getEnvironment(
|
|
183
|
+
environmentName,
|
|
184
|
+
false,
|
|
185
|
+
);
|
|
186
|
+
const connection = this.getApiConnectionForLookup(
|
|
187
|
+
environment,
|
|
188
|
+
connectionName,
|
|
189
|
+
);
|
|
190
|
+
return {
|
|
191
|
+
default: connection.queryMetadata ?? null,
|
|
192
|
+
enforced: connection.queryMetadataEnforced ?? null,
|
|
193
|
+
};
|
|
194
|
+
} catch (error) {
|
|
195
|
+
// Fails open like every other metadata path, but not invisibly: the
|
|
196
|
+
// layer lost here is the enforced one, and no metric covers it.
|
|
197
|
+
logger.debug("No query-metadata layers for connection", {
|
|
198
|
+
connectionName,
|
|
199
|
+
error,
|
|
200
|
+
});
|
|
201
|
+
return { default: null, enforced: null };
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
143
205
|
private async getMalloyConnection(
|
|
144
206
|
environmentName: string,
|
|
145
207
|
connectionName: string,
|
|
@@ -453,6 +515,12 @@ export class ConnectionController {
|
|
|
453
515
|
sqlStatement: string,
|
|
454
516
|
options: string,
|
|
455
517
|
packageName?: string,
|
|
518
|
+
/**
|
|
519
|
+
* The request's per-query metadata fields, unvalidated — this controller is
|
|
520
|
+
* the boundary that turns a bad bag into a 400 rather than letting the
|
|
521
|
+
* connector refuse the statement at dispatch.
|
|
522
|
+
*/
|
|
523
|
+
metadata?: { queryMetadata?: unknown; queryClass?: unknown },
|
|
456
524
|
): Promise<ApiQueryData> {
|
|
457
525
|
// Express parses repeated query parameters (?sqlStatement=a&sqlStatement=b)
|
|
458
526
|
// and array-shaped JSON bodies as `string[]`, not `string`. The route
|
|
@@ -512,6 +580,41 @@ export class ConnectionController {
|
|
|
512
580
|
runSQLOptions.abortSignal = undefined;
|
|
513
581
|
}
|
|
514
582
|
|
|
583
|
+
// Per-query metadata. Validated here, not clamped: a raw-SQL caller gets a
|
|
584
|
+
// 400 telling it which property is wrong instead of a statement the
|
|
585
|
+
// connector refuses at dispatch. `options` is forwarded as RunSQLOptions, so
|
|
586
|
+
// a bag can also arrive inside it — validate that one too, and let the
|
|
587
|
+
// documented field win.
|
|
588
|
+
const suppliedMetadata =
|
|
589
|
+
metadata?.queryMetadata ?? runSQLOptions.queryMetadata;
|
|
590
|
+
let requestMetadata: QueryMetadata | undefined;
|
|
591
|
+
let queryClass: QueryClass | undefined;
|
|
592
|
+
try {
|
|
593
|
+
requestMetadata = parseSuppliedQueryMetadata(suppliedMetadata);
|
|
594
|
+
queryClass = parseQueryClass(metadata?.queryClass);
|
|
595
|
+
} catch (error) {
|
|
596
|
+
throw new BadRequestError((error as Error).message);
|
|
597
|
+
}
|
|
598
|
+
const connectionLayers = await this.connectionQueryMetadata(
|
|
599
|
+
environmentName,
|
|
600
|
+
connectionName,
|
|
601
|
+
);
|
|
602
|
+
const resolvedMetadata = mergeQueryMetadata({
|
|
603
|
+
connection: connectionLayers.default,
|
|
604
|
+
enforced: connectionLayers.enforced,
|
|
605
|
+
request: requestMetadata,
|
|
606
|
+
context: {
|
|
607
|
+
// Raw SQL against a connection is platform maintenance unless the
|
|
608
|
+
// caller says otherwise — it is not a modeled query.
|
|
609
|
+
queryClass: queryClass ?? "ops",
|
|
610
|
+
environment: environmentName,
|
|
611
|
+
package: packageName,
|
|
612
|
+
correlationId: mintCorrelationId(),
|
|
613
|
+
},
|
|
614
|
+
});
|
|
615
|
+
runSQLOptions.queryMetadata = resolvedMetadata.metadata;
|
|
616
|
+
const queryCorrelationId = resolvedMetadata.metadata?.query_id ?? null;
|
|
617
|
+
|
|
515
618
|
// Bound the response with two layered caps:
|
|
516
619
|
//
|
|
517
620
|
// - Row cap (PUBLISHER_MAX_QUERY_ROWS) — pushed to the driver as
|
|
@@ -572,7 +675,7 @@ export class ConnectionController {
|
|
|
572
675
|
throw new ConnectionError((error as Error).message);
|
|
573
676
|
}
|
|
574
677
|
}, getQueryTimeoutMs());
|
|
575
|
-
return { data: JSON.stringify(streamed) };
|
|
678
|
+
return { data: JSON.stringify(streamed), queryCorrelationId };
|
|
576
679
|
}
|
|
577
680
|
|
|
578
681
|
const result = await runWithQueryTimeout(async (signal) => {
|
|
@@ -601,7 +704,7 @@ export class ConnectionController {
|
|
|
601
704
|
);
|
|
602
705
|
}
|
|
603
706
|
|
|
604
|
-
return { data: JSON.stringify(result) };
|
|
707
|
+
return { data: JSON.stringify(result), queryCorrelationId };
|
|
605
708
|
}
|
|
606
709
|
|
|
607
710
|
public async getConnectionTemporaryTable(
|
|
@@ -265,4 +265,29 @@ describe("MaterializationController.createMaterialization validation", () => {
|
|
|
265
265
|
}),
|
|
266
266
|
).rejects.toThrow(BadRequestError);
|
|
267
267
|
});
|
|
268
|
+
|
|
269
|
+
it("rejects a runId the metadata contract cannot carry", async () => {
|
|
270
|
+
// runId becomes the `run_id` property on every statement of the build, so
|
|
271
|
+
// an over-long or unrenderable value is silently truncated and rewritten
|
|
272
|
+
// at dispatch — leaving the caller holding an id that joins to nothing.
|
|
273
|
+
// The neighbouring `trigger` is enum-validated; this is the same kind of
|
|
274
|
+
// boundary and gets the same treatment.
|
|
275
|
+
const { controller } = build();
|
|
276
|
+
await expect(
|
|
277
|
+
controller.createMaterialization("env", "pkg", {
|
|
278
|
+
runContext: { runId: "x".repeat(300) },
|
|
279
|
+
}),
|
|
280
|
+
).rejects.toThrow(BadRequestError);
|
|
281
|
+
await expect(
|
|
282
|
+
controller.createMaterialization("env", "pkg", {
|
|
283
|
+
runContext: { runId: 'has "quotes"' },
|
|
284
|
+
}),
|
|
285
|
+
).rejects.toThrow(BadRequestError);
|
|
286
|
+
});
|
|
287
|
+
|
|
288
|
+
it("still accepts a conforming runId", async () => {
|
|
289
|
+
expect(
|
|
290
|
+
await parse({ runContext: { trigger: "publish", runId: "run-42" } }),
|
|
291
|
+
).toEqual({ runContext: { trigger: "publish", runId: "run-42" } });
|
|
292
|
+
});
|
|
268
293
|
});
|
|
@@ -1,9 +1,15 @@
|
|
|
1
|
+
import type { components } from "../api";
|
|
1
2
|
import { BadRequestError } from "../errors";
|
|
2
3
|
import {
|
|
3
4
|
BuildInstruction,
|
|
4
5
|
ManifestReference,
|
|
5
6
|
} from "../storage/DatabaseInterface";
|
|
6
7
|
import { MaterializationService } from "../service/materialization_service";
|
|
8
|
+
import { queryMetadataViolations } from "../service/query_metadata";
|
|
9
|
+
|
|
10
|
+
type RunContext = components["schemas"]["RunContext"];
|
|
11
|
+
|
|
12
|
+
const RUN_TRIGGERS = ["publish", "on_demand", "scheduler"] as const;
|
|
7
13
|
|
|
8
14
|
export class MaterializationController {
|
|
9
15
|
constructor(private materializationService: MaterializationService) {}
|
|
@@ -31,6 +37,7 @@ export class MaterializationController {
|
|
|
31
37
|
buildInstructions?: BuildInstruction[];
|
|
32
38
|
referenceManifest?: ManifestReference[];
|
|
33
39
|
strictUpstreams?: boolean;
|
|
40
|
+
runContext?: RunContext;
|
|
34
41
|
} {
|
|
35
42
|
const result: {
|
|
36
43
|
forceRefresh?: boolean;
|
|
@@ -38,7 +45,11 @@ export class MaterializationController {
|
|
|
38
45
|
buildInstructions?: BuildInstruction[];
|
|
39
46
|
referenceManifest?: ManifestReference[];
|
|
40
47
|
strictUpstreams?: boolean;
|
|
48
|
+
runContext?: RunContext;
|
|
41
49
|
} = {};
|
|
50
|
+
if (body.runContext !== undefined && body.runContext !== null) {
|
|
51
|
+
result.runContext = this.validateRunContext(body.runContext);
|
|
52
|
+
}
|
|
42
53
|
if (
|
|
43
54
|
body.buildInstructions !== undefined &&
|
|
44
55
|
body.buildInstructions !== null
|
|
@@ -72,6 +83,55 @@ export class MaterializationController {
|
|
|
72
83
|
return result;
|
|
73
84
|
}
|
|
74
85
|
|
|
86
|
+
/**
|
|
87
|
+
* Validate `runContext`, the caller's observability context for one run.
|
|
88
|
+
* `trigger` is a closed enum here even though it feeds a metadata property:
|
|
89
|
+
* the whole point is that a reader can group runs by how they started, which a
|
|
90
|
+
* free-form value would quietly break.
|
|
91
|
+
*
|
|
92
|
+
* Note this is NOT the service-level `trigger` the parser above deliberately
|
|
93
|
+
* refuses. That one decides whether the run counts as scheduled; this one only
|
|
94
|
+
* labels the statements the run issues, so accepting `publish` from a caller
|
|
95
|
+
* forges nothing.
|
|
96
|
+
*/
|
|
97
|
+
private validateRunContext(raw: unknown): RunContext {
|
|
98
|
+
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
|
|
99
|
+
throw new BadRequestError("runContext must be an object");
|
|
100
|
+
}
|
|
101
|
+
const obj = raw as Record<string, unknown>;
|
|
102
|
+
const context: RunContext = {};
|
|
103
|
+
if (obj.trigger !== undefined && obj.trigger !== null) {
|
|
104
|
+
if (
|
|
105
|
+
typeof obj.trigger !== "string" ||
|
|
106
|
+
!RUN_TRIGGERS.includes(obj.trigger as (typeof RUN_TRIGGERS)[number])
|
|
107
|
+
) {
|
|
108
|
+
throw new BadRequestError(
|
|
109
|
+
`runContext.trigger must be one of ${RUN_TRIGGERS.join(" | ")}`,
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
context.trigger = obj.trigger as RunContext["trigger"];
|
|
113
|
+
}
|
|
114
|
+
if (obj.runId !== undefined && obj.runId !== null) {
|
|
115
|
+
if (typeof obj.runId !== "string") {
|
|
116
|
+
throw new BadRequestError("runContext.runId must be a string");
|
|
117
|
+
}
|
|
118
|
+
// Held to the metadata contract like any other caller-supplied
|
|
119
|
+
// property: it becomes the `run_id` on every statement of the build,
|
|
120
|
+
// and a value the contract rejects would otherwise be truncated and
|
|
121
|
+
// rewritten in silence — leaving the caller with an id it cannot join
|
|
122
|
+
// on and no way to know why.
|
|
123
|
+
const violations = queryMetadataViolations({ run_id: obj.runId });
|
|
124
|
+
if (violations.length > 0) {
|
|
125
|
+
throw new BadRequestError(
|
|
126
|
+
`runContext.runId is attached to every statement as the run_id ` +
|
|
127
|
+
`property: ${violations.join("; ")}`,
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
context.runId = obj.runId;
|
|
131
|
+
}
|
|
132
|
+
return context;
|
|
133
|
+
}
|
|
134
|
+
|
|
75
135
|
/**
|
|
76
136
|
* Validate the orchestrated `buildInstructions` payload (BuildInstructions:
|
|
77
137
|
* `{ sources: BuildInstruction[], referenceManifest?, strictUpstreams? }`)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { components } from "../api";
|
|
2
2
|
import { getQueryTimeoutMs } from "../config";
|
|
3
3
|
import { ModelNotFoundError } from "../errors";
|
|
4
|
+
import { logger } from "../logger";
|
|
4
5
|
import { runWithQueryTimeout } from "../query_timeout";
|
|
5
6
|
import { EnvironmentStore } from "../service/environment_store";
|
|
6
7
|
import type { FilterParams } from "../service/filter";
|
|
@@ -133,6 +134,29 @@ export class ModelController {
|
|
|
133
134
|
bypassFilters,
|
|
134
135
|
givens,
|
|
135
136
|
abortSignal,
|
|
137
|
+
{
|
|
138
|
+
environment: environmentName,
|
|
139
|
+
// The environment owns the connection configs, so the default
|
|
140
|
+
// and enforced layers are read here rather than from the model.
|
|
141
|
+
connectionMetadata: (connectionName) => {
|
|
142
|
+
try {
|
|
143
|
+
const connection =
|
|
144
|
+
environment.getApiConnection(connectionName);
|
|
145
|
+
return {
|
|
146
|
+
default: connection.queryMetadata,
|
|
147
|
+
enforced: connection.queryMetadataEnforced,
|
|
148
|
+
};
|
|
149
|
+
} catch (error) {
|
|
150
|
+
// Fails open, and says so: what an unreadable
|
|
151
|
+
// connection costs is the enforced layer.
|
|
152
|
+
logger.debug(
|
|
153
|
+
"No query-metadata layers for connection",
|
|
154
|
+
{ connectionName, error },
|
|
155
|
+
);
|
|
156
|
+
return null;
|
|
157
|
+
}
|
|
158
|
+
},
|
|
159
|
+
},
|
|
136
160
|
),
|
|
137
161
|
getQueryTimeoutMs(),
|
|
138
162
|
);
|
|
@@ -2,8 +2,16 @@ import { validateRenderTags } from "@malloydata/render-validator";
|
|
|
2
2
|
import { components } from "../api";
|
|
3
3
|
import { getQueryTimeoutMs } from "../config";
|
|
4
4
|
import { API_PREFIX } from "../constants";
|
|
5
|
-
import { ModelNotFoundError } from "../errors";
|
|
5
|
+
import { BadRequestError, ModelNotFoundError } from "../errors";
|
|
6
6
|
import { bigIntReplacer } from "../json_utils";
|
|
7
|
+
import { logger } from "../logger";
|
|
8
|
+
import {
|
|
9
|
+
mintCorrelationId,
|
|
10
|
+
parseQueryClass,
|
|
11
|
+
parseSuppliedQueryMetadata,
|
|
12
|
+
type QueryClass,
|
|
13
|
+
type QueryMetadata,
|
|
14
|
+
} from "../service/query_metadata";
|
|
7
15
|
import { runWithQueryTimeout } from "../query_timeout";
|
|
8
16
|
import { EnvironmentStore } from "../service/environment_store";
|
|
9
17
|
import type { FilterParams } from "../service/filter";
|
|
@@ -29,7 +37,26 @@ export class QueryController {
|
|
|
29
37
|
filterParams?: FilterParams,
|
|
30
38
|
bypassFilters?: boolean,
|
|
31
39
|
givens?: Record<string, GivenValue>,
|
|
40
|
+
/**
|
|
41
|
+
* The request's per-query metadata fields, unvalidated — this controller is
|
|
42
|
+
* the boundary that turns a bad bag into a 400 rather than letting the
|
|
43
|
+
* connector refuse the statement at dispatch.
|
|
44
|
+
*/
|
|
45
|
+
metadata?: {
|
|
46
|
+
queryMetadata?: unknown;
|
|
47
|
+
queryClass?: unknown;
|
|
48
|
+
versionId?: string;
|
|
49
|
+
},
|
|
32
50
|
): Promise<ApiQuery> {
|
|
51
|
+
let requestMetadata: QueryMetadata | undefined;
|
|
52
|
+
let queryClass: QueryClass | undefined;
|
|
53
|
+
try {
|
|
54
|
+
requestMetadata = parseSuppliedQueryMetadata(metadata?.queryMetadata);
|
|
55
|
+
queryClass = parseQueryClass(metadata?.queryClass);
|
|
56
|
+
} catch (error) {
|
|
57
|
+
throw new BadRequestError((error as Error).message);
|
|
58
|
+
}
|
|
59
|
+
|
|
33
60
|
const environment = await this.environmentStore.getEnvironment(
|
|
34
61
|
environmentName,
|
|
35
62
|
false,
|
|
@@ -46,20 +73,60 @@ export class QueryController {
|
|
|
46
73
|
if (!model) {
|
|
47
74
|
throw new ModelNotFoundError(`${modelPath} does not exist`);
|
|
48
75
|
} else {
|
|
49
|
-
const {
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
76
|
+
const {
|
|
77
|
+
result,
|
|
78
|
+
compactResult,
|
|
79
|
+
rowLimit,
|
|
80
|
+
rowLimitSource,
|
|
81
|
+
queryCorrelationId,
|
|
82
|
+
} = await runWithQueryTimeout(
|
|
83
|
+
(abortSignal) =>
|
|
84
|
+
model.getQueryResults(
|
|
85
|
+
sourceName,
|
|
86
|
+
queryName,
|
|
87
|
+
query,
|
|
88
|
+
filterParams,
|
|
89
|
+
bypassFilters,
|
|
90
|
+
givens,
|
|
91
|
+
abortSignal,
|
|
92
|
+
{
|
|
93
|
+
request: requestMetadata,
|
|
94
|
+
queryClass,
|
|
95
|
+
environment: environmentName,
|
|
96
|
+
// Always undefined today: the route 501s any versionId
|
|
97
|
+
// before this runs. Wired so that lifting that rejection
|
|
98
|
+
// is the whole change.
|
|
99
|
+
version: metadata?.versionId,
|
|
100
|
+
// Minted here because this is the boundary that returns
|
|
101
|
+
// it; a path with nowhere to put it does not mint one.
|
|
102
|
+
correlationId: mintCorrelationId(),
|
|
103
|
+
// The environment owns the connection configs, so the
|
|
104
|
+
// default and enforced layers are read here rather than
|
|
105
|
+
// from the model.
|
|
106
|
+
connectionMetadata: (connectionName) => {
|
|
107
|
+
try {
|
|
108
|
+
const connection =
|
|
109
|
+
environment.getApiConnection(connectionName);
|
|
110
|
+
return {
|
|
111
|
+
default: connection.queryMetadata,
|
|
112
|
+
enforced: connection.queryMetadataEnforced,
|
|
113
|
+
};
|
|
114
|
+
} catch (error) {
|
|
115
|
+
// Failing open is right — a tag must not fail a
|
|
116
|
+
// query — but this is the one drop with no metric
|
|
117
|
+
// behind it, and what it costs is the ENFORCED
|
|
118
|
+
// layer. Log it so it is diagnosable.
|
|
119
|
+
logger.debug(
|
|
120
|
+
"No query-metadata layers for connection",
|
|
121
|
+
{ connectionName, error },
|
|
122
|
+
);
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
},
|
|
126
|
+
},
|
|
127
|
+
),
|
|
128
|
+
getQueryTimeoutMs(),
|
|
129
|
+
);
|
|
63
130
|
const renderLogs = validateRenderTags(result);
|
|
64
131
|
return {
|
|
65
132
|
result: compactJson
|
|
@@ -78,6 +145,7 @@ export class QueryController {
|
|
|
78
145
|
// deliberate `limit:`/`top:` from the silently-applied default, and
|
|
79
146
|
// so cannot reproduce the MCP envelope's _limit_hit.
|
|
80
147
|
queryRowLimitSource: rowLimitSource,
|
|
148
|
+
queryCorrelationId,
|
|
81
149
|
} as ApiQuery;
|
|
82
150
|
}
|
|
83
151
|
}
|
package/src/mcp/handler_utils.ts
CHANGED
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
type ErrorDetails,
|
|
20
20
|
} from "./error_messages";
|
|
21
21
|
import type { Model } from "../service/model";
|
|
22
|
+
import type { Environment } from "../service/environment";
|
|
22
23
|
import { logger } from "../logger";
|
|
23
24
|
|
|
24
25
|
/**
|
|
@@ -116,6 +117,11 @@ export function classifyToolError(
|
|
|
116
117
|
/**
|
|
117
118
|
* Fetches and validates the Package and Model instances needed for query execution.
|
|
118
119
|
* Handles errors related to package/model access and initial compilation.
|
|
120
|
+
*
|
|
121
|
+
* The resolved Environment comes back with the model because a query needs it for
|
|
122
|
+
* more than the lookup: it owns the connection configs the per-query metadata
|
|
123
|
+
* layers are read from.
|
|
124
|
+
*
|
|
119
125
|
* @returns An object containing the Model instance or a pre-formatted ErrorDetails object.
|
|
120
126
|
*/
|
|
121
127
|
export async function getModelForQuery(
|
|
@@ -123,7 +129,9 @@ export async function getModelForQuery(
|
|
|
123
129
|
environmentName: string,
|
|
124
130
|
packageName: string,
|
|
125
131
|
modelPath: string,
|
|
126
|
-
): Promise<
|
|
132
|
+
): Promise<
|
|
133
|
+
{ model: Model; environment: Environment } | { error: ErrorDetails }
|
|
134
|
+
> {
|
|
127
135
|
try {
|
|
128
136
|
const environment = await environmentStore.getEnvironment(
|
|
129
137
|
environmentName,
|
|
@@ -143,7 +151,7 @@ export async function getModelForQuery(
|
|
|
143
151
|
}
|
|
144
152
|
// Attempt to get the model definition early to catch initial compilation errors
|
|
145
153
|
await model.getModel(); // This might throw ModelCompilationError
|
|
146
|
-
return { model };
|
|
154
|
+
return { model, environment };
|
|
147
155
|
} catch (error) {
|
|
148
156
|
// Handle errors during package/model access or initial compilation
|
|
149
157
|
let errorDetails: ErrorDetails;
|
|
@@ -65,6 +65,14 @@ export interface QueryEnvelope {
|
|
|
65
65
|
_rows_truncated?: boolean;
|
|
66
66
|
_total_rows?: number;
|
|
67
67
|
_returned_rows?: number;
|
|
68
|
+
/**
|
|
69
|
+
* The `query_id` property attached to this query's statements: the join key
|
|
70
|
+
* into the backend's own query record (`QUERY_HISTORY`, `JOBS.labels`, the
|
|
71
|
+
* statement text). Publisher-side and new to both products, like the two
|
|
72
|
+
* `_limit_...` fields. Absent when nothing was attached, which is every query
|
|
73
|
+
* on a deployment that has not enabled query metadata.
|
|
74
|
+
*/
|
|
75
|
+
_query_id?: string;
|
|
68
76
|
warning?: string;
|
|
69
77
|
renderLogErrors?: string[];
|
|
70
78
|
}
|
|
@@ -97,6 +105,7 @@ export function buildQueryEnvelope(
|
|
|
97
105
|
renderLogErrors: string[] = [],
|
|
98
106
|
limit = MAX_RESULT_CHARS,
|
|
99
107
|
rowLimitSource: QueryRowLimitSource = "server_default",
|
|
108
|
+
queryCorrelationId: string | null = null,
|
|
100
109
|
): QueryEnvelope {
|
|
101
110
|
const rowCount = Array.isArray(rows) ? rows.length : 0;
|
|
102
111
|
// Equality, not >=: the cap is pushed into the SQL, so the database cannot
|
|
@@ -132,6 +141,7 @@ export function buildQueryEnvelope(
|
|
|
132
141
|
_query_row_limit: rowLimit,
|
|
133
142
|
_limit_source: rowLimitSource,
|
|
134
143
|
_limit_hit: limitHit,
|
|
144
|
+
...(queryCorrelationId !== null && { _query_id: queryCorrelationId }),
|
|
135
145
|
...(renderLogErrors.length > 0 && { renderLogErrors }),
|
|
136
146
|
};
|
|
137
147
|
|