@malloy-publisher/server 0.0.233 → 0.0.235
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.docker.md +1 -0
- package/dist/app/api-doc.yaml +208 -0
- package/dist/app/assets/{EnvironmentPage-DutP7T8h.js → EnvironmentPage-BsAnavYN.js} +1 -1
- package/dist/app/assets/{HomePage-BcxDrBfl.js → HomePage-CADE138j.js} +1 -1
- package/dist/app/assets/{LightMode-BJukGxgz.js → LightMode-Cfh7KzN8.js} +1 -1
- package/dist/app/assets/{MainPage-DXbwlMeF.js → MainPage-CO3pRlnV.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-BBQksmTU.js → MaterializationsPage-p9YjkRXZ.js} +1 -1
- package/dist/app/assets/{ModelPage-C6tK51uU.js → ModelPage-C1OSTv-x.js} +1 -1
- package/dist/app/assets/{PackagePage-Bo3cwwZE.js → PackagePage-e4kN75YR.js} +1 -1
- package/dist/app/assets/{RouteError-BufkcAKE.js → RouteError-CzbfOkng.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-DICvvKpa.js → ThemeEditorPage-CciagFTq.js} +1 -1
- package/dist/app/assets/{WorkbookPage-Dkwt75Nj.js → WorkbookPage-DNWmkCXa.js} +1 -1
- package/dist/app/assets/{core-C0nunIQT.es-DlMLKZBK.js → core-Rj_4rRnA.es-BZyvITuO.js} +1 -1
- package/dist/app/assets/{index-CmEVVe-8.js → index-CH2AcDzc.js} +4 -4
- package/dist/app/assets/{index-qnhU9CGo.js → index-DQa463gC.js} +2 -2
- package/dist/app/assets/{index-Cs4WVm2z.js → index-DQpV7MyA.js} +1 -1
- package/dist/app/assets/{index-BabP-V-S.js → index-DzaYbhnD.js} +1 -1
- package/dist/app/assets/{index-BusxL5Pt.js → index-VBbcc8s6.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/package_load_worker.mjs +53 -3
- package/dist/server.mjs +835 -97
- package/package.json +12 -12
- package/src/config.ts +35 -1
- package/src/controller/connection.controller.spec.ts +46 -0
- package/src/controller/connection.controller.ts +105 -2
- package/src/controller/materialization.controller.spec.ts +25 -0
- package/src/controller/materialization.controller.ts +60 -0
- package/src/controller/model.controller.ts +24 -0
- package/src/controller/query.controller.ts +83 -15
- package/src/mcp/handler_utils.ts +10 -2
- package/src/mcp/query_envelope.ts +10 -0
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/tools/execute_query_tool.spec.ts +131 -0
- package/src/mcp/tools/execute_query_tool.ts +62 -25
- package/src/mcp_config.spec.ts +919 -0
- package/src/mcp_config.ts +425 -0
- package/src/oom_guards.integration.spec.ts +11 -3
- package/src/package_load/package_load_pool.ts +2 -0
- package/src/package_load/package_load_worker.ts +17 -5
- package/src/package_load/protocol.ts +6 -0
- package/src/query_metadata_metrics.ts +49 -0
- package/src/server.ts +99 -3
- package/src/service/build_plan.spec.ts +125 -0
- package/src/service/build_plan.ts +108 -7
- package/src/service/connection_config.ts +49 -0
- package/src/service/environment.ts +57 -3
- package/src/service/materialization_config_validation.spec.ts +99 -0
- package/src/service/materialization_config_validation.ts +120 -0
- package/src/service/materialization_schedule_surface.spec.ts +124 -0
- package/src/service/materialization_service.spec.ts +119 -0
- package/src/service/materialization_service.ts +186 -3
- package/src/service/materialization_test_fixtures.ts +86 -21
- package/src/service/model.spec.ts +45 -1
- package/src/service/model.ts +145 -19
- package/src/service/package.ts +24 -1
- package/src/service/package_manifest.spec.ts +137 -4
- package/src/service/package_manifest.ts +140 -5
- package/src/service/persist_annotation_validation.spec.ts +12 -0
- package/src/service/persist_annotation_validation.ts +9 -4
- package/src/service/query_metadata.spec.ts +408 -0
- package/src/service/query_metadata.ts +492 -0
- package/src/service/query_metadata_identity.spec.ts +149 -0
|
@@ -18,10 +18,10 @@ export const PACKAGE_SCOPES = ["version", "package"] as const;
|
|
|
18
18
|
export type PackageScope = (typeof PACKAGE_SCOPES)[number];
|
|
19
19
|
|
|
20
20
|
/**
|
|
21
|
-
* Read
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
21
|
+
* Read a `scope` value, defaulting to `"package"` when absent or null. Any other
|
|
22
|
+
* value is a manifest error: scope is load-bearing (it decides version-owned vs
|
|
23
|
+
* cross-version reuse), so a typo must fail loudly rather than silently pick a
|
|
24
|
+
* default. Throws on an invalid value.
|
|
25
25
|
*/
|
|
26
26
|
export function parsePackageScope(raw: unknown): PackageScope {
|
|
27
27
|
if (raw === undefined || raw === null) {
|
|
@@ -36,6 +36,69 @@ export function parsePackageScope(raw: unknown): PackageScope {
|
|
|
36
36
|
);
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
+
/**
|
|
40
|
+
* Resolve the package's scope from its two possible homes: `materialization.scope`
|
|
41
|
+
* (canonical — every other build-behavioral knob lives in that block) and the
|
|
42
|
+
* manifest root (the original home, deprecated).
|
|
43
|
+
*
|
|
44
|
+
* The root form keeps working because scope rides the published artifact: a
|
|
45
|
+
* package published before the move must keep parsing until it is republished.
|
|
46
|
+
* Declaring DIFFERENT values in the two homes throws, because scope decides
|
|
47
|
+
* whether an artifact is version-owned and guessing which one the author meant
|
|
48
|
+
* could reuse a table across versions that was never supposed to be shared.
|
|
49
|
+
*
|
|
50
|
+
* Only a root-ONLY declaration is deprecated. Both homes agreeing is the
|
|
51
|
+
* transition state the server itself writes (see `writePackageManifest`): the
|
|
52
|
+
* envelope for this build, the root for an older publisher that would otherwise
|
|
53
|
+
* silently default to `package`. Warning about that would be warning an operator
|
|
54
|
+
* about something the server did on their behalf and they cannot fix.
|
|
55
|
+
*/
|
|
56
|
+
export function resolvePackageScope(
|
|
57
|
+
rootRaw: unknown,
|
|
58
|
+
materializationRaw: unknown,
|
|
59
|
+
): { scope: PackageScope; warnings: string[] } {
|
|
60
|
+
const envelopeRaw =
|
|
61
|
+
materializationRaw && typeof materializationRaw === "object"
|
|
62
|
+
? (materializationRaw as { scope?: unknown }).scope
|
|
63
|
+
: undefined;
|
|
64
|
+
const rootDeclared = rootRaw !== undefined && rootRaw !== null;
|
|
65
|
+
const envelopeDeclared = envelopeRaw !== undefined && envelopeRaw !== null;
|
|
66
|
+
|
|
67
|
+
// Validate both homes before comparing, so a typo is reported as a typo
|
|
68
|
+
// rather than as a conflict.
|
|
69
|
+
const rootScope = parsePackageScope(rootRaw);
|
|
70
|
+
const envelopeScope = parsePackageScope(envelopeRaw);
|
|
71
|
+
|
|
72
|
+
if (rootDeclared && envelopeDeclared) {
|
|
73
|
+
if (rootScope !== envelopeScope) {
|
|
74
|
+
// Names the fix, because this throw fails the package LOAD: the
|
|
75
|
+
// package is skipped and its only trace is /status loadErrors, so the
|
|
76
|
+
// message is the whole diagnosis. Guessing instead would be worse —
|
|
77
|
+
// picking the wrong one reuses a table across versions that was never
|
|
78
|
+
// meant to be shared, silently.
|
|
79
|
+
throw new Error(
|
|
80
|
+
`Conflicting "scope" in publisher.json: root "${rootScope}" vs ` +
|
|
81
|
+
`"materialization.scope" "${envelopeScope}". The package cannot ` +
|
|
82
|
+
`load until they agree. Edit "materialization": { "scope": ... } ` +
|
|
83
|
+
`to the value you want and delete the root-level "scope" (the ` +
|
|
84
|
+
`server rewrites both homes on its next manifest write).`,
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
return { scope: envelopeScope, warnings: [] };
|
|
88
|
+
}
|
|
89
|
+
if (rootDeclared) {
|
|
90
|
+
return { scope: rootScope, warnings: [SCOPE_ROOT_DEPRECATION] };
|
|
91
|
+
}
|
|
92
|
+
return { scope: envelopeScope, warnings: [] };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const SCOPE_ROOT_DEPRECATION =
|
|
96
|
+
`"scope" at the manifest root is deprecated: declare it as ` +
|
|
97
|
+
`"materialization": { "scope": ... } alongside the other build knobs. The ` +
|
|
98
|
+
`root form still works and will be removed in a future release; until then ` +
|
|
99
|
+
`the server keeps both homes in sync when it writes the manifest, so an ` +
|
|
100
|
+
`older publisher still reads the right value.`;
|
|
101
|
+
|
|
39
102
|
/**
|
|
40
103
|
* The manifest's `materialization.freshness` block, surfaced verbatim for the
|
|
41
104
|
* control plane (which owns the scheduling and query-time gating logic).
|
|
@@ -63,6 +126,18 @@ export interface PackageMaterializationConfig {
|
|
|
63
126
|
* "no policy" from "policy with no valid fields".
|
|
64
127
|
*/
|
|
65
128
|
freshness: PackageFreshnessConfig | null;
|
|
129
|
+
/**
|
|
130
|
+
* Package-level per-query metadata: the least specific model-side layer,
|
|
131
|
+
* overridden per property by a model-file, source or per-request declaration.
|
|
132
|
+
* Null when the manifest declares none.
|
|
133
|
+
*
|
|
134
|
+
* Kept VERBATIM (every string-valued property, conforming or not) rather than
|
|
135
|
+
* filtered to what Malloy will accept. A property that violates the contract
|
|
136
|
+
* has to be visible somewhere to be fixable: publish reports it as a warning
|
|
137
|
+
* from this block, and the runtime clamps it with a metric. Filtering here
|
|
138
|
+
* would make an author's typo disappear with nothing to point at.
|
|
139
|
+
*/
|
|
140
|
+
queryMetadata: Record<string, string> | null;
|
|
66
141
|
}
|
|
67
142
|
|
|
68
143
|
function parseFreshness(raw: unknown): PackageFreshnessConfig | null {
|
|
@@ -83,11 +158,55 @@ function parseFreshness(raw: unknown): PackageFreshnessConfig | null {
|
|
|
83
158
|
return freshness;
|
|
84
159
|
}
|
|
85
160
|
|
|
161
|
+
/**
|
|
162
|
+
* The `materialization.queryMetadata` block: string-valued properties verbatim.
|
|
163
|
+
* A non-object block, or one with no string values, degrades to null so absence
|
|
164
|
+
* on the wire always means "declared nothing usable". A non-string value is
|
|
165
|
+
* dropped here because it cannot round-trip as a property at all; contract
|
|
166
|
+
* violations of the string values are deliberately NOT dropped (see
|
|
167
|
+
* {@link PackageMaterializationConfig.queryMetadata}).
|
|
168
|
+
*
|
|
169
|
+
* Each dropped value is reported, because the drop happens BEFORE the config
|
|
170
|
+
* validation that would otherwise name it: an unquoted `"team": 123` is a
|
|
171
|
+
* plausible hand-edit, and without this it disappears with nothing anywhere to
|
|
172
|
+
* point at.
|
|
173
|
+
*/
|
|
174
|
+
function parseQueryMetadata(raw: unknown): {
|
|
175
|
+
metadata: Record<string, string> | null;
|
|
176
|
+
warnings: string[];
|
|
177
|
+
} {
|
|
178
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
|
179
|
+
return { metadata: null, warnings: [] };
|
|
180
|
+
}
|
|
181
|
+
const out: Record<string, string> = {};
|
|
182
|
+
const warnings: string[] = [];
|
|
183
|
+
for (const [name, value] of Object.entries(raw as Record<string, unknown>)) {
|
|
184
|
+
if (typeof value === "string") {
|
|
185
|
+
out[name] = value;
|
|
186
|
+
} else {
|
|
187
|
+
warnings.push(
|
|
188
|
+
`materialization.queryMetadata: property '${name}' must be a ` +
|
|
189
|
+
`string (got ${value === null ? "null" : typeof value}); it is ` +
|
|
190
|
+
`not attached to any statement`,
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
return {
|
|
195
|
+
metadata: Object.keys(out).length > 0 ? out : null,
|
|
196
|
+
warnings,
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
|
|
86
200
|
/**
|
|
87
201
|
* Read the manifest's `materialization` object, keeping only recognized fields.
|
|
88
202
|
* Returns null when the block is absent so the API field is null rather than an
|
|
89
203
|
* empty object; a non-string schedule degrades to null, and an absent or
|
|
90
204
|
* non-object freshness degrades to null.
|
|
205
|
+
*
|
|
206
|
+
* `materialization.scope` is deliberately NOT read here: scope is a package-level
|
|
207
|
+
* mode rather than a layered knob, and it has two possible homes to reconcile —
|
|
208
|
+
* see {@link resolvePackageScope}, which owns that read and surfaces the
|
|
209
|
+
* resolved value on the package itself.
|
|
91
210
|
*/
|
|
92
211
|
export function parsePackageMaterialization(
|
|
93
212
|
raw: unknown,
|
|
@@ -95,12 +214,28 @@ export function parsePackageMaterialization(
|
|
|
95
214
|
if (!raw || typeof raw !== "object") {
|
|
96
215
|
return null;
|
|
97
216
|
}
|
|
98
|
-
const { schedule, freshness } = raw as {
|
|
217
|
+
const { schedule, freshness, queryMetadata } = raw as {
|
|
99
218
|
schedule?: unknown;
|
|
100
219
|
freshness?: unknown;
|
|
220
|
+
queryMetadata?: unknown;
|
|
101
221
|
};
|
|
102
222
|
return {
|
|
103
223
|
schedule: typeof schedule === "string" ? schedule : null,
|
|
104
224
|
freshness: parseFreshness(freshness),
|
|
225
|
+
queryMetadata: parseQueryMetadata(queryMetadata).metadata,
|
|
105
226
|
};
|
|
106
227
|
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* What the `materialization` parse tolerated but could not keep, for the
|
|
231
|
+
* operator warnings array. Reads the same parse as
|
|
232
|
+
* {@link parsePackageMaterialization} rather than re-deriving it, so the two
|
|
233
|
+
* cannot disagree about what was dropped.
|
|
234
|
+
*/
|
|
235
|
+
export function packageMaterializationWarnings(raw: unknown): string[] {
|
|
236
|
+
if (!raw || typeof raw !== "object") {
|
|
237
|
+
return [];
|
|
238
|
+
}
|
|
239
|
+
const { queryMetadata } = raw as { queryMetadata?: unknown };
|
|
240
|
+
return parseQueryMetadata(queryMetadata).warnings;
|
|
241
|
+
}
|
|
@@ -66,6 +66,18 @@ describe("assertPersistNamesQuoted", () => {
|
|
|
66
66
|
).not.toThrow();
|
|
67
67
|
});
|
|
68
68
|
|
|
69
|
+
it("does not mistake a DOTTED key for the name field", () => {
|
|
70
|
+
// `.` is a word boundary, so a `\b`-anchored pattern reads
|
|
71
|
+
// `queryMetadata.name=` as the persist name and fails the whole model
|
|
72
|
+
// load with a 424 about quoting a name the author never declared.
|
|
73
|
+
expect(() =>
|
|
74
|
+
assertPersistNamesQuoted(
|
|
75
|
+
`#@ persist name="bar" queryMetadata.name=finance`,
|
|
76
|
+
"m.malloy",
|
|
77
|
+
),
|
|
78
|
+
).not.toThrow();
|
|
79
|
+
});
|
|
80
|
+
|
|
69
81
|
it("reports every offending annotation in the message", () => {
|
|
70
82
|
expect(() =>
|
|
71
83
|
assertPersistNamesQuoted(
|
|
@@ -5,11 +5,16 @@ const PERSIST_LINE_PATTERN = /^\s*#@\s+persist\b/;
|
|
|
5
5
|
/**
|
|
6
6
|
* A `name=` key whose value is NOT immediately opened by a single or double
|
|
7
7
|
* quote — i.e. a bare `name=engaged_events` (whitespace around `=` tolerated).
|
|
8
|
-
* `name="..."` and `name='...'` pass.
|
|
9
|
-
*
|
|
10
|
-
*
|
|
8
|
+
* `name="..."` and `name='...'` pass.
|
|
9
|
+
*
|
|
10
|
+
* The lookbehind requires a standalone key: neither an `_`-joined neighbour
|
|
11
|
+
* (`tablename=`, `realization_name=`) nor a DOTTED one (`queryMetadata.name=`)
|
|
12
|
+
* is the persist name. A `\b` word boundary would catch the underscore case but
|
|
13
|
+
* not the dotted one, because `.` is itself a boundary — so a legitimate
|
|
14
|
+
* `#@ persist queryMetadata.name=finance` would fail the load with a 424 about
|
|
15
|
+
* quoting a persist name it never declared.
|
|
11
16
|
*/
|
|
12
|
-
const UNQUOTED_NAME_PATTERN =
|
|
17
|
+
const UNQUOTED_NAME_PATTERN = /(?<![.\w])name\s*=\s*(?!["'])/;
|
|
13
18
|
|
|
14
19
|
/**
|
|
15
20
|
* Reject `#@ persist name=<value>` annotations whose name is unquoted.
|
|
@@ -0,0 +1,408 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
|
2
|
+
import {
|
|
3
|
+
CONTEXT_SHED_ORDER,
|
|
4
|
+
MAX_PROPERTIES,
|
|
5
|
+
MAX_PROPERTY_VALUE_LENGTH,
|
|
6
|
+
mergeQueryMetadata,
|
|
7
|
+
mintCorrelationId,
|
|
8
|
+
parseQueryClass,
|
|
9
|
+
parseSuppliedQueryMetadata,
|
|
10
|
+
queryContextProperties,
|
|
11
|
+
RESERVED_CONTEXT_PROPERTIES,
|
|
12
|
+
queryMetadataAdvisoryWarnings,
|
|
13
|
+
queryMetadataBudgetWarning,
|
|
14
|
+
queryMetadataViolations,
|
|
15
|
+
} from "./query_metadata";
|
|
16
|
+
|
|
17
|
+
const originalMode = process.env.PUBLISHER_QUERY_METADATA;
|
|
18
|
+
|
|
19
|
+
// The feature ships dark, so every test of the resolution logic has to turn it
|
|
20
|
+
// on. The one test that asserts what `off` does sets it back for itself.
|
|
21
|
+
beforeEach(() => {
|
|
22
|
+
process.env.PUBLISHER_QUERY_METADATA = "on";
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
afterEach(() => {
|
|
26
|
+
if (originalMode === undefined) {
|
|
27
|
+
delete process.env.PUBLISHER_QUERY_METADATA;
|
|
28
|
+
} else {
|
|
29
|
+
process.env.PUBLISHER_QUERY_METADATA = originalMode;
|
|
30
|
+
}
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
describe("queryMetadataViolations", () => {
|
|
34
|
+
it("accepts a conforming bag", () => {
|
|
35
|
+
expect(
|
|
36
|
+
queryMetadataViolations({ team: "finance", run_id: "abc-123" }),
|
|
37
|
+
).toEqual([]);
|
|
38
|
+
expect(queryMetadataViolations(undefined)).toEqual([]);
|
|
39
|
+
expect(queryMetadataViolations(null)).toEqual([]);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it("rejects a non-object", () => {
|
|
43
|
+
expect(queryMetadataViolations("team=finance")).toHaveLength(1);
|
|
44
|
+
expect(queryMetadataViolations([1, 2])).toHaveLength(1);
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
it("rejects a property name outside the contract", () => {
|
|
48
|
+
// A dot is the one an author reaches for first, and Malloy throws on it at
|
|
49
|
+
// dispatch — so it has to come back as a message here.
|
|
50
|
+
expect(queryMetadataViolations({ "team.name": "finance" })).toHaveLength(
|
|
51
|
+
1,
|
|
52
|
+
);
|
|
53
|
+
expect(queryMetadataViolations({ "team-name": "finance" })).toHaveLength(
|
|
54
|
+
1,
|
|
55
|
+
);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it("rejects a value that cannot be rendered", () => {
|
|
59
|
+
expect(queryMetadataViolations({ team: 'fin"ance' })).toHaveLength(1);
|
|
60
|
+
expect(queryMetadataViolations({ team: "fin\nance" })).toHaveLength(1);
|
|
61
|
+
expect(queryMetadataViolations({ team: "финансы" })).toHaveLength(1);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
it("rejects a non-string value, naming the property", () => {
|
|
65
|
+
const problems = queryMetadataViolations({ retries: 3 });
|
|
66
|
+
expect(problems).toHaveLength(1);
|
|
67
|
+
expect(problems[0]).toContain("retries");
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
it("rejects an over-long value and too many properties", () => {
|
|
71
|
+
expect(
|
|
72
|
+
queryMetadataViolations({
|
|
73
|
+
team: "x".repeat(MAX_PROPERTY_VALUE_LENGTH + 1),
|
|
74
|
+
}),
|
|
75
|
+
).toHaveLength(1);
|
|
76
|
+
const tooMany: Record<string, string> = {};
|
|
77
|
+
for (let i = 0; i <= MAX_PROPERTIES; i++) tooMany[`p${i}`] = "v";
|
|
78
|
+
expect(queryMetadataViolations(tooMany)).toHaveLength(1);
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
describe("queryMetadataAdvisoryWarnings", () => {
|
|
83
|
+
it("flags a name BigQuery will silently drop", () => {
|
|
84
|
+
// Legal per the contract, kept by every other backend, gone on BigQuery.
|
|
85
|
+
expect(queryMetadataAdvisoryWarnings({ _team: "finance" })).toHaveLength(
|
|
86
|
+
1,
|
|
87
|
+
);
|
|
88
|
+
expect(
|
|
89
|
+
queryMetadataAdvisoryWarnings({ "2team": "finance" }),
|
|
90
|
+
).toHaveLength(1);
|
|
91
|
+
expect(queryMetadataAdvisoryWarnings({ team2: "finance" })).toEqual([]);
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
describe("queryContextProperties", () => {
|
|
96
|
+
it("emits only the fields that are set", () => {
|
|
97
|
+
expect(
|
|
98
|
+
queryContextProperties({
|
|
99
|
+
queryClass: "materialize",
|
|
100
|
+
environment: "prod",
|
|
101
|
+
package: "sales",
|
|
102
|
+
source: "order_rollup",
|
|
103
|
+
trigger: "scheduler",
|
|
104
|
+
runId: "run-7",
|
|
105
|
+
correlationId: "q-1",
|
|
106
|
+
}),
|
|
107
|
+
).toEqual({
|
|
108
|
+
class: "materialize",
|
|
109
|
+
environment: "prod",
|
|
110
|
+
package: "sales",
|
|
111
|
+
source: "order_rollup",
|
|
112
|
+
trigger: "scheduler",
|
|
113
|
+
run_id: "run-7",
|
|
114
|
+
query_id: "q-1",
|
|
115
|
+
});
|
|
116
|
+
expect(queryContextProperties({})).toEqual({});
|
|
117
|
+
});
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
describe("queryMetadataBudgetWarning", () => {
|
|
121
|
+
it("warns when a declaration leaves no room for the server context", () => {
|
|
122
|
+
// 20 is the size of the whole bag, and the server adds its own on top, so
|
|
123
|
+
// a declaration that fills the contract quietly loses properties later.
|
|
124
|
+
expect(queryMetadataBudgetWarning(MAX_PROPERTIES)).toContain(
|
|
125
|
+
"the server adds up to",
|
|
126
|
+
);
|
|
127
|
+
expect(queryMetadataBudgetWarning(1)).toBeUndefined();
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
it("is a count, not a bag, so a caller can sum the maps that share the budget", () => {
|
|
131
|
+
// A connection declares a default AND an enforced map; 6 + 6 is over
|
|
132
|
+
// budget while neither map is.
|
|
133
|
+
const authorBudget = MAX_PROPERTIES - RESERVED_CONTEXT_PROPERTIES;
|
|
134
|
+
expect(queryMetadataBudgetWarning(authorBudget)).toBeUndefined();
|
|
135
|
+
expect(queryMetadataBudgetWarning(authorBudget + 1)).toBeDefined();
|
|
136
|
+
});
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
describe("CONTEXT_SHED_ORDER", () => {
|
|
140
|
+
it("covers every context property exactly once", () => {
|
|
141
|
+
// The shed order also sizes the author budget the declaration warning
|
|
142
|
+
// uses, so a context property added to one list and not the other would
|
|
143
|
+
// quietly move that threshold.
|
|
144
|
+
const every = queryContextProperties({
|
|
145
|
+
queryClass: "interactive",
|
|
146
|
+
environment: "e",
|
|
147
|
+
package: "p",
|
|
148
|
+
version: "v",
|
|
149
|
+
model: "m.malloy",
|
|
150
|
+
source: "s",
|
|
151
|
+
trigger: "publish",
|
|
152
|
+
runId: "r",
|
|
153
|
+
correlationId: "q",
|
|
154
|
+
});
|
|
155
|
+
expect([...CONTEXT_SHED_ORDER].sort()).toEqual(Object.keys(every).sort());
|
|
156
|
+
expect(RESERVED_CONTEXT_PROPERTIES).toBe(Object.keys(every).length);
|
|
157
|
+
});
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
describe("mintCorrelationId", () => {
|
|
161
|
+
it("mints a distinct id per call", () => {
|
|
162
|
+
const first = mintCorrelationId();
|
|
163
|
+
expect(first).toMatch(/^[0-9a-f-]{36}$/);
|
|
164
|
+
expect(mintCorrelationId()).not.toBe(first);
|
|
165
|
+
});
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
describe("mergeQueryMetadata", () => {
|
|
169
|
+
it("returns nothing when every layer is empty", () => {
|
|
170
|
+
expect(mergeQueryMetadata({})).toEqual({ drops: [] });
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
it("merges most-specific-wins per property", () => {
|
|
174
|
+
const resolved = mergeQueryMetadata({
|
|
175
|
+
connection: { team: "finance", tier: "bronze", deployment: "eu" },
|
|
176
|
+
model: { tier: "silver", surface: "marts" },
|
|
177
|
+
request: { tier: "gold" },
|
|
178
|
+
});
|
|
179
|
+
expect(resolved.metadata).toEqual({
|
|
180
|
+
team: "finance",
|
|
181
|
+
tier: "gold",
|
|
182
|
+
surface: "marts",
|
|
183
|
+
deployment: "eu",
|
|
184
|
+
});
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
it("lets context win over an author property of the same name", () => {
|
|
188
|
+
// Otherwise a caller could label its own interactive query as a build and
|
|
189
|
+
// corrupt exactly the attribution this feature exists to produce.
|
|
190
|
+
const resolved = mergeQueryMetadata({
|
|
191
|
+
request: { class: "materialize", package: "not-this-one" },
|
|
192
|
+
context: { queryClass: "interactive", package: "sales" },
|
|
193
|
+
});
|
|
194
|
+
expect(resolved.metadata).toEqual({
|
|
195
|
+
class: "interactive",
|
|
196
|
+
package: "sales",
|
|
197
|
+
});
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
it("reserves query_id: a caller cannot supply the correlation key", () => {
|
|
201
|
+
// The response hands this back as the join key, so a caller-owned value
|
|
202
|
+
// would let two calls claim the same id.
|
|
203
|
+
const resolved = mergeQueryMetadata({
|
|
204
|
+
request: { query_id: "mine" },
|
|
205
|
+
context: { correlationId: "server-minted" },
|
|
206
|
+
});
|
|
207
|
+
expect(resolved.metadata?.query_id).toBe("server-minted");
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
it("lets an enforced property beat every declaration", () => {
|
|
211
|
+
// The connection API is administered; a package annotation and a request
|
|
212
|
+
// are not. A tenant must not be able to relabel the traffic its host is
|
|
213
|
+
// billed for.
|
|
214
|
+
const resolved = mergeQueryMetadata({
|
|
215
|
+
connection: { org: "acme" },
|
|
216
|
+
model: { org: "not_acme" },
|
|
217
|
+
request: { org: "definitely_not_acme" },
|
|
218
|
+
enforced: { org: "acme" },
|
|
219
|
+
});
|
|
220
|
+
expect(resolved.metadata?.org).toBe("acme");
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
it("still lets the server's own context beat an enforced property", () => {
|
|
224
|
+
// Context describes what the server is actually doing; an admin cannot
|
|
225
|
+
// declare a query into a different class.
|
|
226
|
+
const resolved = mergeQueryMetadata({
|
|
227
|
+
enforced: { class: "interactive", org: "acme" },
|
|
228
|
+
context: { queryClass: "materialize" },
|
|
229
|
+
});
|
|
230
|
+
expect(resolved.metadata).toEqual({ class: "materialize", org: "acme" });
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
it("sheds enforced properties after every declared one", () => {
|
|
234
|
+
// The point of enforcing a property is that it is still there at the end:
|
|
235
|
+
// twenty declared properties must not be able to push it out of the bag.
|
|
236
|
+
const request: Record<string, string> = {};
|
|
237
|
+
for (let i = 0; i < MAX_PROPERTIES; i++) request[`p${i}`] = "v";
|
|
238
|
+
const resolved = mergeQueryMetadata({
|
|
239
|
+
request,
|
|
240
|
+
enforced: { org: "acme" },
|
|
241
|
+
context: { queryClass: "interactive" },
|
|
242
|
+
});
|
|
243
|
+
expect(resolved.metadata?.org).toBe("acme");
|
|
244
|
+
expect(resolved.metadata?.class).toBe("interactive");
|
|
245
|
+
expect(resolved.drops.every((d) => d.name.startsWith("p"))).toBe(true);
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
it("keeps a property named __proto__, which the contract allows", () => {
|
|
249
|
+
// `__proto__` passes the name rule, so it reaches the merge — and on an
|
|
250
|
+
// object literal the assignment hits the prototype setter, which ignores
|
|
251
|
+
// a string and drops the property with no drop record and no metric. It
|
|
252
|
+
// is the one silent loss in a module whose whole argument is that a lost
|
|
253
|
+
// property is always accounted for.
|
|
254
|
+
// JSON.parse, not a literal: `__proto__:` in source sets the prototype,
|
|
255
|
+
// while a parsed request body carries it as an own property — which is
|
|
256
|
+
// how it arrives here.
|
|
257
|
+
const resolved = mergeQueryMetadata({
|
|
258
|
+
request: JSON.parse('{"__proto__":"surprise","team":"finance"}'),
|
|
259
|
+
});
|
|
260
|
+
expect(Object.keys(resolved.metadata ?? {}).sort()).toEqual([
|
|
261
|
+
"__proto__",
|
|
262
|
+
"team",
|
|
263
|
+
]);
|
|
264
|
+
expect(resolved.drops).toEqual([]);
|
|
265
|
+
// A string property, not a reshaped bag.
|
|
266
|
+
expect(JSON.stringify(resolved.metadata)).toContain('"__proto__"');
|
|
267
|
+
});
|
|
268
|
+
|
|
269
|
+
it("drops a property whose name violates the contract", () => {
|
|
270
|
+
// Never throws: it would fail the customer's query at dispatch.
|
|
271
|
+
const resolved = mergeQueryMetadata({
|
|
272
|
+
request: { "team.name": "finance", team: "finance" },
|
|
273
|
+
});
|
|
274
|
+
expect(resolved.metadata).toEqual({ team: "finance" });
|
|
275
|
+
expect(resolved.drops).toEqual([
|
|
276
|
+
{ name: "team.name", reason: "invalid_name" },
|
|
277
|
+
]);
|
|
278
|
+
});
|
|
279
|
+
|
|
280
|
+
it("sanitizes an unrenderable value instead of failing", () => {
|
|
281
|
+
const resolved = mergeQueryMetadata({
|
|
282
|
+
request: { note: 'a"b\nc' },
|
|
283
|
+
});
|
|
284
|
+
expect(resolved.metadata).toEqual({ note: "a_b_c" });
|
|
285
|
+
});
|
|
286
|
+
|
|
287
|
+
it("truncates an over-long value", () => {
|
|
288
|
+
const resolved = mergeQueryMetadata({
|
|
289
|
+
request: { note: "x".repeat(MAX_PROPERTY_VALUE_LENGTH + 50) },
|
|
290
|
+
});
|
|
291
|
+
expect(resolved.metadata?.note).toHaveLength(MAX_PROPERTY_VALUE_LENGTH);
|
|
292
|
+
});
|
|
293
|
+
|
|
294
|
+
it("sheds the least specific author property first", () => {
|
|
295
|
+
// The caller's own per-request property is the last one standing: it is
|
|
296
|
+
// most likely its join key, and precedence says the connection-wide
|
|
297
|
+
// default is the cheapest thing to lose.
|
|
298
|
+
const connection: Record<string, string> = {};
|
|
299
|
+
for (let i = 0; i < MAX_PROPERTIES - 2; i++) connection[`c${i}`] = "v";
|
|
300
|
+
const resolved = mergeQueryMetadata({
|
|
301
|
+
connection,
|
|
302
|
+
model: { modelProp: "m" },
|
|
303
|
+
request: { request_id: "7f3a" },
|
|
304
|
+
context: { queryClass: "interactive", package: "sales" },
|
|
305
|
+
});
|
|
306
|
+
expect(resolved.metadata?.request_id).toBe("7f3a");
|
|
307
|
+
expect(resolved.metadata?.modelProp).toBe("m");
|
|
308
|
+
expect(resolved.drops.every((d) => d.name.startsWith("c"))).toBe(true);
|
|
309
|
+
});
|
|
310
|
+
|
|
311
|
+
it("sheds by the layer whose value won, not the first that mentioned it", () => {
|
|
312
|
+
// `tier` is declared on the connection but the request's value is what
|
|
313
|
+
// survives, so it must be shed as a request property, i.e. last.
|
|
314
|
+
const connection: Record<string, string> = { tier: "bronze" };
|
|
315
|
+
for (let i = 0; i < MAX_PROPERTIES; i++) connection[`c${i}`] = "v";
|
|
316
|
+
const resolved = mergeQueryMetadata({
|
|
317
|
+
connection,
|
|
318
|
+
request: { tier: "gold" },
|
|
319
|
+
context: { queryClass: "interactive" },
|
|
320
|
+
});
|
|
321
|
+
expect(resolved.metadata?.tier).toBe("gold");
|
|
322
|
+
});
|
|
323
|
+
|
|
324
|
+
it("enforces the property cap by dropping author properties, keeping context", () => {
|
|
325
|
+
const request: Record<string, string> = {};
|
|
326
|
+
for (let i = 0; i < MAX_PROPERTIES; i++) request[`p${i}`] = "v";
|
|
327
|
+
const resolved = mergeQueryMetadata({
|
|
328
|
+
request,
|
|
329
|
+
context: { queryClass: "index", package: "sales" },
|
|
330
|
+
});
|
|
331
|
+
expect(Object.keys(resolved.metadata ?? {})).toHaveLength(MAX_PROPERTIES);
|
|
332
|
+
// Context survived; the overflow came out of the author's properties.
|
|
333
|
+
expect(resolved.metadata?.class).toBe("index");
|
|
334
|
+
expect(resolved.metadata?.package).toBe("sales");
|
|
335
|
+
expect(resolved.drops.map((d) => d.reason)).toContain("property_cap");
|
|
336
|
+
});
|
|
337
|
+
|
|
338
|
+
it("keeps the serialized bag inside Snowflake's tag limit", () => {
|
|
339
|
+
// db-snowflake slices an over-long JSON tag, which leaves it unparseable —
|
|
340
|
+
// so whole properties come out here instead.
|
|
341
|
+
const request: Record<string, string> = {};
|
|
342
|
+
for (let i = 0; i < 12; i++) {
|
|
343
|
+
request[`p${i}`] = "x".repeat(MAX_PROPERTY_VALUE_LENGTH);
|
|
344
|
+
}
|
|
345
|
+
const resolved = mergeQueryMetadata({
|
|
346
|
+
request,
|
|
347
|
+
context: { queryClass: "interactive" },
|
|
348
|
+
});
|
|
349
|
+
expect(JSON.stringify(resolved.metadata).length).toBeLessThanOrEqual(
|
|
350
|
+
2000,
|
|
351
|
+
);
|
|
352
|
+
expect(resolved.drops.map((d) => d.reason)).toContain("serialized_cap");
|
|
353
|
+
expect(resolved.metadata?.class).toBe("interactive");
|
|
354
|
+
});
|
|
355
|
+
|
|
356
|
+
it("attaches nothing by default: the feature ships dark", () => {
|
|
357
|
+
// The release default. Every statement leaves exactly as Malloy compiled
|
|
358
|
+
// it until a deployment opts in, because this is the one feature that
|
|
359
|
+
// touches all of them — and on the comment-carrying backends it changes
|
|
360
|
+
// the statement text.
|
|
361
|
+
delete process.env.PUBLISHER_QUERY_METADATA;
|
|
362
|
+
expect(
|
|
363
|
+
mergeQueryMetadata({
|
|
364
|
+
request: { team: "finance" },
|
|
365
|
+
context: { queryClass: "interactive", correlationId: "corr-1" },
|
|
366
|
+
}),
|
|
367
|
+
).toEqual({ drops: [] });
|
|
368
|
+
});
|
|
369
|
+
|
|
370
|
+
it("attaches nothing when PUBLISHER_QUERY_METADATA=off", () => {
|
|
371
|
+
process.env.PUBLISHER_QUERY_METADATA = "off";
|
|
372
|
+
expect(
|
|
373
|
+
mergeQueryMetadata({
|
|
374
|
+
request: { team: "finance" },
|
|
375
|
+
context: { queryClass: "interactive", package: "sales" },
|
|
376
|
+
}),
|
|
377
|
+
).toEqual({ drops: [] });
|
|
378
|
+
});
|
|
379
|
+
});
|
|
380
|
+
|
|
381
|
+
describe("parseSuppliedQueryMetadata", () => {
|
|
382
|
+
it("returns the bag when it conforms, undefined when empty", () => {
|
|
383
|
+
expect(parseSuppliedQueryMetadata({ team: "finance" })).toEqual({
|
|
384
|
+
team: "finance",
|
|
385
|
+
});
|
|
386
|
+
expect(parseSuppliedQueryMetadata({})).toBeUndefined();
|
|
387
|
+
expect(parseSuppliedQueryMetadata(undefined)).toBeUndefined();
|
|
388
|
+
});
|
|
389
|
+
|
|
390
|
+
it("throws listing every violation, for the caller's 400", () => {
|
|
391
|
+
expect(() =>
|
|
392
|
+
parseSuppliedQueryMetadata({ "team.name": "finance", note: 'a"b' }),
|
|
393
|
+
).toThrow(/team.name.*note|note.*team.name/s);
|
|
394
|
+
});
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
describe("parseQueryClass", () => {
|
|
398
|
+
it("accepts the four classes and passes through absence", () => {
|
|
399
|
+
expect(parseQueryClass("materialize")).toBe("materialize");
|
|
400
|
+
expect(parseQueryClass(undefined)).toBeUndefined();
|
|
401
|
+
expect(parseQueryClass(null)).toBeUndefined();
|
|
402
|
+
});
|
|
403
|
+
|
|
404
|
+
it("throws on an unrecognized class rather than ignoring it", () => {
|
|
405
|
+
expect(() => parseQueryClass("backfill")).toThrow(/queryClass/);
|
|
406
|
+
expect(() => parseQueryClass(7)).toThrow(/queryClass/);
|
|
407
|
+
});
|
|
408
|
+
});
|