@malloy-publisher/server 0.0.203 → 0.0.205
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build.ts +10 -1
- package/dist/app/api-doc.yaml +146 -0
- package/dist/app/assets/{EnvironmentPage-BVQ7glKP.js → EnvironmentPage-CAge6UHD.js} +1 -1
- package/dist/app/assets/HomePage-DhTe8qpa.js +1 -0
- package/dist/app/assets/{MainPage-bYOWcgDP.js → MainPage-CeTxxGex.js} +2 -2
- package/dist/app/assets/MaterializationsPage-CpDHB70t.js +1 -0
- package/dist/app/assets/ModelPage-D9sSMb75.js +1 -0
- package/dist/app/assets/PackagePage-LRqQWrFY.js +1 -0
- package/dist/app/assets/{RouteError-_J-EBz7W.js → RouteError-xT6kuCNw.js} +1 -1
- package/dist/app/assets/{WorkbookPage-Bjs9Nm-_.js → WorkbookPage-DsIh9svZ.js} +1 -1
- package/dist/app/assets/{core-BPLlx5VM.es-C2ARtwWI.js → core-C2sQrwVu.es-Bjem0hym.js} +1 -1
- package/dist/app/assets/{index-CqUWJELr.js → index-BdOZDcce.js} +2 -2
- package/dist/app/assets/index-DHHAcY5o.js +1812 -0
- package/dist/app/assets/index-RX3QOTde.js +455 -0
- package/dist/app/assets/index.umd-D2WH3D-f.js +2469 -0
- package/dist/app/index.html +1 -1
- package/dist/package_load_worker.mjs +392 -67
- package/dist/runtime/publisher.js +318 -0
- package/dist/server.mjs +982 -346
- package/package.json +15 -14
- package/scripts/bake-duckdb-extensions.js +104 -0
- package/src/controller/watch-mode.controller.ts +176 -46
- package/src/ducklake_version.spec.ts +43 -0
- package/src/ducklake_version.ts +26 -0
- package/src/errors.spec.ts +21 -0
- package/src/errors.ts +18 -1
- package/src/mcp/error_messages.spec.ts +35 -0
- package/src/mcp/error_messages.ts +14 -1
- package/src/mcp/handler_utils.ts +12 -0
- package/src/package_load/package_load_pool.ts +0 -5
- package/src/package_load/package_load_worker.ts +41 -99
- package/src/package_load/protocol.ts +1 -7
- package/src/runtime/publisher.js +318 -0
- package/src/server.ts +479 -2
- package/src/service/annotations.spec.ts +118 -0
- package/src/service/annotations.ts +91 -0
- package/src/service/authorize.spec.ts +132 -0
- package/src/service/authorize.ts +241 -0
- package/src/service/authorize_integration.spec.ts +932 -0
- package/src/service/compile_authorize.spec.ts +85 -0
- package/src/service/connection.ts +1 -1
- package/src/service/environment.ts +67 -9
- package/src/service/environment_store.ts +142 -11
- package/src/service/filter.spec.ts +14 -3
- package/src/service/filter.ts +5 -1
- package/src/service/filter_bypass.spec.ts +418 -0
- package/src/service/given.ts +37 -12
- package/src/service/givens_integration.spec.ts +34 -7
- package/src/service/materialization_service.ts +25 -20
- package/src/service/materialized_table_gc.spec.ts +6 -5
- package/src/service/materialized_table_gc.ts +2 -50
- package/src/service/model.spec.ts +203 -8
- package/src/service/model.ts +349 -155
- package/src/service/package.ts +17 -6
- package/src/service/package_worker_path.spec.ts +113 -0
- package/src/service/quoting.ts +0 -20
- package/src/service/restricted_mode.spec.ts +299 -0
- package/src/service/source_extraction.ts +226 -0
- package/src/storage/StorageManager.ts +73 -0
- package/src/storage/duckdb/DuckDBConnection.ts +70 -124
- package/tests/fixtures/authorize-compile/model.malloy +9 -0
- package/tests/fixtures/authorize-compile/publisher.json +4 -0
- package/tests/fixtures/html-pages-nopublic/model.malloy +1 -0
- package/tests/fixtures/html-pages-nopublic/publisher.json +5 -0
- package/tests/fixtures/html-pages-test/data.csv +3 -0
- package/tests/fixtures/html-pages-test/public/assets/app.css +3 -0
- package/tests/fixtures/html-pages-test/public/data.json +1 -0
- package/tests/fixtures/html-pages-test/public/index.html +9 -0
- package/tests/fixtures/html-pages-test/public/sub/page2.html +9 -0
- package/tests/fixtures/html-pages-test/publisher.json +5 -0
- package/tests/fixtures/html-pages-test/report.malloy +1 -0
- package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +92 -0
- package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +138 -0
- package/tests/integration/html_pages/html_pages.integration.spec.ts +378 -0
- package/tests/integration/watch-mode/watch_mode.integration.spec.ts +421 -0
- package/tests/unit/duckdb/attached_databases.test.ts +111 -0
- package/tests/unit/duckdb/duckdb_connection.test.ts +181 -0
- package/tests/unit/duckdb/repositories.test.ts +208 -0
- package/dist/app/assets/HomePage-D9drXoZX.js +0 -1
- package/dist/app/assets/ModelPage-DT0gjNy1.js +0 -1
- package/dist/app/assets/PackagePage-N1ZBNJul.js +0 -1
- package/dist/app/assets/index-BeNwIeYQ.js +0 -454
- package/dist/app/assets/index-Dx7qi2LO.js +0 -1803
- package/dist/app/assets/index.umd-BXm2lnUO.js +0 -1145
|
@@ -0,0 +1,418 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Filter-param (`#(filter)`) enforcement across source-derivation paths.
|
|
3
|
+
*
|
|
4
|
+
* A `#(filter)` annotation marks a source as protected: a required filter (e.g.
|
|
5
|
+
* the implicit multi-tenant `org_id` boundary) must be supplied before the
|
|
6
|
+
* source can be read. The aim is to enforce that without narrowing the query
|
|
7
|
+
* shapes `execute_query` accepts.
|
|
8
|
+
*
|
|
9
|
+
* Cases guarded here:
|
|
10
|
+
* - Direct read of a protected source: rejected (400) naming the missing
|
|
11
|
+
* required filter; scoped to the supplied values once provided.
|
|
12
|
+
* - Reaching a protected source under an ad-hoc name (alias / extend / chain):
|
|
13
|
+
* enforced identically — the query still runs, scoped — rather than being
|
|
14
|
+
* refused for its shape.
|
|
15
|
+
* - Unprotected sources are unaffected; `bypassFilters` skips enforcement.
|
|
16
|
+
*
|
|
17
|
+
* Language-level escapes (import, raw tables, raw SQL) are out of scope here and
|
|
18
|
+
* covered in `restricted_mode.spec.ts`.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { DuckDBConnection } from "@malloydata/db-duckdb";
|
|
22
|
+
import { Connection } from "@malloydata/malloy";
|
|
23
|
+
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
|
24
|
+
import fs from "fs/promises";
|
|
25
|
+
import os from "os";
|
|
26
|
+
import path from "path";
|
|
27
|
+
import { BadRequestError } from "../errors";
|
|
28
|
+
import { Model } from "./model";
|
|
29
|
+
import type { FilterParams } from "./filter";
|
|
30
|
+
|
|
31
|
+
const TEST_DIR = path.join(os.tmpdir(), "filter-bypass-tests");
|
|
32
|
+
const TEST_DB_DIR = path.join(TEST_DIR, "db");
|
|
33
|
+
const TEST_DB_PATH = path.join(TEST_DB_DIR, "test.duckdb");
|
|
34
|
+
const TEST_PKG_DIR = path.join(TEST_DIR, "pkg");
|
|
35
|
+
|
|
36
|
+
let duckdbConnection: DuckDBConnection;
|
|
37
|
+
|
|
38
|
+
// Three tenants. org_id is the partition key, so any cross-org leak shows up
|
|
39
|
+
// directly as extra rows when we `group_by: org_id` (acme has 2 products).
|
|
40
|
+
const SEED_SQL = `
|
|
41
|
+
CREATE TABLE IF NOT EXISTS products (
|
|
42
|
+
org_id VARCHAR,
|
|
43
|
+
product_name VARCHAR
|
|
44
|
+
);
|
|
45
|
+
INSERT INTO products VALUES
|
|
46
|
+
('acme', 'AcmeAnvil'),
|
|
47
|
+
('acme', 'AcmeRocket'),
|
|
48
|
+
('globex', 'GlobexPhone'),
|
|
49
|
+
('initech', 'InitechStapler');
|
|
50
|
+
|
|
51
|
+
CREATE TABLE IF NOT EXISTS orders (
|
|
52
|
+
org_id VARCHAR,
|
|
53
|
+
category VARCHAR,
|
|
54
|
+
region VARCHAR
|
|
55
|
+
);
|
|
56
|
+
INSERT INTO orders VALUES
|
|
57
|
+
('acme', 'widgets', 'US'),
|
|
58
|
+
('acme', 'gadgets', 'EU'),
|
|
59
|
+
('globex', 'widgets', 'US'),
|
|
60
|
+
('initech', 'gadgets', 'APAC');
|
|
61
|
+
|
|
62
|
+
CREATE TABLE IF NOT EXISTS events (
|
|
63
|
+
org_id VARCHAR,
|
|
64
|
+
kind VARCHAR
|
|
65
|
+
);
|
|
66
|
+
INSERT INTO events VALUES
|
|
67
|
+
('acme', 'click'),
|
|
68
|
+
('globex', 'view'),
|
|
69
|
+
('initech', 'scroll');
|
|
70
|
+
`;
|
|
71
|
+
|
|
72
|
+
// products: implicit Organization filter only (the multi-tenant boundary).
|
|
73
|
+
const PRODUCTS_MODEL = `
|
|
74
|
+
#(filter) name=Organization dimension=org_id type=equal implicit required
|
|
75
|
+
source: products is duckdb.table('products') extend {
|
|
76
|
+
measure: n is count()
|
|
77
|
+
view: by_org is {
|
|
78
|
+
group_by: org_id, product_name
|
|
79
|
+
aggregate: n
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
`;
|
|
83
|
+
|
|
84
|
+
// orders: implicit Organization + required-explicit Category + optional Region.
|
|
85
|
+
const ORDERS_MODEL = `
|
|
86
|
+
#(filter) name=Organization dimension=org_id type=equal implicit required
|
|
87
|
+
#(filter) name=Category dimension=category type=equal required
|
|
88
|
+
#(filter) name=Region dimension=region type=in
|
|
89
|
+
source: orders is duckdb.table('orders') extend {
|
|
90
|
+
measure: n is count()
|
|
91
|
+
}
|
|
92
|
+
`;
|
|
93
|
+
|
|
94
|
+
// A model that neither defines nor imports any protected source. Used as the
|
|
95
|
+
// target for the unprotected-source regression.
|
|
96
|
+
const ANALYTICS_MODEL = `
|
|
97
|
+
source: metrics is duckdb.table('events') extend {
|
|
98
|
+
measure: c is count()
|
|
99
|
+
}
|
|
100
|
+
`;
|
|
101
|
+
|
|
102
|
+
beforeAll(async () => {
|
|
103
|
+
await fs.mkdir(TEST_DB_DIR, { recursive: true });
|
|
104
|
+
await fs.mkdir(TEST_PKG_DIR, { recursive: true });
|
|
105
|
+
duckdbConnection = new DuckDBConnection("duckdb", TEST_DB_PATH, TEST_DB_DIR);
|
|
106
|
+
for (const stmt of SEED_SQL.trim().split(";").filter(Boolean)) {
|
|
107
|
+
await duckdbConnection.runSQL(stmt.trim() + ";");
|
|
108
|
+
}
|
|
109
|
+
await fs.writeFile(
|
|
110
|
+
path.join(TEST_PKG_DIR, "products.malloy"),
|
|
111
|
+
PRODUCTS_MODEL,
|
|
112
|
+
"utf-8",
|
|
113
|
+
);
|
|
114
|
+
await fs.writeFile(
|
|
115
|
+
path.join(TEST_PKG_DIR, "orders.malloy"),
|
|
116
|
+
ORDERS_MODEL,
|
|
117
|
+
"utf-8",
|
|
118
|
+
);
|
|
119
|
+
await fs.writeFile(
|
|
120
|
+
path.join(TEST_PKG_DIR, "analytics.malloy"),
|
|
121
|
+
ANALYTICS_MODEL,
|
|
122
|
+
"utf-8",
|
|
123
|
+
);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
afterAll(async () => {
|
|
127
|
+
try {
|
|
128
|
+
await duckdbConnection.close();
|
|
129
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
130
|
+
await fs.rm(TEST_DIR, { recursive: true, force: true });
|
|
131
|
+
} catch {
|
|
132
|
+
// Ignore cleanup errors
|
|
133
|
+
}
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
function getConnections(): Map<string, Connection> {
|
|
137
|
+
const map = new Map<string, Connection>();
|
|
138
|
+
map.set("duckdb", duckdbConnection);
|
|
139
|
+
return map;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
type Row = Record<string, unknown>;
|
|
143
|
+
|
|
144
|
+
function asRows(compactResult: unknown): Row[] {
|
|
145
|
+
return compactResult as Row[];
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async function makeModel(modelPath: string): Promise<Model> {
|
|
149
|
+
return Model.create("test-pkg", TEST_PKG_DIR, modelPath, getConnections());
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** Run an ad-hoc query string and return the result rows. */
|
|
153
|
+
async function runAdHoc(
|
|
154
|
+
model: Model,
|
|
155
|
+
query: string,
|
|
156
|
+
filterParams?: FilterParams,
|
|
157
|
+
): Promise<Row[]> {
|
|
158
|
+
const { compactResult } = await model.getQueryResults(
|
|
159
|
+
undefined,
|
|
160
|
+
undefined,
|
|
161
|
+
query,
|
|
162
|
+
filterParams,
|
|
163
|
+
);
|
|
164
|
+
return asRows(compactResult);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Assert an ad-hoc query is rejected with a 400 whose message names a missing
|
|
169
|
+
* required filter (`expectedSubstring`, e.g. "Organization"). If the query
|
|
170
|
+
* instead succeeds, the assertion fails reporting the row count — a filter
|
|
171
|
+
* bypass / data leak slipped through.
|
|
172
|
+
*/
|
|
173
|
+
async function expectFilterRejected(
|
|
174
|
+
model: Model,
|
|
175
|
+
query: string,
|
|
176
|
+
filterParams: FilterParams | undefined,
|
|
177
|
+
expectedSubstring: string,
|
|
178
|
+
): Promise<void> {
|
|
179
|
+
let leakedRows: number | undefined;
|
|
180
|
+
try {
|
|
181
|
+
const { compactResult } = await model.getQueryResults(
|
|
182
|
+
undefined,
|
|
183
|
+
undefined,
|
|
184
|
+
query,
|
|
185
|
+
filterParams,
|
|
186
|
+
);
|
|
187
|
+
leakedRows = asRows(compactResult).length;
|
|
188
|
+
} catch (error) {
|
|
189
|
+
expect(error).toBeInstanceOf(BadRequestError);
|
|
190
|
+
expect((error as Error).message).toContain(expectedSubstring);
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
throw new Error(
|
|
194
|
+
`Expected a 400 naming "${expectedSubstring}", but the query succeeded ` +
|
|
195
|
+
`and returned ${leakedRows} rows (FILTER BYPASS / LEAK).`,
|
|
196
|
+
);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/** Assert every returned row is scoped to acme and the count matches. */
|
|
200
|
+
function expectAcmeScoped(rows: Row[], expectedRows: number): void {
|
|
201
|
+
expect(rows.length).toBe(expectedRows);
|
|
202
|
+
for (const row of rows) expect(row.org_id).toBe("acme");
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// ===========================================================================
|
|
206
|
+
// Enforced: direct reads of a protected source.
|
|
207
|
+
// ===========================================================================
|
|
208
|
+
|
|
209
|
+
describe("direct read of a protected source is enforced", () => {
|
|
210
|
+
it("rejects a direct query with no filter params", async () => {
|
|
211
|
+
const model = await makeModel("products.malloy");
|
|
212
|
+
await expectFilterRejected(
|
|
213
|
+
model,
|
|
214
|
+
"run: products -> { group_by: org_id, product_name; aggregate: n is count() }",
|
|
215
|
+
undefined,
|
|
216
|
+
"Organization",
|
|
217
|
+
);
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
it("scopes a direct query when Organization is supplied", async () => {
|
|
221
|
+
const model = await makeModel("products.malloy");
|
|
222
|
+
const rows = await runAdHoc(
|
|
223
|
+
model,
|
|
224
|
+
"run: products -> { group_by: org_id, product_name; aggregate: n is count() }",
|
|
225
|
+
{ Organization: "acme" },
|
|
226
|
+
);
|
|
227
|
+
expectAcmeScoped(rows, 2); // acme has exactly 2 products
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
// A predefined view on the trusted source is still the direct path.
|
|
231
|
+
it("enforces filters on a direct named-view read", async () => {
|
|
232
|
+
const model = await makeModel("products.malloy");
|
|
233
|
+
await expectFilterRejected(
|
|
234
|
+
model,
|
|
235
|
+
"run: products -> by_org",
|
|
236
|
+
undefined,
|
|
237
|
+
"Organization",
|
|
238
|
+
);
|
|
239
|
+
const rows = await runAdHoc(model, "run: products -> by_org", {
|
|
240
|
+
Organization: "acme",
|
|
241
|
+
});
|
|
242
|
+
expect(rows.length).toBe(2);
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
// orders — implicit Organization + required-explicit Category. Both required
|
|
246
|
+
// filters are enforced; the error names whichever is still missing.
|
|
247
|
+
it("rejects orders with no filter params (names Organization)", async () => {
|
|
248
|
+
const model = await makeModel("orders.malloy");
|
|
249
|
+
await expectFilterRejected(
|
|
250
|
+
model,
|
|
251
|
+
"run: orders -> { group_by: org_id, category; aggregate: n is count() }",
|
|
252
|
+
undefined,
|
|
253
|
+
"Organization",
|
|
254
|
+
);
|
|
255
|
+
});
|
|
256
|
+
|
|
257
|
+
it("rejects orders with only Category supplied (still missing Organization)", async () => {
|
|
258
|
+
const model = await makeModel("orders.malloy");
|
|
259
|
+
await expectFilterRejected(
|
|
260
|
+
model,
|
|
261
|
+
"run: orders -> { group_by: org_id, category; aggregate: n is count() }",
|
|
262
|
+
{ Category: "widgets" },
|
|
263
|
+
"Organization",
|
|
264
|
+
);
|
|
265
|
+
});
|
|
266
|
+
|
|
267
|
+
it("rejects orders with only Organization supplied (still missing Category)", async () => {
|
|
268
|
+
const model = await makeModel("orders.malloy");
|
|
269
|
+
await expectFilterRejected(
|
|
270
|
+
model,
|
|
271
|
+
"run: orders -> { group_by: org_id, category; aggregate: n is count() }",
|
|
272
|
+
{ Organization: "acme" },
|
|
273
|
+
"Category",
|
|
274
|
+
);
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
it("scopes orders when both Organization and Category are supplied", async () => {
|
|
278
|
+
const model = await makeModel("orders.malloy");
|
|
279
|
+
const rows = await runAdHoc(
|
|
280
|
+
model,
|
|
281
|
+
"run: orders -> { group_by: org_id, category; aggregate: n is count() }",
|
|
282
|
+
{ Organization: "acme", Category: "widgets" },
|
|
283
|
+
);
|
|
284
|
+
expect(rows.length).toBe(1); // acme + widgets → one group
|
|
285
|
+
expect(rows[0].org_id).toBe("acme");
|
|
286
|
+
expect(rows[0].category).toBe("widgets");
|
|
287
|
+
});
|
|
288
|
+
});
|
|
289
|
+
|
|
290
|
+
// ===========================================================================
|
|
291
|
+
// Enforced: alias / extend / chain of a protected source.
|
|
292
|
+
//
|
|
293
|
+
// Reaching a protected source under an ad-hoc name carries the SAME filter
|
|
294
|
+
// requirement. The query is not rejected for its shape — it runs, scoped — so
|
|
295
|
+
// the `execute_query` surface stays intact.
|
|
296
|
+
// ===========================================================================
|
|
297
|
+
|
|
298
|
+
interface EnforcedVector {
|
|
299
|
+
label: string;
|
|
300
|
+
modelPath: string;
|
|
301
|
+
query: string;
|
|
302
|
+
/** Required filter named in the missing-param rejection. */
|
|
303
|
+
missing: string;
|
|
304
|
+
/** Params satisfying every required filter on the protected source. */
|
|
305
|
+
validParams: FilterParams;
|
|
306
|
+
/** Rows expected once scoped to acme. */
|
|
307
|
+
expectedRows: number;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
const enforcedVectors: EnforcedVector[] = [
|
|
311
|
+
{
|
|
312
|
+
// Plain alias — a pure rename of the protected source.
|
|
313
|
+
label: "alias (source a is products)",
|
|
314
|
+
modelPath: "products.malloy",
|
|
315
|
+
query: "source: a is products\nrun: a -> { group_by: org_id, product_name; aggregate: n is count() }",
|
|
316
|
+
missing: "Organization",
|
|
317
|
+
validParams: { Organization: "acme" },
|
|
318
|
+
expectedRows: 2,
|
|
319
|
+
},
|
|
320
|
+
{
|
|
321
|
+
// Extend with an extra measure. The body adds to the curated surface but
|
|
322
|
+
// does not touch the filter dimension, so enforcement is unaffected.
|
|
323
|
+
label: "extend (source e is products extend { … })",
|
|
324
|
+
modelPath: "products.malloy",
|
|
325
|
+
query: "source: e is products extend { measure: rc is count() }\nrun: e -> { group_by: org_id, product_name; aggregate: rc }",
|
|
326
|
+
missing: "Organization",
|
|
327
|
+
validParams: { Organization: "acme" },
|
|
328
|
+
expectedRows: 2,
|
|
329
|
+
},
|
|
330
|
+
{
|
|
331
|
+
// Chained derivation — protection is inherited link by link and resolved
|
|
332
|
+
// back to products.
|
|
333
|
+
label: "chained (a is products; b is a)",
|
|
334
|
+
modelPath: "products.malloy",
|
|
335
|
+
query: "source: a is products\nsource: b is a\nrun: b -> { group_by: org_id, product_name; aggregate: n is count() }",
|
|
336
|
+
missing: "Organization",
|
|
337
|
+
validParams: { Organization: "acme" },
|
|
338
|
+
expectedRows: 2,
|
|
339
|
+
},
|
|
340
|
+
{
|
|
341
|
+
// Ad-hoc `#(filter)` annotation in the query text declaring a NON-required
|
|
342
|
+
// Organization filter on the alias, attempting to drop the requirement.
|
|
343
|
+
// Annotations written in the query are not honored — only the protected
|
|
344
|
+
// source's own annotation counts — so the requirement still stands.
|
|
345
|
+
label: "ad-hoc #(filter) annotation override is ignored",
|
|
346
|
+
modelPath: "products.malloy",
|
|
347
|
+
query:
|
|
348
|
+
"#(filter) name=Organization dimension=org_id type=equal\n" +
|
|
349
|
+
"source: a is products extend {}\n" +
|
|
350
|
+
"run: a -> { group_by: org_id, product_name; aggregate: n is count() }",
|
|
351
|
+
missing: "Organization",
|
|
352
|
+
validParams: { Organization: "acme" },
|
|
353
|
+
expectedRows: 2,
|
|
354
|
+
},
|
|
355
|
+
{
|
|
356
|
+
// orders alias — confirms enforcement on a multi-required-filter source.
|
|
357
|
+
label: "orders alias (source a is orders)",
|
|
358
|
+
modelPath: "orders.malloy",
|
|
359
|
+
query: "source: a is orders\nrun: a -> { group_by: org_id, category; aggregate: n is count() }",
|
|
360
|
+
missing: "Organization",
|
|
361
|
+
validParams: { Organization: "acme", Category: "widgets" },
|
|
362
|
+
expectedRows: 1,
|
|
363
|
+
},
|
|
364
|
+
{
|
|
365
|
+
// orders extend.
|
|
366
|
+
label: "orders extend",
|
|
367
|
+
modelPath: "orders.malloy",
|
|
368
|
+
query: "source: e is orders extend { measure: rc is count() }\nrun: e -> { group_by: org_id, category; aggregate: rc }",
|
|
369
|
+
missing: "Organization",
|
|
370
|
+
validParams: { Organization: "acme", Category: "widgets" },
|
|
371
|
+
expectedRows: 1,
|
|
372
|
+
},
|
|
373
|
+
];
|
|
374
|
+
|
|
375
|
+
describe("alias/extend/chain of a protected source is enforced (not restricted)", () => {
|
|
376
|
+
for (const v of enforcedVectors) {
|
|
377
|
+
describe(v.label, () => {
|
|
378
|
+
it("rejects when the required filter is missing", async () => {
|
|
379
|
+
const model = await makeModel(v.modelPath);
|
|
380
|
+
await expectFilterRejected(model, v.query, undefined, v.missing);
|
|
381
|
+
});
|
|
382
|
+
|
|
383
|
+
it("runs scoped when valid filter params are supplied", async () => {
|
|
384
|
+
const model = await makeModel(v.modelPath);
|
|
385
|
+
const rows = await runAdHoc(model, v.query, v.validParams);
|
|
386
|
+
expectAcmeScoped(rows, v.expectedRows);
|
|
387
|
+
});
|
|
388
|
+
});
|
|
389
|
+
}
|
|
390
|
+
});
|
|
391
|
+
|
|
392
|
+
// ===========================================================================
|
|
393
|
+
// Regression: unprotected sources stay open; bypassFilters short-circuits
|
|
394
|
+
// filter enforcement.
|
|
395
|
+
// ===========================================================================
|
|
396
|
+
|
|
397
|
+
describe("regressions", () => {
|
|
398
|
+
it("runs an unprotected source with no filter params (no spurious demand)", async () => {
|
|
399
|
+
const model = await makeModel("analytics.malloy");
|
|
400
|
+
const rows = await runAdHoc(
|
|
401
|
+
model,
|
|
402
|
+
"run: metrics -> { group_by: org_id; aggregate: c is count() }",
|
|
403
|
+
);
|
|
404
|
+
expect(rows.length).toBe(3); // all three orgs, unfiltered
|
|
405
|
+
});
|
|
406
|
+
|
|
407
|
+
it("bypassFilters short-circuits filter enforcement on a derivation", async () => {
|
|
408
|
+
const model = await makeModel("products.malloy");
|
|
409
|
+
const { compactResult } = await model.getQueryResults(
|
|
410
|
+
undefined,
|
|
411
|
+
undefined,
|
|
412
|
+
"source: a is products\nrun: a -> { group_by: org_id; aggregate: n is count() }",
|
|
413
|
+
{},
|
|
414
|
+
true,
|
|
415
|
+
);
|
|
416
|
+
expect(asRows(compactResult).length).toBe(3);
|
|
417
|
+
});
|
|
418
|
+
});
|
package/src/service/given.ts
CHANGED
|
@@ -13,14 +13,18 @@
|
|
|
13
13
|
* deps, so it's safe to bundle into the worker entry).
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
|
+
import type { Annotations } from "@malloydata/malloy";
|
|
17
|
+
import { isReservedRoute } from "./annotations";
|
|
18
|
+
|
|
16
19
|
/**
|
|
17
20
|
* Duck-typed shape of a Malloy SDK `Given` instance (the value type
|
|
18
|
-
* of `Model.givens`).
|
|
21
|
+
* of `Model.givens`). `Given` itself isn't re-exported from the
|
|
22
|
+
* package root, but the `Annotations` view it returns is.
|
|
19
23
|
*/
|
|
20
24
|
export interface MalloyGiven {
|
|
21
25
|
readonly name: string;
|
|
22
26
|
readonly type: { type: string; filterType?: string };
|
|
23
|
-
|
|
27
|
+
readonly annotations: Annotations;
|
|
24
28
|
}
|
|
25
29
|
|
|
26
30
|
/**
|
|
@@ -32,6 +36,14 @@ export interface MalloyGivenApi {
|
|
|
32
36
|
name: string;
|
|
33
37
|
type: string;
|
|
34
38
|
annotations?: string[];
|
|
39
|
+
/**
|
|
40
|
+
* The given's default as a Malloy source literal — one literal per declared
|
|
41
|
+
* `type`. Examples across the type range: `'WN'` or `"WN"` (string), `2003`
|
|
42
|
+
* (number), `true` (boolean), `@2024-01-01` (date), `f'WN'` (filter). Omitted
|
|
43
|
+
* when the given has no default. Consumers render/prefill it per `type` (e.g.
|
|
44
|
+
* unquote a string).
|
|
45
|
+
*/
|
|
46
|
+
default?: string;
|
|
35
47
|
}
|
|
36
48
|
|
|
37
49
|
/**
|
|
@@ -46,16 +58,18 @@ export interface MalloyGivenApi {
|
|
|
46
58
|
* location either; matching that floor. A future PR can add a
|
|
47
59
|
* sanitised package-relative path if a client needs it.
|
|
48
60
|
*
|
|
49
|
-
* - `default`
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
61
|
+
* - `default` is surfaced as the rendered source literal
|
|
62
|
+
* (`given._internal.defaultText` — e.g. a string `'WN'`, number
|
|
63
|
+
* `2003`, boolean `true`, date `@2024-01-01`, or filter `f'WN'`).
|
|
64
|
+
* Malloy's public surface still exposes only the parsed `.default`
|
|
65
|
+
* AST; `_internal.defaultText` is the already-rendered string, so we
|
|
66
|
+
* forward it verbatim rather than re-implement the printer. Omitted
|
|
67
|
+
* when the given has no default.
|
|
53
68
|
*
|
|
54
|
-
* `annotations` is restricted to
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
* surface contract.
|
|
69
|
+
* `annotations` is restricted to app-route annotations (bracketed,
|
|
70
|
+
* caller-facing, e.g. `#(doc)`), excluding Malloy's reserved routes
|
|
71
|
+
* (plain `#` tags, `#"` doc strings, `##!` pragmas), which aren't part
|
|
72
|
+
* of the given's surface contract.
|
|
59
73
|
*
|
|
60
74
|
* Type rendering: `GivenTypeDef` is typed as `AtomicTypeDef |
|
|
61
75
|
* FilterExpressionParamTypeDef`, but Malloy's grammar only emits
|
|
@@ -75,6 +89,17 @@ export function malloyGivenToApi(given: MalloyGiven): MalloyGivenApi {
|
|
|
75
89
|
return {
|
|
76
90
|
name: given.name,
|
|
77
91
|
type: renderedType,
|
|
78
|
-
annotations: given.
|
|
92
|
+
annotations: given.annotations
|
|
93
|
+
.forRoute(undefined)
|
|
94
|
+
.filter((note) => !isReservedRoute(note.route))
|
|
95
|
+
.map((note) => note.text),
|
|
96
|
+
// `_internal.defaultText` is the already-rendered source literal of the
|
|
97
|
+
// given's default. It lives on Malloy's private `_internal` (the public
|
|
98
|
+
// surface exposes only the parsed `.default` AST node, not a stringified
|
|
99
|
+
// form), so we reach it through a localized cast rather than widening the
|
|
100
|
+
// duck-typed `MalloyGiven` — which would collide with the SDK `Given`'s
|
|
101
|
+
// own private `_internal` at every `as MalloyGiven` cast site.
|
|
102
|
+
default: (given as { _internal?: { defaultText?: string } })._internal
|
|
103
|
+
?.defaultText,
|
|
79
104
|
};
|
|
80
105
|
}
|
|
@@ -123,8 +123,36 @@ describe("givens introspection", () => {
|
|
|
123
123
|
|
|
124
124
|
expect(region).toBeDefined();
|
|
125
125
|
expect(region?.type).toBe("string");
|
|
126
|
+
expect(region?.default).toBe("'US'");
|
|
126
127
|
expect(cutoff).toBeDefined();
|
|
127
128
|
expect(cutoff?.type).toBe("date");
|
|
129
|
+
expect(cutoff?.default).toBe("@2024-02-01");
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
it("omits default for a given declared without one", async () => {
|
|
133
|
+
await fs.writeFile(
|
|
134
|
+
path.join(TEST_PKG_DIR, "mixed_defaults.malloy"),
|
|
135
|
+
`##! experimental.givens
|
|
136
|
+
|
|
137
|
+
given: with_default :: string is 'WN'
|
|
138
|
+
given: no_default :: string
|
|
139
|
+
|
|
140
|
+
source: orders is duckdb.table('orders') extend {
|
|
141
|
+
primary_key: order_id
|
|
142
|
+
}
|
|
143
|
+
`,
|
|
144
|
+
);
|
|
145
|
+
const model = await Model.create(
|
|
146
|
+
"test-pkg",
|
|
147
|
+
TEST_PKG_DIR,
|
|
148
|
+
"mixed_defaults.malloy",
|
|
149
|
+
getConnections(),
|
|
150
|
+
);
|
|
151
|
+
const byName = new Map(
|
|
152
|
+
((await model.getModel()).givens ?? []).map((g) => [g.name, g]),
|
|
153
|
+
);
|
|
154
|
+
expect(byName.get("with_default")?.default).toBe("'WN'");
|
|
155
|
+
expect(byName.get("no_default")?.default).toBeUndefined();
|
|
128
156
|
});
|
|
129
157
|
|
|
130
158
|
it("attaches the model-level givens list to every source", async () => {
|
|
@@ -179,14 +207,13 @@ describe("givens introspection", () => {
|
|
|
179
207
|
const region = compiledModel.givens?.[0];
|
|
180
208
|
expect(region?.name).toBe("region_filter");
|
|
181
209
|
|
|
182
|
-
// The
|
|
183
|
-
// Only
|
|
210
|
+
// The given declares two app-route annotations (`#(doc)`, `#(label)`).
|
|
211
|
+
// Only app routes land on the wire; Malloy-reserved routes — the
|
|
212
|
+
// model-level `##!` pragma, plain `#` tags, `#"` doc strings — must
|
|
213
|
+
// not leak onto the given's surface.
|
|
184
214
|
const annotations = region?.annotations ?? [];
|
|
185
215
|
expect(annotations.length).toBeGreaterThanOrEqual(2);
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
}
|
|
189
|
-
// Negative assertion: no pragma leakage.
|
|
190
|
-
expect(annotations.some((a) => a.startsWith("##!"))).toBe(false);
|
|
216
|
+
expect(annotations.some((a) => a.startsWith("##"))).toBe(false);
|
|
217
|
+
expect(annotations.some((a) => a.startsWith('#"'))).toBe(false);
|
|
191
218
|
});
|
|
192
219
|
});
|
|
@@ -27,7 +27,7 @@ import {
|
|
|
27
27
|
liveTableKey,
|
|
28
28
|
} from "./materialized_table_gc";
|
|
29
29
|
import { Model } from "./model";
|
|
30
|
-
import {
|
|
30
|
+
import { splitTablePath } from "./quoting";
|
|
31
31
|
import { resolveEnvironmentId } from "./resolve_environment";
|
|
32
32
|
|
|
33
33
|
/**
|
|
@@ -91,12 +91,17 @@ export function manifestTableKey(
|
|
|
91
91
|
* running a zero-row SELECT. Returns `true` if the table resolves,
|
|
92
92
|
* `false` if the query fails (assumed "table not found").
|
|
93
93
|
*/
|
|
94
|
+
/**
|
|
95
|
+
* `tableName` is interpolated verbatim into the probe SQL — the caller
|
|
96
|
+
* supplies it already quoted for the dialect (the `#@ persist name=…`
|
|
97
|
+
* contract), matching how Malloy substitutes the name on the read side.
|
|
98
|
+
*/
|
|
94
99
|
export async function tablePhysicallyExists(
|
|
95
100
|
connection: MalloyConnection,
|
|
96
|
-
|
|
101
|
+
tableName: string,
|
|
97
102
|
): Promise<boolean> {
|
|
98
103
|
try {
|
|
99
|
-
await connection.runSQL(`SELECT 1 FROM ${
|
|
104
|
+
await connection.runSQL(`SELECT 1 FROM ${tableName} WHERE 1=0`);
|
|
100
105
|
return true;
|
|
101
106
|
} catch {
|
|
102
107
|
return false;
|
|
@@ -731,7 +736,7 @@ export class MaterializationService {
|
|
|
731
736
|
|
|
732
737
|
// getBuildPlan() throws if the tag is missing, so check first to
|
|
733
738
|
// keep plain models in the same package buildable.
|
|
734
|
-
const modelTag = malloyModel.
|
|
739
|
+
const modelTag = malloyModel.annotations.parseAsTag("!").tag;
|
|
735
740
|
if (!modelTag.has("experimental", "persistence")) {
|
|
736
741
|
logger.debug(
|
|
737
742
|
"Model has no ##! experimental.persistence tag, skipping",
|
|
@@ -775,7 +780,7 @@ export class MaterializationService {
|
|
|
775
780
|
const tableOwners = new Map<string, string>();
|
|
776
781
|
for (const [sourceID, source] of Object.entries(allSources)) {
|
|
777
782
|
const tableName =
|
|
778
|
-
source.
|
|
783
|
+
source.annotations.parseAsTag("@").tag.text("name") || source.name;
|
|
779
784
|
const key = `${source.connectionName}::${tableName}`;
|
|
780
785
|
const existing = tableOwners.get(key);
|
|
781
786
|
if (existing) {
|
|
@@ -844,12 +849,16 @@ export class MaterializationService {
|
|
|
844
849
|
|
|
845
850
|
const connectionName = persistSource.connectionName;
|
|
846
851
|
const tableName =
|
|
847
|
-
persistSource.
|
|
852
|
+
persistSource.annotations.parseAsTag("@").tag.text("name") ||
|
|
848
853
|
persistSource.name;
|
|
849
|
-
const {
|
|
850
|
-
const stagingTableName = `${
|
|
851
|
-
|
|
852
|
-
|
|
854
|
+
const { bareName } = splitTablePath(tableName);
|
|
855
|
+
const stagingTableName = `${tableName}${stagingSuffix(buildId)}`;
|
|
856
|
+
|
|
857
|
+
// Table names go into DDL verbatim. Malloy assumes a table name handed
|
|
858
|
+
// to it (here, via the build manifest) is already quoted for the
|
|
859
|
+
// dialect and substitutes it into generated SQL as-is; our DDL has to
|
|
860
|
+
// match that exact identifier or the CREATE and the read diverge. The
|
|
861
|
+
// model author owns quoting the `#@ persist name=...` value.
|
|
853
862
|
|
|
854
863
|
// Guard: refuse to overwrite a pre-existing table that was not
|
|
855
864
|
// created by a previous materialization build. Without this check a
|
|
@@ -858,7 +867,7 @@ export class MaterializationService {
|
|
|
858
867
|
// DROP TABLE below would silently destroy it.
|
|
859
868
|
const tableKey = manifestTableKey(connectionName, tableName);
|
|
860
869
|
if (!knownMaterializedTables.has(tableKey)) {
|
|
861
|
-
if (await tablePhysicallyExists(connection,
|
|
870
|
+
if (await tablePhysicallyExists(connection, tableName)) {
|
|
862
871
|
throw new BadRequestError(
|
|
863
872
|
`Refusing to materialize source '${persistSource.name}': ` +
|
|
864
873
|
`target table '${tableName}' already exists on connection ` +
|
|
@@ -877,26 +886,22 @@ export class MaterializationService {
|
|
|
877
886
|
|
|
878
887
|
const startTime = performance.now();
|
|
879
888
|
|
|
880
|
-
await connection.runSQL(
|
|
881
|
-
`DROP TABLE IF EXISTS ${quoted(stagingTableName)}`,
|
|
882
|
-
);
|
|
889
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
|
|
883
890
|
|
|
884
891
|
// If any step after CREATE throws we must best-effort drop the
|
|
885
892
|
// staging table, else it orphans under a name that GC will never
|
|
886
893
|
// find (no manifest row is written for a failed build).
|
|
887
894
|
try {
|
|
888
895
|
await connection.runSQL(
|
|
889
|
-
`CREATE TABLE ${
|
|
896
|
+
`CREATE TABLE ${stagingTableName} AS (${buildSQL})`,
|
|
890
897
|
);
|
|
891
|
-
await connection.runSQL(`DROP TABLE IF EXISTS ${
|
|
898
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${tableName}`);
|
|
892
899
|
await connection.runSQL(
|
|
893
|
-
`ALTER TABLE ${
|
|
900
|
+
`ALTER TABLE ${stagingTableName} RENAME TO ${bareName}`,
|
|
894
901
|
);
|
|
895
902
|
} catch (err) {
|
|
896
903
|
try {
|
|
897
|
-
await connection.runSQL(
|
|
898
|
-
`DROP TABLE IF EXISTS ${quoted(stagingTableName)}`,
|
|
899
|
-
);
|
|
904
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
|
|
900
905
|
} catch (cleanupErr) {
|
|
901
906
|
logger.warn(
|
|
902
907
|
"Build: failed to clean up staging table after a failed rebuild; physical leak",
|