@malloy-publisher/server 0.0.205 → 0.0.207
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +394 -395
- package/dist/app/assets/{EnvironmentPage-CAge6UHD.js → EnvironmentPage-BScgHmkw.js} +1 -1
- package/dist/app/assets/{HomePage-DhTe8qpa.js → HomePage-CGedji_w.js} +1 -1
- package/dist/app/assets/{MainPage-CeTxxGex.js → MainPage-DWfF4jXW.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-CpDHB70t.js → MaterializationsPage-B9PDlk8c.js} +1 -1
- package/dist/app/assets/{ModelPage-D9sSMb75.js → ModelPage-BiNOgK_e.js} +1 -1
- package/dist/app/assets/{PackagePage-LRqQWrFY.js → PackagePage-DAN5V7gu.js} +1 -1
- package/dist/app/assets/{RouteError-xT6kuCNw.js → RouteError-CEnIzuKN.js} +1 -1
- package/dist/app/assets/{WorkbookPage-DsIh9svZ.js → WorkbookPage-gA1ceqHP.js} +1 -1
- package/dist/app/assets/{core-C2sQrwVu.es-Bjem0hym.js → core-AOmIKwkc.es-Dclu1Fga.js} +1 -1
- package/dist/app/assets/{index-BdOZDcce.js → index-DGGe8UpP.js} +1 -1
- package/dist/app/assets/{index-RX3QOTde.js → index-DtlPzNxc.js} +127 -127
- package/dist/app/assets/{index-DHHAcY5o.js → index-uu6UpHd2.js} +1 -1
- package/dist/app/assets/{index.umd-D2WH3D-f.js → index.umd-DDq93YX4.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +18 -8
- package/dist/package_load_worker.mjs +19 -2
- package/dist/server.mjs +1272 -1299
- package/package.json +1 -1
- package/src/constants.ts +12 -0
- package/src/controller/materialization.controller.ts +79 -35
- package/src/controller/package.controller.spec.ts +179 -0
- package/src/controller/package.controller.ts +60 -73
- package/src/dto/package.dto.ts +16 -1
- package/src/errors.spec.ts +12 -0
- package/src/errors.ts +18 -0
- package/src/health.spec.ts +34 -1
- package/src/instrumentation.ts +33 -17
- package/src/materialization_metrics.ts +121 -0
- package/src/package_load/package_load_pool.ts +7 -1
- package/src/package_load/package_load_worker.ts +44 -4
- package/src/package_load/protocol.ts +7 -1
- package/src/server-old.ts +7 -149
- package/src/server.ts +9 -188
- package/src/service/authorize_integration.spec.ts +67 -0
- package/src/service/environment.ts +270 -12
- package/src/service/environment_store.spec.ts +0 -81
- package/src/service/environment_store.ts +0 -23
- package/src/service/explore_visibility.spec.ts +434 -0
- package/src/service/exports_probe.spec.ts +107 -0
- package/src/service/manifest_loader.spec.ts +99 -0
- package/src/service/manifest_loader.ts +95 -0
- package/src/service/materialization_service.spec.ts +584 -500
- package/src/service/materialization_service.ts +839 -657
- package/src/service/model.ts +419 -15
- package/src/service/package.spec.ts +14 -2
- package/src/service/package.ts +339 -29
- package/src/service/package_rollback.spec.ts +190 -0
- package/src/service/package_worker_path.spec.ts +223 -0
- package/src/service/query_boundary.spec.ts +470 -0
- package/src/storage/DatabaseInterface.ts +35 -57
- package/src/storage/StorageManager.mock.ts +0 -9
- package/src/storage/StorageManager.ts +7 -290
- package/src/storage/duckdb/DuckDBRepository.ts +2 -35
- package/src/storage/duckdb/MaterializationRepository.ts +52 -27
- package/src/storage/duckdb/schema.ts +4 -20
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +194 -0
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +369 -264
- package/tests/unit/duckdb/legacy_schema_migration.test.ts +7 -4
- package/src/controller/manifest.controller.ts +0 -38
- package/src/service/manifest_service.spec.ts +0 -206
- package/src/service/manifest_service.ts +0 -117
- package/src/service/materialized_table_gc.spec.ts +0 -384
- package/src/service/materialized_table_gc.ts +0 -231
- package/src/storage/duckdb/DuckDBManifestStore.ts +0 -70
- package/src/storage/duckdb/ManifestRepository.ts +0 -120
- package/src/storage/duckdb/manifest_store.spec.ts +0 -133
- package/src/storage/ducklake/DuckLakeManifestStore.ts +0 -155
- package/tests/unit/storage/StorageManager.test.ts +0 -166
|
@@ -12,7 +12,16 @@ const PROJECT_NAME = "test-project";
|
|
|
12
12
|
const PACKAGE_NAME = "persist-test";
|
|
13
13
|
const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
/** Statuses from which no background round is in flight. */
|
|
16
|
+
const SETTLED_STATUSES = [
|
|
17
|
+
"BUILD_PLAN_READY",
|
|
18
|
+
"MANIFEST_FILE_READY",
|
|
19
|
+
"FAILED",
|
|
20
|
+
"CANCELLED",
|
|
21
|
+
];
|
|
22
|
+
const TERMINAL_STATUSES = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"];
|
|
23
|
+
|
|
24
|
+
describe("Materialization REST API: auto-run + two-round (E2E)", () => {
|
|
16
25
|
let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
|
|
17
26
|
let baseUrl: string;
|
|
18
27
|
|
|
@@ -82,45 +91,60 @@ describe("Materialization & Manifest REST API (E2E)", () => {
|
|
|
82
91
|
return `${baseUrl}${API}${p}`;
|
|
83
92
|
}
|
|
84
93
|
|
|
94
|
+
// Most tests here exercise the control-plane two-round flow, so default to
|
|
95
|
+
// pauseBetweenPhases=true (pause at BUILD_PLAN_READY). The auto-run group
|
|
96
|
+
// overrides this with pauseBetweenPhases=false.
|
|
85
97
|
async function createMaterialization(
|
|
86
98
|
body: Record<string, unknown> = {},
|
|
87
99
|
): Promise<Response> {
|
|
88
100
|
return fetch(url("/materializations"), {
|
|
89
101
|
method: "POST",
|
|
90
102
|
headers: { "Content-Type": "application/json" },
|
|
91
|
-
body: JSON.stringify(body),
|
|
103
|
+
body: JSON.stringify({ pauseBetweenPhases: true, ...body }),
|
|
92
104
|
});
|
|
93
105
|
}
|
|
94
106
|
|
|
95
|
-
async function
|
|
107
|
+
async function getMaterialization(
|
|
96
108
|
id: string,
|
|
109
|
+
): Promise<Record<string, unknown>> {
|
|
110
|
+
const res = await fetch(url(`/materializations/${id}`));
|
|
111
|
+
expect(res.status).toBe(200);
|
|
112
|
+
return (await res.json()) as Record<string, unknown>;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Poll until `status` satisfies `done`, returning the record. */
|
|
116
|
+
async function pollUntil(
|
|
117
|
+
id: string,
|
|
118
|
+
done: (status: string) => boolean,
|
|
97
119
|
timeoutMs = 30_000,
|
|
98
120
|
): Promise<Record<string, unknown>> {
|
|
99
121
|
const deadline = Date.now() + timeoutMs;
|
|
100
122
|
while (Date.now() < deadline) {
|
|
101
|
-
const
|
|
102
|
-
|
|
103
|
-
const data = (await res.json()) as Record<string, unknown>;
|
|
104
|
-
const status = data.status as string;
|
|
105
|
-
if (["SUCCESS", "FAILED", "CANCELLED"].includes(status)) {
|
|
123
|
+
const data = await getMaterialization(id);
|
|
124
|
+
if (done(data.status as string)) {
|
|
106
125
|
return data;
|
|
107
126
|
}
|
|
108
|
-
await new Promise((r) => setTimeout(r,
|
|
127
|
+
await new Promise((r) => setTimeout(r, 250));
|
|
109
128
|
}
|
|
110
|
-
throw new Error(`Materialization ${id} did not reach
|
|
129
|
+
throw new Error(`Materialization ${id} did not reach the expected state`);
|
|
111
130
|
}
|
|
112
131
|
|
|
132
|
+
const pollUntilSettled = (id: string) =>
|
|
133
|
+
pollUntil(id, (s) => SETTLED_STATUSES.includes(s));
|
|
134
|
+
const pollUntilTerminal = (id: string) =>
|
|
135
|
+
pollUntil(id, (s) => TERMINAL_STATUSES.includes(s));
|
|
136
|
+
|
|
113
137
|
/**
|
|
114
|
-
*
|
|
115
|
-
*
|
|
138
|
+
* Drive a materialization to a terminal state and delete its record so it
|
|
139
|
+
* doesn't hold the per-package active slot for later tests.
|
|
116
140
|
*/
|
|
117
141
|
async function cleanup(id: string): Promise<void> {
|
|
118
142
|
const res = await fetch(url(`/materializations/${id}`));
|
|
119
143
|
if (res.status !== 200) return;
|
|
120
|
-
const data = (await res.json()) as Record<string, unknown>;
|
|
121
|
-
const status = data.status as string;
|
|
122
144
|
|
|
123
|
-
|
|
145
|
+
// Let any in-flight round settle before acting on it.
|
|
146
|
+
const settled = await pollUntilSettled(id);
|
|
147
|
+
if (!TERMINAL_STATUSES.includes(settled.status as string)) {
|
|
124
148
|
await fetch(url(`/materializations/${id}?action=stop`), {
|
|
125
149
|
method: "POST",
|
|
126
150
|
});
|
|
@@ -129,104 +153,298 @@ describe("Materialization & Manifest REST API (E2E)", () => {
|
|
|
129
153
|
await fetch(url(`/materializations/${id}`), { method: "DELETE" });
|
|
130
154
|
}
|
|
131
155
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
156
|
+
/** First planned source from a BUILD_PLAN_READY materialization. */
|
|
157
|
+
function firstPlannedSource(
|
|
158
|
+
materialization: Record<string, unknown>,
|
|
159
|
+
): Record<string, unknown> {
|
|
160
|
+
const plan = materialization.buildPlan as Record<string, unknown>;
|
|
161
|
+
expect(plan).toBeDefined();
|
|
162
|
+
const sources = plan.sources as Record<string, Record<string, unknown>>;
|
|
163
|
+
const values = Object.values(sources);
|
|
164
|
+
expect(values.length).toBeGreaterThan(0);
|
|
165
|
+
return values[0];
|
|
166
|
+
}
|
|
136
167
|
|
|
137
|
-
|
|
138
|
-
if (materializationId) await cleanup(materializationId);
|
|
139
|
-
});
|
|
168
|
+
// ── Group A0: Auto-run lifecycle (default) ───────────────────────
|
|
140
169
|
|
|
170
|
+
describe("auto-run lifecycle (default)", () => {
|
|
141
171
|
it(
|
|
142
|
-
"
|
|
172
|
+
"runs all phases on create, self-assigns names, and auto-loads",
|
|
143
173
|
async () => {
|
|
144
|
-
//
|
|
174
|
+
// pauseBetweenPhases=false (default): the publisher runs compile +
|
|
175
|
+
// plan + build + load in one pass with no Round 2.
|
|
145
176
|
const createRes = await createMaterialization({
|
|
146
|
-
|
|
177
|
+
pauseBetweenPhases: false,
|
|
147
178
|
});
|
|
148
179
|
expect(createRes.status).toBe(201);
|
|
149
180
|
const created = (await createRes.json()) as Record<string, unknown>;
|
|
150
181
|
expect(created.status).toBe("PENDING");
|
|
182
|
+
expect(created.pauseBetweenPhases).toBe(false);
|
|
183
|
+
const id = created.id as string;
|
|
184
|
+
|
|
185
|
+
// It settles at MANIFEST_FILE_READY without any build instruction.
|
|
186
|
+
const built = await pollUntil(
|
|
187
|
+
id,
|
|
188
|
+
(s) => TERMINAL_STATUSES.includes(s),
|
|
189
|
+
90_000,
|
|
190
|
+
);
|
|
191
|
+
expect(built.status).toBe("MANIFEST_FILE_READY");
|
|
192
|
+
|
|
193
|
+
// The manifest carries the self-assigned physical table name (from
|
|
194
|
+
// `#@ persist name="order_summary"`).
|
|
195
|
+
const manifest = built.manifest as Record<string, unknown>;
|
|
196
|
+
expect(manifest).toBeDefined();
|
|
197
|
+
const entries = manifest.entries as Record<
|
|
198
|
+
string,
|
|
199
|
+
Record<string, unknown>
|
|
200
|
+
>;
|
|
201
|
+
const values = Object.values(entries);
|
|
202
|
+
expect(values.length).toBe(1);
|
|
203
|
+
expect(values[0].physicalTableName).toBe("order_summary");
|
|
204
|
+
expect(values[0].sourceName).toBe("order_summary");
|
|
205
|
+
expect(values[0].realization).toBe("COPY");
|
|
206
|
+
|
|
207
|
+
// Cleanup: delete the record and drop the self-built table.
|
|
208
|
+
const deleteRes = await fetch(
|
|
209
|
+
url(`/materializations/${id}?dropTables=true`),
|
|
210
|
+
{ method: "DELETE" },
|
|
211
|
+
);
|
|
212
|
+
expect(deleteRes.status).toBe(204);
|
|
213
|
+
},
|
|
214
|
+
{ timeout: 120_000 },
|
|
215
|
+
);
|
|
216
|
+
|
|
217
|
+
it("rejects action=build on an auto-run materialization (409)", async () => {
|
|
218
|
+
// Force a pause so we can observe a BUILD_PLAN_READY record, but mark it
|
|
219
|
+
// auto-run via metadata by creating a real auto-run and racing build is
|
|
220
|
+
// flaky; instead assert the guard on a freshly created auto-run that we
|
|
221
|
+
// immediately try to build. Auto-run never needs Round 2.
|
|
222
|
+
const createRes = await createMaterialization({
|
|
223
|
+
pauseBetweenPhases: false,
|
|
224
|
+
});
|
|
225
|
+
expect(createRes.status).toBe(201);
|
|
226
|
+
const { id } = (await createRes.json()) as { id: string };
|
|
227
|
+
|
|
228
|
+
// Drive to terminal so there is no in-flight round, then assert build
|
|
229
|
+
// is rejected (auto-run records never accept action=build).
|
|
230
|
+
await pollUntil(id, (s) => TERMINAL_STATUSES.includes(s), 90_000);
|
|
231
|
+
|
|
232
|
+
const buildRes = await fetch(
|
|
233
|
+
url(`/materializations/${id}?action=build`),
|
|
234
|
+
{
|
|
235
|
+
method: "POST",
|
|
236
|
+
headers: { "Content-Type": "application/json" },
|
|
237
|
+
body: JSON.stringify({
|
|
238
|
+
sources: [
|
|
239
|
+
{
|
|
240
|
+
buildId: "x",
|
|
241
|
+
materializedTableId: "mt",
|
|
242
|
+
physicalTableName: "t",
|
|
243
|
+
realization: "COPY",
|
|
244
|
+
},
|
|
245
|
+
],
|
|
246
|
+
}),
|
|
247
|
+
},
|
|
248
|
+
);
|
|
249
|
+
expect(buildRes.status).toBe(409);
|
|
250
|
+
|
|
251
|
+
await fetch(url(`/materializations/${id}?dropTables=true`), {
|
|
252
|
+
method: "DELETE",
|
|
253
|
+
});
|
|
254
|
+
});
|
|
255
|
+
});
|
|
256
|
+
|
|
257
|
+
// ── Group A: Full two-round lifecycle (happy path) ────────────────
|
|
258
|
+
|
|
259
|
+
describe("full two-round lifecycle", () => {
|
|
260
|
+
it(
|
|
261
|
+
"plans (round 1), builds on control-plane instruction (round 2), then deletes",
|
|
262
|
+
async () => {
|
|
263
|
+
// Round 1: create kicks off compile + plan in the background.
|
|
264
|
+
const createRes = await createMaterialization();
|
|
265
|
+
expect(createRes.status).toBe(201);
|
|
266
|
+
const created = (await createRes.json()) as Record<string, unknown>;
|
|
267
|
+
expect(created.status).toBe("PENDING");
|
|
151
268
|
expect(created.id).toBeDefined();
|
|
152
|
-
|
|
269
|
+
const id = created.id as string;
|
|
153
270
|
|
|
154
|
-
//
|
|
271
|
+
// List should include the new run.
|
|
155
272
|
const listRes = await fetch(url("/materializations"));
|
|
156
273
|
expect(listRes.status).toBe(200);
|
|
157
274
|
const list = (await listRes.json()) as Record<string, unknown>[];
|
|
158
|
-
expect(list.some((m) => m.id ===
|
|
275
|
+
expect(list.some((m) => m.id === id)).toBe(true);
|
|
159
276
|
|
|
160
|
-
//
|
|
161
|
-
const
|
|
162
|
-
|
|
277
|
+
// Round 1 completes at BUILD_PLAN_READY with a plan for our source.
|
|
278
|
+
const planned = await pollUntil(
|
|
279
|
+
id,
|
|
280
|
+
(s) => s === "BUILD_PLAN_READY" || TERMINAL_STATUSES.includes(s),
|
|
163
281
|
);
|
|
164
|
-
expect(
|
|
165
|
-
const
|
|
166
|
-
expect(
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
282
|
+
expect(planned.status).toBe("BUILD_PLAN_READY");
|
|
283
|
+
const source = firstPlannedSource(planned);
|
|
284
|
+
expect(source.name).toBe("order_summary");
|
|
285
|
+
expect(typeof source.buildId).toBe("string");
|
|
286
|
+
|
|
287
|
+
// Round 2: control plane instructs a COPY build into a physical name.
|
|
288
|
+
const physicalTableName = "order_summary_built";
|
|
289
|
+
const buildRes = await fetch(
|
|
290
|
+
url(`/materializations/${id}?action=build`),
|
|
291
|
+
{
|
|
292
|
+
method: "POST",
|
|
293
|
+
headers: { "Content-Type": "application/json" },
|
|
294
|
+
body: JSON.stringify({
|
|
295
|
+
sources: [
|
|
296
|
+
{
|
|
297
|
+
buildId: source.buildId,
|
|
298
|
+
sourceID: source.sourceID,
|
|
299
|
+
materializedTableId: "mt-order-summary",
|
|
300
|
+
physicalTableName,
|
|
301
|
+
realization: "COPY",
|
|
302
|
+
},
|
|
303
|
+
],
|
|
304
|
+
}),
|
|
305
|
+
},
|
|
172
306
|
);
|
|
173
|
-
expect(
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
const
|
|
185
|
-
expect(
|
|
186
|
-
const
|
|
187
|
-
string,
|
|
188
|
-
unknown
|
|
189
|
-
>;
|
|
190
|
-
expect(manifest.entries).toBeDefined();
|
|
191
|
-
const entries = manifest.entries as Record<string, unknown>;
|
|
192
|
-
expect(Object.keys(entries).length).toBeGreaterThan(0);
|
|
193
|
-
const firstEntry = Object.values(entries)[0] as Record<
|
|
307
|
+
expect(buildRes.status).toBe(202);
|
|
308
|
+
|
|
309
|
+
// Round 2 completes at MANIFEST_FILE_READY with an inline manifest.
|
|
310
|
+
const built = await pollUntil(
|
|
311
|
+
id,
|
|
312
|
+
(s) =>
|
|
313
|
+
s === "MANIFEST_FILE_READY" ||
|
|
314
|
+
s === "FAILED" ||
|
|
315
|
+
s === "CANCELLED",
|
|
316
|
+
);
|
|
317
|
+
expect(built.status).toBe("MANIFEST_FILE_READY");
|
|
318
|
+
const manifest = built.manifest as Record<string, unknown>;
|
|
319
|
+
expect(manifest).toBeDefined();
|
|
320
|
+
const entries = manifest.entries as Record<
|
|
194
321
|
string,
|
|
195
|
-
unknown
|
|
322
|
+
Record<string, unknown>
|
|
196
323
|
>;
|
|
197
|
-
|
|
324
|
+
const entry = entries[source.buildId as string];
|
|
325
|
+
expect(entry).toBeDefined();
|
|
326
|
+
expect(entry.physicalTableName).toBe(physicalTableName);
|
|
327
|
+
expect(entry.sourceName).toBe("order_summary");
|
|
328
|
+
|
|
329
|
+
// A terminal materialization can be deleted; dropTables=true also
|
|
330
|
+
// drops the physical table this run produced in Round 2.
|
|
331
|
+
const deleteRes = await fetch(
|
|
332
|
+
url(`/materializations/${id}?dropTables=true`),
|
|
333
|
+
{ method: "DELETE" },
|
|
334
|
+
);
|
|
335
|
+
expect(deleteRes.status).toBe(204);
|
|
198
336
|
|
|
199
|
-
//
|
|
200
|
-
const
|
|
337
|
+
// It's gone.
|
|
338
|
+
const getRes = await fetch(url(`/materializations/${id}`));
|
|
339
|
+
expect(getRes.status).toBe(404);
|
|
340
|
+
},
|
|
341
|
+
{ timeout: 90_000 },
|
|
342
|
+
);
|
|
343
|
+
});
|
|
344
|
+
|
|
345
|
+
// ── Group A1: Serve-time routing (the v0 payoff) ─────────────────
|
|
346
|
+
//
|
|
347
|
+
// Materialization only pays off if *served* queries scan the materialized
|
|
348
|
+
// table instead of recomputing from the base table. The auto-run path builds
|
|
349
|
+
// the table and auto-loads the manifest into the serving models in one
|
|
350
|
+
// process, so routing is provable in-process: capture the live SQL (scans the
|
|
351
|
+
// base CSV), run an auto-run materialization, then assert both the executed
|
|
352
|
+
// query SQL and the /compile preview SQL now scan the physical table and no
|
|
353
|
+
// longer touch the base CSV. The /compile half guards the parity fix
|
|
354
|
+
// (Environment.compileSource threads the package's bound manifest).
|
|
355
|
+
describe("serve-time routing (auto-load)", () => {
|
|
356
|
+
const MODEL_PATH = "persist_test.malloy";
|
|
357
|
+
const QUERY = "run: order_summary -> { aggregate: c is count() }";
|
|
358
|
+
|
|
359
|
+
async function executedSql(): Promise<string> {
|
|
360
|
+
const res = await fetch(
|
|
361
|
+
`${baseUrl}${API}/models/${MODEL_PATH}/query`,
|
|
362
|
+
{
|
|
201
363
|
method: "POST",
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
364
|
+
headers: { "Content-Type": "application/json" },
|
|
365
|
+
body: JSON.stringify({ query: QUERY }),
|
|
366
|
+
},
|
|
367
|
+
);
|
|
368
|
+
expect(res.status).toBe(200);
|
|
369
|
+
const body = (await res.json()) as { result: string };
|
|
370
|
+
return (JSON.parse(body.result) as { sql: string }).sql;
|
|
371
|
+
}
|
|
209
372
|
|
|
210
|
-
|
|
373
|
+
async function compiledSql(): Promise<string> {
|
|
374
|
+
const res = await fetch(
|
|
375
|
+
`${baseUrl}${API}/models/${MODEL_PATH}/compile`,
|
|
376
|
+
{
|
|
377
|
+
method: "POST",
|
|
378
|
+
headers: { "Content-Type": "application/json" },
|
|
379
|
+
body: JSON.stringify({ source: QUERY, includeSql: true }),
|
|
380
|
+
},
|
|
381
|
+
);
|
|
382
|
+
expect(res.status).toBe(200);
|
|
383
|
+
const body = (await res.json()) as { status: string; sql?: string };
|
|
384
|
+
expect(body.status).toBe("success");
|
|
385
|
+
expect(body.sql).toBeDefined();
|
|
386
|
+
return body.sql as string;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
it(
|
|
390
|
+
"routes served + compiled queries to the materialized table after auto-load",
|
|
391
|
+
async () => {
|
|
392
|
+
// Reset to live: a prior group may have left an in-memory binding.
|
|
393
|
+
// Auto-load never writes manifestLocation to publisher.json, so a
|
|
394
|
+
// reload re-reads the live model.
|
|
395
|
+
await fetch(`${baseUrl}${API}?reload=true`);
|
|
396
|
+
|
|
397
|
+
// Baseline: with nothing materialized, both paths recompute from the
|
|
398
|
+
// base CSV.
|
|
399
|
+
expect(await executedSql()).toContain("data/orders.csv");
|
|
400
|
+
expect(await compiledSql()).toContain("data/orders.csv");
|
|
401
|
+
|
|
402
|
+
// Build + auto-load in one pass (self-assigns physicalTableName =
|
|
403
|
+
// "order_summary" from `#@ persist name="order_summary"`).
|
|
404
|
+
const createRes = await createMaterialization({
|
|
405
|
+
pauseBetweenPhases: false,
|
|
406
|
+
});
|
|
407
|
+
expect(createRes.status).toBe(201);
|
|
408
|
+
const { id } = (await createRes.json()) as { id: string };
|
|
409
|
+
const built = await pollUntil(
|
|
410
|
+
id,
|
|
411
|
+
(s) => TERMINAL_STATUSES.includes(s),
|
|
412
|
+
90_000,
|
|
413
|
+
);
|
|
414
|
+
expect(built.status).toBe("MANIFEST_FILE_READY");
|
|
415
|
+
|
|
416
|
+
// The payoff: the served query now scans the materialized table and
|
|
417
|
+
// no longer recomputes from the base CSV, and /compile previews the
|
|
418
|
+
// same routed SQL it would execute.
|
|
419
|
+
const routedExecuted = await executedSql();
|
|
420
|
+
const routedCompiled = await compiledSql();
|
|
421
|
+
expect(routedExecuted).not.toContain("data/orders.csv");
|
|
422
|
+
expect(routedExecuted).toContain("order_summary");
|
|
423
|
+
expect(routedCompiled).not.toContain("data/orders.csv");
|
|
424
|
+
expect(routedCompiled).toContain("order_summary");
|
|
425
|
+
|
|
426
|
+
// Cleanup: drop the table + record, then reload back to live so the
|
|
427
|
+
// dangling binding doesn't leak into later groups.
|
|
211
428
|
const deleteRes = await fetch(
|
|
212
|
-
url(`/materializations/${
|
|
429
|
+
url(`/materializations/${id}?dropTables=true`),
|
|
213
430
|
{ method: "DELETE" },
|
|
214
431
|
);
|
|
215
432
|
expect(deleteRes.status).toBe(204);
|
|
216
|
-
|
|
433
|
+
await fetch(`${baseUrl}${API}?reload=true`);
|
|
217
434
|
},
|
|
218
|
-
{ timeout:
|
|
435
|
+
{ timeout: 120_000 },
|
|
219
436
|
);
|
|
220
437
|
});
|
|
221
438
|
|
|
222
|
-
// ── Group B:
|
|
439
|
+
// ── Group B: State machine and error cases ───────────────────────
|
|
223
440
|
|
|
224
|
-
describe("
|
|
225
|
-
it("
|
|
441
|
+
describe("state machine and errors", () => {
|
|
442
|
+
it("stops a plan-ready materialization (-> CANCELLED)", async () => {
|
|
226
443
|
const createRes = await createMaterialization();
|
|
227
444
|
expect(createRes.status).toBe(201);
|
|
228
|
-
const
|
|
229
|
-
|
|
445
|
+
const { id } = (await createRes.json()) as { id: string };
|
|
446
|
+
|
|
447
|
+
await pollUntil(id, (s) => s === "BUILD_PLAN_READY");
|
|
230
448
|
|
|
231
449
|
const stopRes = await fetch(
|
|
232
450
|
url(`/materializations/${id}?action=stop`),
|
|
@@ -241,230 +459,117 @@ describe("Materialization & Manifest REST API (E2E)", () => {
|
|
|
241
459
|
await cleanup(id);
|
|
242
460
|
});
|
|
243
461
|
|
|
244
|
-
it("
|
|
462
|
+
it("rejects a second concurrent materialization with 409", async () => {
|
|
245
463
|
const first = await createMaterialization();
|
|
246
464
|
expect(first.status).toBe(201);
|
|
247
|
-
const
|
|
248
|
-
const firstId = firstData.id as string;
|
|
465
|
+
const { id } = (await first.json()) as { id: string };
|
|
249
466
|
|
|
250
467
|
const second = await createMaterialization();
|
|
251
468
|
expect(second.status).toBe(409);
|
|
252
469
|
|
|
253
|
-
await cleanup(
|
|
470
|
+
await cleanup(id);
|
|
254
471
|
});
|
|
255
472
|
|
|
256
|
-
it("
|
|
473
|
+
it("rejects building from a non-plan-ready state with 409", async () => {
|
|
257
474
|
const createRes = await createMaterialization();
|
|
258
475
|
expect(createRes.status).toBe(201);
|
|
259
|
-
const
|
|
260
|
-
const id = created.id as string;
|
|
476
|
+
const { id } = (await createRes.json()) as { id: string };
|
|
261
477
|
|
|
478
|
+
await pollUntil(id, (s) => s === "BUILD_PLAN_READY");
|
|
262
479
|
await fetch(url(`/materializations/${id}?action=stop`), {
|
|
263
480
|
method: "POST",
|
|
264
481
|
});
|
|
482
|
+
await pollUntil(id, (s) => s === "CANCELLED");
|
|
265
483
|
|
|
266
|
-
const
|
|
267
|
-
url(`/materializations/${id}?action=
|
|
484
|
+
const buildRes = await fetch(
|
|
485
|
+
url(`/materializations/${id}?action=build`),
|
|
268
486
|
{
|
|
269
487
|
method: "POST",
|
|
488
|
+
headers: { "Content-Type": "application/json" },
|
|
489
|
+
body: JSON.stringify({
|
|
490
|
+
sources: [
|
|
491
|
+
{
|
|
492
|
+
buildId: "deadbeef",
|
|
493
|
+
materializedTableId: "mt",
|
|
494
|
+
physicalTableName: "t",
|
|
495
|
+
realization: "COPY",
|
|
496
|
+
},
|
|
497
|
+
],
|
|
498
|
+
}),
|
|
270
499
|
},
|
|
271
500
|
);
|
|
272
|
-
expect(
|
|
501
|
+
expect(buildRes.status).toBe(409);
|
|
273
502
|
|
|
274
|
-
await
|
|
503
|
+
await fetch(url(`/materializations/${id}`), { method: "DELETE" });
|
|
275
504
|
});
|
|
276
505
|
|
|
277
|
-
it("
|
|
506
|
+
it("rejects a build instruction with an unknown buildId (400)", async () => {
|
|
278
507
|
const createRes = await createMaterialization();
|
|
279
508
|
expect(createRes.status).toBe(201);
|
|
280
|
-
const
|
|
281
|
-
const id = created.id as string;
|
|
509
|
+
const { id } = (await createRes.json()) as { id: string };
|
|
282
510
|
|
|
283
|
-
|
|
284
|
-
method: "DELETE",
|
|
285
|
-
});
|
|
286
|
-
expect(deleteRes.status).toBe(409);
|
|
287
|
-
|
|
288
|
-
await cleanup(id);
|
|
289
|
-
});
|
|
290
|
-
|
|
291
|
-
it("should return 404 for a non-existent materialization", async () => {
|
|
292
|
-
const res = await fetch(
|
|
293
|
-
url("/materializations/non-existent-id-12345"),
|
|
294
|
-
);
|
|
295
|
-
expect(res.status).toBe(404);
|
|
296
|
-
});
|
|
297
|
-
});
|
|
298
|
-
|
|
299
|
-
// ── Group C: Package Teardown ────────────────────────────────────
|
|
300
|
-
|
|
301
|
-
describe("package teardown", () => {
|
|
302
|
-
it(
|
|
303
|
-
"dryRun reports stale entries without dropping tables or deleting rows",
|
|
304
|
-
async () => {
|
|
305
|
-
// Run a full build so there are manifest entries to tear down.
|
|
306
|
-
const createRes = await createMaterialization({
|
|
307
|
-
autoLoadManifest: true,
|
|
308
|
-
});
|
|
309
|
-
expect(createRes.status).toBe(201);
|
|
310
|
-
const created = (await createRes.json()) as Record<string, unknown>;
|
|
311
|
-
const id = created.id as string;
|
|
312
|
-
await fetch(url(`/materializations/${id}?action=start`), {
|
|
313
|
-
method: "POST",
|
|
314
|
-
});
|
|
315
|
-
const terminal = await pollUntilTerminal(id);
|
|
316
|
-
expect(terminal.status).toBe("SUCCESS");
|
|
511
|
+
await pollUntil(id, (s) => s === "BUILD_PLAN_READY");
|
|
317
512
|
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
// dryRun teardown — should report entries but not actually drop them.
|
|
323
|
-
const teardownRes = await fetch(url("/materializations/teardown"), {
|
|
513
|
+
const buildRes = await fetch(
|
|
514
|
+
url(`/materializations/${id}?action=build`),
|
|
515
|
+
{
|
|
324
516
|
method: "POST",
|
|
325
517
|
headers: { "Content-Type": "application/json" },
|
|
326
|
-
body: JSON.stringify({
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
expect(manifestRes.status).toBe(200);
|
|
340
|
-
const manifest = (await manifestRes.json()) as Record<
|
|
341
|
-
string,
|
|
342
|
-
unknown
|
|
343
|
-
>;
|
|
344
|
-
const entries = manifest.entries as Record<string, unknown>;
|
|
345
|
-
expect(Object.keys(entries).length).toBeGreaterThan(0);
|
|
346
|
-
},
|
|
347
|
-
{ timeout: 60_000 },
|
|
348
|
-
);
|
|
349
|
-
|
|
350
|
-
it(
|
|
351
|
-
"live teardown drops stale manifest entries and cleans up tables",
|
|
352
|
-
async () => {
|
|
353
|
-
// Build so there are manifest entries.
|
|
354
|
-
const createRes = await createMaterialization({
|
|
355
|
-
autoLoadManifest: true,
|
|
356
|
-
});
|
|
357
|
-
expect(createRes.status).toBe(201);
|
|
358
|
-
const created = (await createRes.json()) as Record<string, unknown>;
|
|
359
|
-
const id = created.id as string;
|
|
360
|
-
await fetch(url(`/materializations/${id}?action=start`), {
|
|
361
|
-
method: "POST",
|
|
362
|
-
});
|
|
363
|
-
const terminal = await pollUntilTerminal(id);
|
|
364
|
-
expect(terminal.status).toBe("SUCCESS");
|
|
365
|
-
|
|
366
|
-
await fetch(url(`/materializations/${id}`), { method: "DELETE" });
|
|
518
|
+
body: JSON.stringify({
|
|
519
|
+
sources: [
|
|
520
|
+
{
|
|
521
|
+
buildId: "not-a-real-build-id",
|
|
522
|
+
materializedTableId: "mt",
|
|
523
|
+
physicalTableName: "t",
|
|
524
|
+
realization: "COPY",
|
|
525
|
+
},
|
|
526
|
+
],
|
|
527
|
+
}),
|
|
528
|
+
},
|
|
529
|
+
);
|
|
530
|
+
expect(buildRes.status).toBe(400);
|
|
367
531
|
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
const teardownRes = await fetch(url("/materializations/teardown"), {
|
|
371
|
-
method: "POST",
|
|
372
|
-
headers: { "Content-Type": "application/json" },
|
|
373
|
-
body: JSON.stringify({}),
|
|
374
|
-
});
|
|
375
|
-
expect(teardownRes.status).toBe(200);
|
|
376
|
-
const teardownResult = (await teardownRes.json()) as Record<
|
|
377
|
-
string,
|
|
378
|
-
unknown
|
|
379
|
-
>;
|
|
380
|
-
const dropped = teardownResult.dropped as Record<string, unknown>[];
|
|
381
|
-
expect(dropped.length).toBeGreaterThan(0);
|
|
382
|
-
expect((teardownResult.errors as unknown[]).length).toBe(0);
|
|
383
|
-
|
|
384
|
-
// Manifest should be empty after live teardown.
|
|
385
|
-
const manifestRes = await fetch(url("/manifest"));
|
|
386
|
-
expect(manifestRes.status).toBe(200);
|
|
387
|
-
const manifest = (await manifestRes.json()) as Record<
|
|
388
|
-
string,
|
|
389
|
-
unknown
|
|
390
|
-
>;
|
|
391
|
-
const entries = manifest.entries as Record<string, unknown>;
|
|
392
|
-
expect(Object.keys(entries).length).toBe(0);
|
|
393
|
-
},
|
|
394
|
-
{ timeout: 60_000 },
|
|
395
|
-
);
|
|
532
|
+
await cleanup(id);
|
|
533
|
+
});
|
|
396
534
|
|
|
397
|
-
it("
|
|
535
|
+
it("rejects deleting a non-terminal materialization with 409", async () => {
|
|
398
536
|
const createRes = await createMaterialization();
|
|
399
537
|
expect(createRes.status).toBe(201);
|
|
400
|
-
const
|
|
401
|
-
const id = created.id as string;
|
|
538
|
+
const { id } = (await createRes.json()) as { id: string };
|
|
402
539
|
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
540
|
+
await pollUntil(id, (s) => s === "BUILD_PLAN_READY");
|
|
541
|
+
|
|
542
|
+
const deleteRes = await fetch(url(`/materializations/${id}`), {
|
|
543
|
+
method: "DELETE",
|
|
407
544
|
});
|
|
408
|
-
expect(
|
|
545
|
+
expect(deleteRes.status).toBe(409);
|
|
409
546
|
|
|
410
547
|
await cleanup(id);
|
|
411
548
|
});
|
|
412
549
|
|
|
413
|
-
it(
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
const first = await createMaterialization({
|
|
418
|
-
autoLoadManifest: true,
|
|
419
|
-
});
|
|
420
|
-
expect(first.status).toBe(201);
|
|
421
|
-
const firstData = (await first.json()) as Record<string, unknown>;
|
|
422
|
-
const firstId = firstData.id as string;
|
|
423
|
-
await fetch(url(`/materializations/${firstId}?action=start`), {
|
|
424
|
-
method: "POST",
|
|
425
|
-
});
|
|
426
|
-
const firstTerminal = await pollUntilTerminal(firstId);
|
|
427
|
-
expect(firstTerminal.status).toBe("SUCCESS");
|
|
428
|
-
await fetch(url(`/materializations/${firstId}`), {
|
|
429
|
-
method: "DELETE",
|
|
430
|
-
});
|
|
550
|
+
it("rejects an unsupported action with 400", async () => {
|
|
551
|
+
const createRes = await createMaterialization();
|
|
552
|
+
expect(createRes.status).toBe(201);
|
|
553
|
+
const { id } = (await createRes.json()) as { id: string };
|
|
431
554
|
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
autoLoadManifest: true,
|
|
438
|
-
});
|
|
439
|
-
expect(second.status).toBe(201);
|
|
440
|
-
const secondData = (await second.json()) as Record<string, unknown>;
|
|
441
|
-
const secondId = secondData.id as string;
|
|
442
|
-
await fetch(url(`/materializations/${secondId}?action=start`), {
|
|
555
|
+
await pollUntil(id, (s) => s === "BUILD_PLAN_READY");
|
|
556
|
+
|
|
557
|
+
const res = await fetch(
|
|
558
|
+
url(`/materializations/${id}?action=frobnicate`),
|
|
559
|
+
{
|
|
443
560
|
method: "POST",
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
const metadata = secondTerminal.metadata as Record<string, unknown>;
|
|
449
|
-
// forceRefresh should actually rebuild, not skip.
|
|
450
|
-
expect(metadata.sourcesBuilt).toBeGreaterThan(0);
|
|
451
|
-
expect(metadata.sourcesSkipped).toBe(0);
|
|
452
|
-
// Post-build GC step ran (arrays present even if empty).
|
|
453
|
-
expect(metadata.gcDropped).toBeDefined();
|
|
454
|
-
expect(metadata.gcErrors).toBeDefined();
|
|
455
|
-
|
|
456
|
-
// Manifest should still have entries after rebuild.
|
|
457
|
-
const manifestRes = await fetch(url("/manifest"));
|
|
458
|
-
const manifest = (await manifestRes.json()) as Record<
|
|
459
|
-
string,
|
|
460
|
-
unknown
|
|
461
|
-
>;
|
|
462
|
-
const entries = manifest.entries as Record<string, unknown>;
|
|
463
|
-
expect(Object.keys(entries).length).toBeGreaterThan(0);
|
|
561
|
+
},
|
|
562
|
+
);
|
|
563
|
+
expect(res.status).toBe(400);
|
|
464
564
|
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
)
|
|
565
|
+
await cleanup(id);
|
|
566
|
+
});
|
|
567
|
+
|
|
568
|
+
it("returns 404 for a non-existent materialization", async () => {
|
|
569
|
+
const res = await fetch(
|
|
570
|
+
url("/materializations/non-existent-id-12345"),
|
|
571
|
+
);
|
|
572
|
+
expect(res.status).toBe(404);
|
|
573
|
+
});
|
|
469
574
|
});
|
|
470
575
|
});
|