@malloy-publisher/server 0.0.227 → 0.0.229
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.docker.md +8 -8
- package/dist/app/api-doc.yaml +84 -1
- package/dist/app/assets/EnvironmentPage-QOoHiVeJ.js +1 -0
- package/dist/app/assets/{HomePage-CXguJsXS.js → HomePage-C71GOfVW.js} +1 -1
- package/dist/app/assets/{LightMode-ZsshUznu.js → LightMode-CrgCAwLe.js} +1 -1
- package/dist/app/assets/MainPage-BG5__FN3.js +2 -0
- package/dist/app/assets/MaterializationsPage-DE6PnrDR.js +1 -0
- package/dist/app/assets/ModelPage-CcBjcbLm.js +1 -0
- package/dist/app/assets/PackagePage-JTy3ztkB.js +1 -0
- package/dist/app/assets/{RouteError-Chn7lL96.js → RouteError-Cymbp47a.js} +1 -1
- package/dist/app/assets/ThemeEditorPage-C_nMnHr8.js +1 -0
- package/dist/app/assets/WorkbookPage-CPQu-DQx.js +1 -0
- package/dist/app/assets/{core-vVgoO8IR.es-BD_THWs_.js → core-Coi3caGs.es-CSOmajHS.js} +10 -10
- package/dist/app/assets/{index-gEWxu09x.js → index-CM2qhQCI.js} +1 -1
- package/dist/app/assets/{index-D6YtyiJ0.js → index-CcuuST2X.js} +1 -1
- package/dist/app/assets/{index-OEjKNSYb.js → index-CkmABCAw.js} +388 -241
- package/dist/app/assets/{index-BioohWQj.js → index-DlWCXghy.js} +1 -1
- package/dist/app/assets/index-DxArlgRD.js +529 -0
- package/dist/app/index.html +1 -1
- package/dist/default-publisher.config.json +7 -7
- package/dist/instrumentation.mjs +1 -1
- package/dist/package_load_worker.mjs +1 -1
- package/dist/runtime/publisher.js +5 -0
- package/dist/server.mjs +7675 -899
- package/package.json +4 -2
- package/publisher.config.example.bigquery.json +7 -7
- package/publisher.config.example.duckdb.json +7 -7
- package/publisher.config.json +7 -11
- package/src/config.spec.ts +89 -3
- package/src/config.ts +102 -1
- package/src/controller/materialization.controller.spec.ts +9 -0
- package/src/controller/materialization.controller.ts +15 -0
- package/src/controller/package.controller.ts +62 -31
- package/src/default-publisher.config.json +7 -7
- package/src/materialization_metrics.ts +33 -0
- package/src/mcp/handler_utils.spec.ts +108 -0
- package/src/mcp/handler_utils.ts +98 -4
- package/src/mcp/server.protocol.spec.ts +58 -0
- package/src/mcp/server.ts +29 -1
- package/src/mcp/skills/build_skills_bundle.ts +29 -9
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/skills/skills_bundle.spec.ts +31 -2
- package/src/mcp/tools/compile_tool.spec.ts +207 -0
- package/src/mcp/tools/compile_tool.ts +177 -0
- package/src/mcp/tools/execute_query_tool.spec.ts +143 -0
- package/src/mcp/tools/execute_query_tool.ts +37 -4
- package/src/mcp/tools/get_context_tool.ts +1 -1
- package/src/mcp/tools/reload_package_tool.spec.ts +232 -0
- package/src/mcp/tools/reload_package_tool.ts +158 -0
- package/src/query_param_utils.ts +11 -0
- package/src/runtime/publisher.js +5 -0
- package/src/server.ts +74 -14
- package/src/service/connection.spec.ts +8 -7
- package/src/service/cron_evaluator.spec.ts +79 -0
- package/src/service/cron_evaluator.ts +105 -0
- package/src/service/environment.ts +188 -19
- package/src/service/environment_store.spec.ts +372 -2
- package/src/service/environment_store.ts +149 -17
- package/src/service/environment_store_anchoring.spec.ts +107 -0
- package/src/service/manifest_loader.spec.ts +3 -1
- package/src/service/manifest_loader.ts +9 -5
- package/src/service/materialization_scheduler.spec.ts +260 -0
- package/src/service/materialization_scheduler.ts +304 -0
- package/src/service/materialization_service.spec.ts +167 -4
- package/src/service/materialization_service.ts +121 -8
- package/src/service/materialization_test_fixtures.ts +10 -1
- package/src/service/model.spec.ts +92 -0
- package/src/service/model.ts +58 -7
- package/src/service/package.ts +121 -14
- package/src/service/package_quote_bound_tables.spec.ts +171 -0
- package/src/service/package_reload_safety.spec.ts +193 -0
- package/src/service/persistence_policy.spec.ts +31 -0
- package/src/service/quoting.spec.ts +57 -1
- package/src/service/quoting.ts +33 -0
- package/src/storage/DatabaseInterface.ts +15 -0
- package/src/storage/duckdb/DuckDBRepository.ts +17 -0
- package/src/storage/duckdb/MaterializationRepository.spec.ts +154 -0
- package/src/storage/duckdb/MaterializationRepository.ts +56 -0
- package/tests/fixtures/persist-schedule-test/data/orders.csv +5 -0
- package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +11 -0
- package/tests/fixtures/persist-schedule-test/publisher.json +7 -0
- package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +5 -0
- package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +11 -0
- package/tests/fixtures/persist-schedule-warn-test/publisher.json +6 -0
- package/tests/fixtures/query-givens/data/orders.csv +7 -0
- package/tests/fixtures/query-givens/model.malloy +34 -0
- package/tests/fixtures/query-givens/publisher.json +5 -0
- package/tests/integration/materialization/environment_list.integration.spec.ts +133 -0
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +7 -2
- package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +150 -0
- package/tests/integration/materialization/schedule_edit.integration.spec.ts +126 -0
- package/tests/integration/materialization/scheduler.integration.spec.ts +200 -0
- package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +172 -0
- package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +143 -0
- package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +22 -22
- package/tests/integration/query_givens/query_givens.integration.spec.ts +146 -0
- package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +121 -0
- package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +110 -0
- package/dist/app/assets/EnvironmentPage-DvOJ7L_b.js +0 -1
- package/dist/app/assets/MainPage-BIe0VwBa.js +0 -2
- package/dist/app/assets/MaterializationsPage-BuZ6UJVx.js +0 -1
- package/dist/app/assets/ModelPage-DsPf-s8B.js +0 -1
- package/dist/app/assets/PackagePage-CEVNAKZa.js +0 -1
- package/dist/app/assets/ThemeEditorPage-DWC_FdNU.js +0 -1
- package/dist/app/assets/WorkbookPage-CGrsFz8p.js +0 -1
- package/dist/app/assets/index-DNUZpnaa.js +0 -527
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it } from "bun:test";
|
|
2
|
+
import { DuckDBConnection } from "./DuckDBConnection";
|
|
3
|
+
import { MaterializationRepository } from "./MaterializationRepository";
|
|
4
|
+
import { initializeSchema } from "./schema";
|
|
5
|
+
|
|
6
|
+
// Capture the SQL + params of the last query without touching a real DuckDB.
|
|
7
|
+
function fakeRepo() {
|
|
8
|
+
const calls: Array<{ sql: string; params: unknown[] }> = [];
|
|
9
|
+
const db = {
|
|
10
|
+
all: async (sql: string, params: unknown[]) => {
|
|
11
|
+
calls.push({ sql, params });
|
|
12
|
+
return [];
|
|
13
|
+
},
|
|
14
|
+
} as unknown as DuckDBConnection;
|
|
15
|
+
return { repo: new MaterializationRepository(db), calls };
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
describe("MaterializationRepository list bounds", () => {
|
|
19
|
+
it("per-package list is unbounded when no limit is given (no LIMIT clause)", async () => {
|
|
20
|
+
const { repo, calls } = fakeRepo();
|
|
21
|
+
await repo.list("env-1", "pkg-a");
|
|
22
|
+
expect(calls[0].sql).not.toContain("LIMIT");
|
|
23
|
+
expect(calls[0].sql).not.toContain("OFFSET");
|
|
24
|
+
expect(calls[0].params).toEqual(["env-1", "pkg-a"]);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
it("per-package list honors an explicit limit/offset", async () => {
|
|
28
|
+
const { repo, calls } = fakeRepo();
|
|
29
|
+
await repo.list("env-1", "pkg-a", { limit: 5, offset: 10 });
|
|
30
|
+
expect(calls[0].sql).toContain("LIMIT ?");
|
|
31
|
+
expect(calls[0].sql).toContain("OFFSET ?");
|
|
32
|
+
expect(calls[0].params).toEqual(["env-1", "pkg-a", 5, 10]);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
it("environment-scoped list always applies a default cap (500)", async () => {
|
|
36
|
+
const { repo, calls } = fakeRepo();
|
|
37
|
+
await repo.listByEnvironment("env-1");
|
|
38
|
+
expect(calls[0].sql).toContain("LIMIT ?");
|
|
39
|
+
expect(calls[0].params).toEqual(["env-1", 500]);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it("environment-scoped list honors an explicit limit over the default", async () => {
|
|
43
|
+
const { repo, calls } = fakeRepo();
|
|
44
|
+
await repo.listByEnvironment("env-1", { limit: 25 });
|
|
45
|
+
expect(calls[0].params).toEqual(["env-1", 25]);
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
// The restart-recovery anchor (MaterializationScheduler.recoveryAnchor) hangs on
|
|
50
|
+
// getLatestScheduledFireAt's `json_extract_string(metadata,'$.trigger')` filter,
|
|
51
|
+
// which a fake db can't exercise. Drive the real query against an in-memory
|
|
52
|
+
// DuckDB so the trigger filter and ORDER BY are covered, not just the SQL shape.
|
|
53
|
+
describe("MaterializationRepository.getLatestScheduledFireAt (real DuckDB)", () => {
|
|
54
|
+
const ENV_ID = "env-sched";
|
|
55
|
+
const PKG = "pkg-a";
|
|
56
|
+
const dbs: DuckDBConnection[] = [];
|
|
57
|
+
|
|
58
|
+
afterEach(async () => {
|
|
59
|
+
while (dbs.length) await dbs.pop()!.close();
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
async function freshRepo(): Promise<{
|
|
63
|
+
db: DuckDBConnection;
|
|
64
|
+
repo: MaterializationRepository;
|
|
65
|
+
}> {
|
|
66
|
+
const db = new DuckDBConnection(":memory:");
|
|
67
|
+
dbs.push(db);
|
|
68
|
+
await db.initialize();
|
|
69
|
+
await initializeSchema(db);
|
|
70
|
+
// materializations.environment_id is a FK to environments(id).
|
|
71
|
+
await db.run(
|
|
72
|
+
`INSERT INTO environments (id, name, path, created_at, updated_at)
|
|
73
|
+
VALUES (?, ?, ?, ?, ?)`,
|
|
74
|
+
[
|
|
75
|
+
ENV_ID,
|
|
76
|
+
"sched-env",
|
|
77
|
+
"/tmp/sched-env",
|
|
78
|
+
"2026-01-01T00:00:00.000Z",
|
|
79
|
+
"2026-01-01T00:00:00.000Z",
|
|
80
|
+
],
|
|
81
|
+
);
|
|
82
|
+
return { db, repo: new MaterializationRepository(db) };
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Insert a terminal run directly (active_key NULL so repeated inserts for the
|
|
86
|
+
// same package don't trip the unique active-key index). `trigger === undefined`
|
|
87
|
+
// stores metadata NULL; `null` stores a trigger-less object; a string stores
|
|
88
|
+
// that trigger.
|
|
89
|
+
async function insertRun(
|
|
90
|
+
db: DuckDBConnection,
|
|
91
|
+
id: string,
|
|
92
|
+
createdAt: string,
|
|
93
|
+
trigger: string | null | undefined,
|
|
94
|
+
): Promise<void> {
|
|
95
|
+
const metadata =
|
|
96
|
+
trigger === undefined
|
|
97
|
+
? null
|
|
98
|
+
: JSON.stringify(
|
|
99
|
+
trigger === null ? { note: "manual" } : { trigger },
|
|
100
|
+
);
|
|
101
|
+
await db.run(
|
|
102
|
+
`INSERT INTO materializations
|
|
103
|
+
(id, environment_id, package_name, status, active_key, metadata, manifest, created_at, updated_at)
|
|
104
|
+
VALUES (?, ?, ?, 'MANIFEST_FILE_READY', NULL, ?, NULL, ?, ?)`,
|
|
105
|
+
[id, ENV_ID, PKG, metadata, createdAt, createdAt],
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const day = (d: string) => `2026-${d}T12:00:00.000Z`;
|
|
110
|
+
|
|
111
|
+
it("returns the created_at of the newest SCHEDULER run", async () => {
|
|
112
|
+
const { db, repo } = await freshRepo();
|
|
113
|
+
await insertRun(db, "s1", day("01-15"), "SCHEDULER");
|
|
114
|
+
await insertRun(db, "s2", day("03-15"), "SCHEDULER");
|
|
115
|
+
const at = await repo.getLatestScheduledFireAt(ENV_ID, PKG);
|
|
116
|
+
expect(at).not.toBeNull();
|
|
117
|
+
expect(at!.toISOString().slice(0, 10)).toBe("2026-03-15");
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it("ignores ON_DEMAND and trigger-less runs even when they are newer", async () => {
|
|
121
|
+
const { db, repo } = await freshRepo();
|
|
122
|
+
await insertRun(db, "s1", day("02-01"), "SCHEDULER"); // the only match
|
|
123
|
+
await insertRun(db, "d1", day("06-01"), "ON_DEMAND"); // newer, wrong trigger
|
|
124
|
+
await insertRun(db, "n1", day("07-01"), null); // newer, no trigger key
|
|
125
|
+
await insertRun(db, "m1", day("08-01"), undefined); // newer, metadata NULL
|
|
126
|
+
const at = await repo.getLatestScheduledFireAt(ENV_ID, PKG);
|
|
127
|
+
expect(at).not.toBeNull();
|
|
128
|
+
expect(at!.toISOString().slice(0, 10)).toBe("2026-02-01");
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
it("returns null when the package has no SCHEDULER run", async () => {
|
|
132
|
+
const { db, repo } = await freshRepo();
|
|
133
|
+
await insertRun(db, "d1", day("06-01"), "ON_DEMAND");
|
|
134
|
+
await insertRun(db, "m1", day("07-01"), undefined);
|
|
135
|
+
expect(await repo.getLatestScheduledFireAt(ENV_ID, PKG)).toBeNull();
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
it("scopes to the requested package (ignores another package's SCHEDULER run)", async () => {
|
|
139
|
+
const { db, repo } = await freshRepo();
|
|
140
|
+
await db.run(
|
|
141
|
+
`INSERT INTO materializations
|
|
142
|
+
(id, environment_id, package_name, status, active_key, metadata, manifest, created_at, updated_at)
|
|
143
|
+
VALUES (?, ?, 'other-pkg', 'MANIFEST_FILE_READY', NULL, ?, NULL, ?, ?)`,
|
|
144
|
+
[
|
|
145
|
+
"x1",
|
|
146
|
+
ENV_ID,
|
|
147
|
+
JSON.stringify({ trigger: "SCHEDULER" }),
|
|
148
|
+
day("09-01"),
|
|
149
|
+
day("09-01"),
|
|
150
|
+
],
|
|
151
|
+
);
|
|
152
|
+
expect(await repo.getLatestScheduledFireAt(ENV_ID, PKG)).toBeNull();
|
|
153
|
+
});
|
|
154
|
+
});
|
|
@@ -22,6 +22,15 @@ function activeKeyFor(environmentId: string, packageName: string): string {
|
|
|
22
22
|
return `${environmentId}|${packageName}`;
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
+
/**
|
|
26
|
+
* Default cap for the environment-wide materialization list. That query spans
|
|
27
|
+
* every package in the environment, so on a busy env (e.g. a frequent schedule)
|
|
28
|
+
* the history grows without bound; when a caller passes no explicit limit, fall
|
|
29
|
+
* back to this bound rather than loading and serializing every row. Callers that
|
|
30
|
+
* page (the SDK "show more") always pass their own limit and are unaffected.
|
|
31
|
+
*/
|
|
32
|
+
const DEFAULT_ENVIRONMENT_LIST_LIMIT = 500;
|
|
33
|
+
|
|
25
34
|
/**
|
|
26
35
|
* Thrown when an atomic insert loses a race on (environment, package) active
|
|
27
36
|
* materialization. Surfaced separately from a generic DB error so the service
|
|
@@ -53,6 +62,10 @@ export class MaterializationRepository {
|
|
|
53
62
|
return new Date();
|
|
54
63
|
}
|
|
55
64
|
|
|
65
|
+
// Per-package list: intentionally UNBOUNDED when no limit is passed —
|
|
66
|
+
// existing callers may rely on the full history for a single package. Only
|
|
67
|
+
// the environment-wide `listByEnvironment` (which spans every package)
|
|
68
|
+
// applies a default cap; do not add one here.
|
|
56
69
|
async list(
|
|
57
70
|
environmentId: string,
|
|
58
71
|
packageName: string,
|
|
@@ -73,6 +86,49 @@ export class MaterializationRepository {
|
|
|
73
86
|
return rows.map(this.mapRow);
|
|
74
87
|
}
|
|
75
88
|
|
|
89
|
+
/** All materializations across every package in an environment, newest first. */
|
|
90
|
+
async listByEnvironment(
|
|
91
|
+
environmentId: string,
|
|
92
|
+
options?: { limit?: number; offset?: number },
|
|
93
|
+
): Promise<Materialization[]> {
|
|
94
|
+
let sql =
|
|
95
|
+
"SELECT * FROM materializations WHERE environment_id = ? ORDER BY created_at DESC";
|
|
96
|
+
const params: unknown[] = [environmentId];
|
|
97
|
+
// Always bound the result: an uncapped env-wide fetch is unbounded.
|
|
98
|
+
sql += " LIMIT ?";
|
|
99
|
+
params.push(options?.limit ?? DEFAULT_ENVIRONMENT_LIST_LIMIT);
|
|
100
|
+
if (options?.offset !== undefined) {
|
|
101
|
+
sql += " OFFSET ?";
|
|
102
|
+
params.push(options.offset);
|
|
103
|
+
}
|
|
104
|
+
const rows = await this.db.all<Record<string, unknown>>(sql, params);
|
|
105
|
+
return rows.map(this.mapRow);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* `created_at` of the newest scheduler-fired (metadata.trigger=SCHEDULER)
|
|
110
|
+
* materialization for (env, package), or null if none. The standalone
|
|
111
|
+
* scheduler uses this to recover a missed occurrence across a restart:
|
|
112
|
+
* re-anchoring `nextFireAt` to `nextAfter(lastScheduledFire)` fires exactly
|
|
113
|
+
* one catch-up after downtime (then jumps forward), rather than skipping the
|
|
114
|
+
* occurrence entirely. `metadata` is a JSON column, so the trigger is read
|
|
115
|
+
* with `json_extract_string`.
|
|
116
|
+
*/
|
|
117
|
+
async getLatestScheduledFireAt(
|
|
118
|
+
environmentId: string,
|
|
119
|
+
packageName: string,
|
|
120
|
+
): Promise<Date | null> {
|
|
121
|
+
const row = await this.db.get<{ created_at: unknown }>(
|
|
122
|
+
`SELECT created_at FROM materializations
|
|
123
|
+
WHERE environment_id = ? AND package_name = ?
|
|
124
|
+
AND json_extract_string(metadata, '$.trigger') = 'SCHEDULER'
|
|
125
|
+
ORDER BY created_at DESC
|
|
126
|
+
LIMIT 1`,
|
|
127
|
+
[environmentId, packageName],
|
|
128
|
+
);
|
|
129
|
+
return row?.created_at ? new Date(row.created_at as string) : null;
|
|
130
|
+
}
|
|
131
|
+
|
|
76
132
|
async getById(id: string): Promise<Materialization | null> {
|
|
77
133
|
const row = await this.db.get<Record<string, unknown>>(
|
|
78
134
|
"SELECT * FROM materializations WHERE id = ?",
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
##! experimental.persistence
|
|
2
|
+
|
|
3
|
+
source: raw_orders is duckdb.table('data/orders.csv')
|
|
4
|
+
|
|
5
|
+
#@ persist name="order_summary"
|
|
6
|
+
source: order_summary is raw_orders -> {
|
|
7
|
+
group_by: category
|
|
8
|
+
aggregate:
|
|
9
|
+
total_orders is count()
|
|
10
|
+
total_revenue is amount.sum()
|
|
11
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
##! experimental.persistence
|
|
2
|
+
|
|
3
|
+
source: raw_orders is duckdb.table('data/orders.csv')
|
|
4
|
+
|
|
5
|
+
#@ persist name="order_summary"
|
|
6
|
+
source: order_summary is raw_orders -> {
|
|
7
|
+
group_by: category
|
|
8
|
+
aggregate:
|
|
9
|
+
total_orders is count()
|
|
10
|
+
total_revenue is amount.sum()
|
|
11
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "persist-schedule-warn-test",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "DuckDB persist fixture with a schedule but no version scope — a load-tolerated persistence-policy warning (schedule requires scope: version)",
|
|
5
|
+
"materialization": { "schedule": "0 6 * * *" }
|
|
6
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
##! experimental.givens
|
|
2
|
+
|
|
3
|
+
given: target_region :: string is 'US'
|
|
4
|
+
given: min_amount :: number is 0
|
|
5
|
+
given: role :: string is 'guest'
|
|
6
|
+
|
|
7
|
+
#(filter) dimension=status type=equal
|
|
8
|
+
source: orders is duckdb.table('data/orders.csv') extend {
|
|
9
|
+
primary_key: id
|
|
10
|
+
|
|
11
|
+
measure:
|
|
12
|
+
order_count is count()
|
|
13
|
+
|
|
14
|
+
view: by_given_region is {
|
|
15
|
+
where: region = $target_region and amount >= $min_amount
|
|
16
|
+
aggregate: order_count
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// Gated variant for authorize-interaction coverage: only $role = 'admin' may
|
|
21
|
+
// read it. Shares the givens above so we can prove the name-vs-value-vs-authorize
|
|
22
|
+
// status matrix on a gated source.
|
|
23
|
+
#(authorize) "$role = 'admin'"
|
|
24
|
+
source: gated_orders is duckdb.table('data/orders.csv') extend {
|
|
25
|
+
primary_key: id
|
|
26
|
+
|
|
27
|
+
measure:
|
|
28
|
+
order_count is count()
|
|
29
|
+
|
|
30
|
+
view: by_given_region is {
|
|
31
|
+
where: region = $target_region and amount >= $min_amount
|
|
32
|
+
aggregate: order_count
|
|
33
|
+
}
|
|
34
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/// <reference types="bun-types" />
|
|
2
|
+
|
|
3
|
+
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
|
4
|
+
import path from "path";
|
|
5
|
+
import { fileURLToPath } from "url";
|
|
6
|
+
import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
|
|
7
|
+
|
|
8
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
9
|
+
const __dirname = path.dirname(__filename);
|
|
10
|
+
|
|
11
|
+
const PROJECT = "env-mat-list-project";
|
|
12
|
+
const PKG_A = "persist-test";
|
|
13
|
+
const PKG_B = "persist-schedule-test";
|
|
14
|
+
const ENV_MATS_PATH = `/api/v0/environments/${PROJECT}/packages/materializations`;
|
|
15
|
+
|
|
16
|
+
// Verifies the environment-scoped list:
|
|
17
|
+
// GET /environments/{env}/packages/materializations returns runs from every
|
|
18
|
+
// package in the env, newest-first, each labeled with its packageName.
|
|
19
|
+
describe("Environment-scoped materialization list", () => {
|
|
20
|
+
let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
|
|
21
|
+
let baseUrl: string;
|
|
22
|
+
|
|
23
|
+
const fixture = (name: string) =>
|
|
24
|
+
path.resolve(__dirname, `../../fixtures/${name}`);
|
|
25
|
+
|
|
26
|
+
async function createAndFinish(pkg: string): Promise<string> {
|
|
27
|
+
const res = await fetch(
|
|
28
|
+
`${baseUrl}/api/v0/environments/${PROJECT}/packages/${pkg}/materializations`,
|
|
29
|
+
{
|
|
30
|
+
method: "POST",
|
|
31
|
+
headers: { "Content-Type": "application/json" },
|
|
32
|
+
body: JSON.stringify({ forceRefresh: true }),
|
|
33
|
+
},
|
|
34
|
+
);
|
|
35
|
+
expect(res.status).toBe(201);
|
|
36
|
+
const created = (await res.json()) as { id: string };
|
|
37
|
+
const deadline = Date.now() + 60_000;
|
|
38
|
+
while (Date.now() < deadline) {
|
|
39
|
+
const poll = await fetch(
|
|
40
|
+
`${baseUrl}/api/v0/environments/${PROJECT}/packages/${pkg}/materializations/${created.id}`,
|
|
41
|
+
);
|
|
42
|
+
const m = (await poll.json()) as { status: string };
|
|
43
|
+
if (
|
|
44
|
+
["MANIFEST_FILE_READY", "FAILED", "CANCELLED"].includes(m.status)
|
|
45
|
+
) {
|
|
46
|
+
break;
|
|
47
|
+
}
|
|
48
|
+
await new Promise((r) => setTimeout(r, 250));
|
|
49
|
+
}
|
|
50
|
+
return created.id;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
beforeAll(async () => {
|
|
54
|
+
env = await startRestE2E();
|
|
55
|
+
baseUrl = env.baseUrl;
|
|
56
|
+
|
|
57
|
+
const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
|
|
58
|
+
method: "POST",
|
|
59
|
+
headers: { "Content-Type": "application/json" },
|
|
60
|
+
body: JSON.stringify({
|
|
61
|
+
name: PROJECT,
|
|
62
|
+
packages: [
|
|
63
|
+
{ name: PKG_A, location: fixture(PKG_A) },
|
|
64
|
+
{ name: PKG_B, location: fixture(PKG_B) },
|
|
65
|
+
],
|
|
66
|
+
connections: [],
|
|
67
|
+
}),
|
|
68
|
+
});
|
|
69
|
+
if (!createRes.ok) {
|
|
70
|
+
throw new Error(
|
|
71
|
+
`Failed to create project (${createRes.status}): ${await createRes.text()}`,
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Wait for both packages to load.
|
|
76
|
+
for (const pkg of [PKG_A, PKG_B]) {
|
|
77
|
+
const deadline = Date.now() + 30_000;
|
|
78
|
+
while (Date.now() < deadline) {
|
|
79
|
+
const res = await fetch(
|
|
80
|
+
`${baseUrl}/api/v0/environments/${PROJECT}/packages/${pkg}`,
|
|
81
|
+
);
|
|
82
|
+
if (res.ok) break;
|
|
83
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
afterAll(async () => {
|
|
89
|
+
if (baseUrl) {
|
|
90
|
+
try {
|
|
91
|
+
await fetch(`${baseUrl}/api/v0/environments/${PROJECT}`, {
|
|
92
|
+
method: "DELETE",
|
|
93
|
+
});
|
|
94
|
+
} catch {
|
|
95
|
+
// best-effort
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
await env?.stop();
|
|
99
|
+
env = null;
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
it("returns materializations from every package in the environment", async () => {
|
|
103
|
+
await createAndFinish(PKG_A);
|
|
104
|
+
await createAndFinish(PKG_B);
|
|
105
|
+
|
|
106
|
+
const res = await fetch(`${baseUrl}${ENV_MATS_PATH}`);
|
|
107
|
+
expect(res.status).toBe(200);
|
|
108
|
+
const rows = (await res.json()) as Array<{
|
|
109
|
+
packageName: string;
|
|
110
|
+
createdAt?: string;
|
|
111
|
+
}>;
|
|
112
|
+
|
|
113
|
+
const packages = new Set(rows.map((r) => r.packageName));
|
|
114
|
+
expect(packages.has(PKG_A)).toBe(true);
|
|
115
|
+
expect(packages.has(PKG_B)).toBe(true);
|
|
116
|
+
|
|
117
|
+
// Newest-first ordering.
|
|
118
|
+
const times = rows
|
|
119
|
+
.map((r) => (r.createdAt ? new Date(r.createdAt).getTime() : 0))
|
|
120
|
+
.filter((t) => t > 0);
|
|
121
|
+
const sorted = [...times].sort((a, b) => b - a);
|
|
122
|
+
expect(times).toEqual(sorted);
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
it("does not shadow the per-package list route", async () => {
|
|
126
|
+
const res = await fetch(
|
|
127
|
+
`${baseUrl}/api/v0/environments/${PROJECT}/packages/${PKG_A}/materializations`,
|
|
128
|
+
);
|
|
129
|
+
expect(res.status).toBe(200);
|
|
130
|
+
const rows = (await res.json()) as Array<{ packageName: string }>;
|
|
131
|
+
expect(rows.every((r) => r.packageName === PKG_A)).toBe(true);
|
|
132
|
+
});
|
|
133
|
+
});
|
|
@@ -346,9 +346,14 @@ describe("Materialization REST API: single-call (E2E)", () => {
|
|
|
346
346
|
const routedExecuted = await executedSql();
|
|
347
347
|
const routedCompiled = await compiledSql();
|
|
348
348
|
expect(routedExecuted).not.toContain("data/orders.csv");
|
|
349
|
-
expect(routedExecuted).toContain("order_summary");
|
|
350
349
|
expect(routedCompiled).not.toContain("data/orders.csv");
|
|
351
|
-
|
|
350
|
+
// The bound table is referenced QUOTED for the connection's dialect,
|
|
351
|
+
// byte-compatible with the CREATE side's quoteTablePath: a
|
|
352
|
+
// case-folding engine (Snowflake uppercases unquoted identifiers)
|
|
353
|
+
// can only resolve the case-preserved created table through the
|
|
354
|
+
// same quoting (Package.quoteBoundTableNames).
|
|
355
|
+
expect(routedExecuted).toContain('"order_summary"');
|
|
356
|
+
expect(routedCompiled).toContain('"order_summary"');
|
|
352
357
|
|
|
353
358
|
// Cleanup: drop the table + record, then reload back to live so the
|
|
354
359
|
// dangling binding doesn't leak into later groups.
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/// <reference types="bun-types" />
|
|
2
|
+
|
|
3
|
+
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
|
4
|
+
import fsp from "fs/promises";
|
|
5
|
+
import os from "os";
|
|
6
|
+
import path from "path";
|
|
7
|
+
import { fileURLToPath } from "url";
|
|
8
|
+
import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
|
|
9
|
+
|
|
10
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
11
|
+
const __dirname = path.dirname(__filename);
|
|
12
|
+
|
|
13
|
+
const PROJECT_NAME = "rebind-safety-project";
|
|
14
|
+
const PACKAGE_NAME = "persist-schedule-warn-test";
|
|
15
|
+
const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
|
|
16
|
+
|
|
17
|
+
// Guards the orchestrated control plane's post-build rebind PATCH against the
|
|
18
|
+
// editable-policy gate added in this PR. The fixture loads with a pre-existing,
|
|
19
|
+
// load-tolerated persistence-policy warning (a schedule without scope: version),
|
|
20
|
+
// so if the policy gate ran on a rebind PATCH it would reject and fail the
|
|
21
|
+
// orchestrated run. These tests assert it does not: a manifestLocation-only
|
|
22
|
+
// PATCH, and a PATCH carrying explicit scope: null / materialization: null, both
|
|
23
|
+
// succeed and leave scope/materialization untouched (null == absent, not a wipe).
|
|
24
|
+
describe("Orchestrated rebind PATCH safety", () => {
|
|
25
|
+
let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
|
|
26
|
+
let baseUrl: string;
|
|
27
|
+
|
|
28
|
+
beforeAll(async () => {
|
|
29
|
+
env = await startRestE2E();
|
|
30
|
+
baseUrl = env.baseUrl;
|
|
31
|
+
|
|
32
|
+
const fixtureDir = path.resolve(
|
|
33
|
+
__dirname,
|
|
34
|
+
"../../fixtures/persist-schedule-warn-test",
|
|
35
|
+
);
|
|
36
|
+
const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
|
|
37
|
+
method: "POST",
|
|
38
|
+
headers: { "Content-Type": "application/json" },
|
|
39
|
+
body: JSON.stringify({
|
|
40
|
+
name: PROJECT_NAME,
|
|
41
|
+
packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
|
|
42
|
+
connections: [],
|
|
43
|
+
}),
|
|
44
|
+
});
|
|
45
|
+
if (!createRes.ok) {
|
|
46
|
+
throw new Error(
|
|
47
|
+
`Failed to create test project (${createRes.status}): ${await createRes.text()}`,
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const deadline = Date.now() + 30_000;
|
|
52
|
+
while (Date.now() < deadline) {
|
|
53
|
+
const res = await fetch(`${baseUrl}${API}`);
|
|
54
|
+
if (res.ok) break;
|
|
55
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
afterAll(async () => {
|
|
60
|
+
if (baseUrl) {
|
|
61
|
+
try {
|
|
62
|
+
await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
|
|
63
|
+
method: "DELETE",
|
|
64
|
+
});
|
|
65
|
+
} catch {
|
|
66
|
+
// best-effort
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
await env?.stop();
|
|
70
|
+
env = null;
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
async function getPackage(): Promise<Record<string, unknown>> {
|
|
74
|
+
const res = await fetch(`${baseUrl}${API}`);
|
|
75
|
+
expect(res.status).toBe(200);
|
|
76
|
+
return (await res.json()) as Record<string, unknown>;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function scheduleOf(
|
|
80
|
+
pkg: Record<string, unknown>,
|
|
81
|
+
): string | null | undefined {
|
|
82
|
+
return (pkg.materialization as { schedule?: string | null } | null)
|
|
83
|
+
?.schedule;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
it("loads the fixture despite its load-tolerated policy warning", async () => {
|
|
87
|
+
// scope defaults to "package" (no scope in the fixture), and the schedule
|
|
88
|
+
// is present — the exact combination that warns but is load-tolerated.
|
|
89
|
+
const pkg = await getPackage();
|
|
90
|
+
expect(pkg.scope ?? "package").toBe("package");
|
|
91
|
+
expect(scheduleOf(pkg)).toBe("0 6 * * *");
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
it("a manifestLocation-only rebind PATCH succeeds and preserves scope/materialization", async () => {
|
|
95
|
+
const manifestFile = path.join(
|
|
96
|
+
os.tmpdir(),
|
|
97
|
+
`rebind-empty-manifest-${PROJECT_NAME}.json`,
|
|
98
|
+
);
|
|
99
|
+
await fsp.writeFile(
|
|
100
|
+
manifestFile,
|
|
101
|
+
JSON.stringify({
|
|
102
|
+
builtAt: new Date(0).toISOString(),
|
|
103
|
+
strict: false,
|
|
104
|
+
entries: {},
|
|
105
|
+
}),
|
|
106
|
+
"utf8",
|
|
107
|
+
);
|
|
108
|
+
|
|
109
|
+
// The control plane's finalize PATCH: name/manifestLocation only, no
|
|
110
|
+
// scope/materialization keys. Must not run the policy gate.
|
|
111
|
+
const res = await fetch(`${baseUrl}${API}`, {
|
|
112
|
+
method: "PATCH",
|
|
113
|
+
headers: { "Content-Type": "application/json" },
|
|
114
|
+
body: JSON.stringify({
|
|
115
|
+
name: PACKAGE_NAME,
|
|
116
|
+
manifestLocation: manifestFile,
|
|
117
|
+
}),
|
|
118
|
+
});
|
|
119
|
+
expect(res.ok).toBe(true);
|
|
120
|
+
|
|
121
|
+
const pkg = await getPackage();
|
|
122
|
+
expect(pkg.scope ?? "package").toBe("package");
|
|
123
|
+
expect(scheduleOf(pkg)).toBe("0 6 * * *");
|
|
124
|
+
|
|
125
|
+
// Reset the binding so the next test starts from the live package.
|
|
126
|
+
await fetch(`${baseUrl}${API}`, {
|
|
127
|
+
method: "PATCH",
|
|
128
|
+
headers: { "Content-Type": "application/json" },
|
|
129
|
+
body: JSON.stringify({ name: PACKAGE_NAME, manifestLocation: null }),
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
it("a PATCH with explicit scope: null / materialization: null preserves both (null == absent, no wipe, no reject)", async () => {
|
|
134
|
+
const res = await fetch(`${baseUrl}${API}`, {
|
|
135
|
+
method: "PATCH",
|
|
136
|
+
headers: { "Content-Type": "application/json" },
|
|
137
|
+
body: JSON.stringify({
|
|
138
|
+
name: PACKAGE_NAME,
|
|
139
|
+
scope: null,
|
|
140
|
+
materialization: null,
|
|
141
|
+
}),
|
|
142
|
+
});
|
|
143
|
+
// The gate must treat null as absent: not rejected by the warning...
|
|
144
|
+
expect(res.ok).toBe(true);
|
|
145
|
+
// ...and the persisted policy is preserved, not wiped.
|
|
146
|
+
const pkg = await getPackage();
|
|
147
|
+
expect(pkg.scope ?? "package").toBe("package");
|
|
148
|
+
expect(scheduleOf(pkg)).toBe("0 6 * * *");
|
|
149
|
+
});
|
|
150
|
+
});
|