@malloy-publisher/server 0.0.228 → 0.0.229
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +84 -1
- package/dist/app/assets/EnvironmentPage-QOoHiVeJ.js +1 -0
- package/dist/app/assets/{HomePage-Bkwc9Woc.js → HomePage-C71GOfVW.js} +1 -1
- package/dist/app/assets/{LightMode-Bum_KBpN.js → LightMode-CrgCAwLe.js} +1 -1
- package/dist/app/assets/MainPage-BG5__FN3.js +2 -0
- package/dist/app/assets/MaterializationsPage-DE6PnrDR.js +1 -0
- package/dist/app/assets/ModelPage-CcBjcbLm.js +1 -0
- package/dist/app/assets/PackagePage-JTy3ztkB.js +1 -0
- package/dist/app/assets/{RouteError-DmJLpLXm.js → RouteError-Cymbp47a.js} +1 -1
- package/dist/app/assets/ThemeEditorPage-C_nMnHr8.js +1 -0
- package/dist/app/assets/WorkbookPage-CPQu-DQx.js +1 -0
- package/dist/app/assets/{core-CEDZMHV1.es-_yGzNgNH.js → core-Coi3caGs.es-CSOmajHS.js} +10 -10
- package/dist/app/assets/{index-DDMrjIT3.js → index-CM2qhQCI.js} +1 -1
- package/dist/app/assets/{index-CE9xhdra.js → index-CcuuST2X.js} +1 -1
- package/dist/app/assets/{index-OEjKNSYb.js → index-CkmABCAw.js} +388 -241
- package/dist/app/assets/{index-CdmFub34.js → index-DlWCXghy.js} +1 -1
- package/dist/app/assets/index-DxArlgRD.js +529 -0
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +1 -1
- package/dist/package_load_worker.mjs +1 -1
- package/dist/server.mjs +6934 -36
- package/package.json +4 -2
- package/src/config.spec.ts +87 -1
- package/src/config.ts +102 -1
- package/src/controller/materialization.controller.spec.ts +9 -0
- package/src/controller/materialization.controller.ts +15 -0
- package/src/materialization_metrics.ts +33 -0
- package/src/mcp/skills/build_skills_bundle.ts +29 -9
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/skills/skills_bundle.spec.ts +31 -2
- package/src/query_param_utils.ts +11 -0
- package/src/server.ts +67 -9
- package/src/service/connection.spec.ts +8 -7
- package/src/service/cron_evaluator.spec.ts +79 -0
- package/src/service/cron_evaluator.ts +105 -0
- package/src/service/environment.ts +118 -13
- package/src/service/environment_store.spec.ts +372 -2
- package/src/service/environment_store.ts +149 -17
- package/src/service/environment_store_anchoring.spec.ts +107 -0
- package/src/service/manifest_loader.spec.ts +3 -1
- package/src/service/manifest_loader.ts +9 -5
- package/src/service/materialization_scheduler.spec.ts +260 -0
- package/src/service/materialization_scheduler.ts +304 -0
- package/src/service/materialization_service.spec.ts +167 -4
- package/src/service/materialization_service.ts +121 -8
- package/src/service/materialization_test_fixtures.ts +10 -1
- package/src/service/package.ts +90 -1
- package/src/service/package_quote_bound_tables.spec.ts +171 -0
- package/src/service/persistence_policy.spec.ts +31 -0
- package/src/service/quoting.spec.ts +57 -1
- package/src/service/quoting.ts +33 -0
- package/src/storage/DatabaseInterface.ts +15 -0
- package/src/storage/duckdb/DuckDBRepository.ts +17 -0
- package/src/storage/duckdb/MaterializationRepository.spec.ts +154 -0
- package/src/storage/duckdb/MaterializationRepository.ts +56 -0
- package/tests/fixtures/persist-schedule-test/data/orders.csv +5 -0
- package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +11 -0
- package/tests/fixtures/persist-schedule-test/publisher.json +7 -0
- package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +5 -0
- package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +11 -0
- package/tests/fixtures/persist-schedule-warn-test/publisher.json +6 -0
- package/tests/integration/materialization/environment_list.integration.spec.ts +133 -0
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +7 -2
- package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +150 -0
- package/tests/integration/materialization/schedule_edit.integration.spec.ts +126 -0
- package/tests/integration/materialization/scheduler.integration.spec.ts +200 -0
- package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +172 -0
- package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +143 -0
- package/dist/app/assets/EnvironmentPage-EW2lbGvb.js +0 -1
- package/dist/app/assets/MainPage-oiEy7TNM.js +0 -2
- package/dist/app/assets/MaterializationsPage-C_VJsTgU.js +0 -1
- package/dist/app/assets/ModelPage-z8REqAmk.js +0 -1
- package/dist/app/assets/PackagePage-C2Vtt1Ln.js +0 -1
- package/dist/app/assets/ThemeEditorPage-BywFjC7A.js +0 -1
- package/dist/app/assets/WorkbookPage-DCMizDMR.js +0 -1
- package/dist/app/assets/index-EqslXZ44.js +0 -527
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/// <reference types="bun-types" />
|
|
2
|
+
|
|
3
|
+
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
|
4
|
+
import path from "path";
|
|
5
|
+
import { fileURLToPath } from "url";
|
|
6
|
+
import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
|
|
7
|
+
|
|
8
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
9
|
+
const __dirname = path.dirname(__filename);
|
|
10
|
+
|
|
11
|
+
const PROJECT_NAME = "sched-edit-project";
|
|
12
|
+
const PACKAGE_NAME = "persist-schedule-test";
|
|
13
|
+
const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
|
|
14
|
+
|
|
15
|
+
// Exercises the editable-schedule PATCH path: materialization.schedule and
|
|
16
|
+
// scope are writable via updatePackage, validated with the same rules a publish
|
|
17
|
+
// enforces (schedule requires scope: version, valid 5-field cron), and
|
|
18
|
+
// persisted to publisher.json.
|
|
19
|
+
describe("Editable materialization schedule (PATCH)", () => {
|
|
20
|
+
let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
|
|
21
|
+
let baseUrl: string;
|
|
22
|
+
|
|
23
|
+
beforeAll(async () => {
|
|
24
|
+
env = await startRestE2E();
|
|
25
|
+
baseUrl = env.baseUrl;
|
|
26
|
+
|
|
27
|
+
const fixtureDir = path.resolve(
|
|
28
|
+
__dirname,
|
|
29
|
+
"../../fixtures/persist-schedule-test",
|
|
30
|
+
);
|
|
31
|
+
const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
|
|
32
|
+
method: "POST",
|
|
33
|
+
headers: { "Content-Type": "application/json" },
|
|
34
|
+
body: JSON.stringify({
|
|
35
|
+
name: PROJECT_NAME,
|
|
36
|
+
packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
|
|
37
|
+
connections: [],
|
|
38
|
+
}),
|
|
39
|
+
});
|
|
40
|
+
if (!createRes.ok) {
|
|
41
|
+
throw new Error(
|
|
42
|
+
`Failed to create test project (${createRes.status}): ${await createRes.text()}`,
|
|
43
|
+
);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const deadline = Date.now() + 30_000;
|
|
47
|
+
while (Date.now() < deadline) {
|
|
48
|
+
const res = await fetch(`${baseUrl}${API}`);
|
|
49
|
+
if (res.ok) break;
|
|
50
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
afterAll(async () => {
|
|
55
|
+
if (baseUrl) {
|
|
56
|
+
try {
|
|
57
|
+
await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
|
|
58
|
+
method: "DELETE",
|
|
59
|
+
});
|
|
60
|
+
} catch {
|
|
61
|
+
// best-effort
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
await env?.stop();
|
|
65
|
+
env = null;
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
async function getPackage(): Promise<Record<string, unknown>> {
|
|
69
|
+
const res = await fetch(`${baseUrl}${API}`);
|
|
70
|
+
expect(res.status).toBe(200);
|
|
71
|
+
return (await res.json()) as Record<string, unknown>;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function scheduleOf(
|
|
75
|
+
pkg: Record<string, unknown>,
|
|
76
|
+
): string | null | undefined {
|
|
77
|
+
return (pkg.materialization as { schedule?: string | null } | null)
|
|
78
|
+
?.schedule;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async function patch(body: Record<string, unknown>): Promise<Response> {
|
|
82
|
+
return fetch(`${baseUrl}${API}`, {
|
|
83
|
+
method: "PATCH",
|
|
84
|
+
headers: { "Content-Type": "application/json" },
|
|
85
|
+
body: JSON.stringify({ name: PACKAGE_NAME, ...body }),
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
it("starts from the fixture's version-scoped schedule", async () => {
|
|
90
|
+
const pkg = await getPackage();
|
|
91
|
+
expect(pkg.scope).toBe("version");
|
|
92
|
+
expect(scheduleOf(pkg)).toBe("0 6 * * *");
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
it("edits and persists a new schedule", async () => {
|
|
96
|
+
const res = await patch({
|
|
97
|
+
materialization: { schedule: "*/30 * * * *" },
|
|
98
|
+
});
|
|
99
|
+
expect(res.ok).toBe(true);
|
|
100
|
+
expect(scheduleOf(await getPackage())).toBe("*/30 * * * *");
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
it("rejects an invalid cron (400) and leaves the schedule unchanged", async () => {
|
|
104
|
+
const res = await patch({ materialization: { schedule: "not a cron" } });
|
|
105
|
+
expect(res.status).toBe(400);
|
|
106
|
+
expect(scheduleOf(await getPackage())).toBe("*/30 * * * *");
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it("rejects a schedule without version scope (400)", async () => {
|
|
110
|
+
const res = await patch({
|
|
111
|
+
scope: "package",
|
|
112
|
+
materialization: { schedule: "0 6 * * *" },
|
|
113
|
+
});
|
|
114
|
+
expect(res.status).toBe(400);
|
|
115
|
+
// Prior good state preserved (scope + schedule both unchanged).
|
|
116
|
+
const pkg = await getPackage();
|
|
117
|
+
expect(pkg.scope).toBe("version");
|
|
118
|
+
expect(scheduleOf(pkg)).toBe("*/30 * * * *");
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
it("clears the schedule with schedule: null", async () => {
|
|
122
|
+
const res = await patch({ materialization: { schedule: null } });
|
|
123
|
+
expect(res.ok).toBe(true);
|
|
124
|
+
expect(scheduleOf(await getPackage())).toBeNull();
|
|
125
|
+
});
|
|
126
|
+
});
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
/// <reference types="bun-types" />
|
|
2
|
+
|
|
3
|
+
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
|
4
|
+
import fsp from "fs/promises";
|
|
5
|
+
import os from "os";
|
|
6
|
+
import path from "path";
|
|
7
|
+
import { fileURLToPath } from "url";
|
|
8
|
+
import { MaterializationScheduler } from "../../../src/service/materialization_scheduler";
|
|
9
|
+
import { MaterializationService } from "../../../src/service/materialization_service";
|
|
10
|
+
import { environmentStore } from "../../../src/server";
|
|
11
|
+
import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
|
|
12
|
+
|
|
13
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
14
|
+
const __dirname = path.dirname(__filename);
|
|
15
|
+
|
|
16
|
+
const PROJECT_NAME = "sched-project";
|
|
17
|
+
const PACKAGE_NAME = "persist-schedule-test";
|
|
18
|
+
const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
|
|
19
|
+
|
|
20
|
+
const TERMINAL = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"];
|
|
21
|
+
|
|
22
|
+
// Deterministic clock for the scheduler: we call tick(nowMs) explicitly, so the
|
|
23
|
+
// fixture's daily cron ("0 6 * * *") fires without waiting on a real minute.
|
|
24
|
+
const T0 = Date.parse("2026-01-01T00:00:00Z");
|
|
25
|
+
const THREE_DAYS_MS = 3 * 24 * 60 * 60 * 1000;
|
|
26
|
+
|
|
27
|
+
describe("MaterializationScheduler (E2E, real build)", () => {
|
|
28
|
+
let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
|
|
29
|
+
let baseUrl: string;
|
|
30
|
+
let scheduler: MaterializationScheduler;
|
|
31
|
+
|
|
32
|
+
beforeAll(async () => {
|
|
33
|
+
env = await startRestE2E();
|
|
34
|
+
baseUrl = env.baseUrl;
|
|
35
|
+
|
|
36
|
+
const fixtureDir = path.resolve(
|
|
37
|
+
__dirname,
|
|
38
|
+
"../../fixtures/persist-schedule-test",
|
|
39
|
+
);
|
|
40
|
+
const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
|
|
41
|
+
method: "POST",
|
|
42
|
+
headers: { "Content-Type": "application/json" },
|
|
43
|
+
body: JSON.stringify({
|
|
44
|
+
name: PROJECT_NAME,
|
|
45
|
+
packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
|
|
46
|
+
connections: [],
|
|
47
|
+
}),
|
|
48
|
+
});
|
|
49
|
+
if (!createRes.ok) {
|
|
50
|
+
throw new Error(
|
|
51
|
+
`Failed to create test project (${createRes.status}): ${await createRes.text()}`,
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Wait for the package to load (so it's in the environmentStore the
|
|
56
|
+
// scheduler sweeps).
|
|
57
|
+
const deadline = Date.now() + 30_000;
|
|
58
|
+
while (Date.now() < deadline) {
|
|
59
|
+
const res = await fetch(`${baseUrl}${API}`);
|
|
60
|
+
if (res.ok) break;
|
|
61
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Drive the real scheduler against the server's real environmentStore and
|
|
65
|
+
// a real MaterializationService (shares the same repository). We call
|
|
66
|
+
// tick() by hand with a controlled clock rather than relying on the timer.
|
|
67
|
+
scheduler = new MaterializationScheduler(
|
|
68
|
+
environmentStore,
|
|
69
|
+
new MaterializationService(environmentStore),
|
|
70
|
+
{ tickIntervalMs: 60_000, maxFiresPerTick: 10 },
|
|
71
|
+
);
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
afterAll(async () => {
|
|
75
|
+
if (baseUrl) {
|
|
76
|
+
try {
|
|
77
|
+
await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
|
|
78
|
+
method: "DELETE",
|
|
79
|
+
});
|
|
80
|
+
} catch {
|
|
81
|
+
// best-effort
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
await env?.stop();
|
|
85
|
+
env = null;
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
async function listMaterializations(): Promise<
|
|
89
|
+
Array<Record<string, unknown>>
|
|
90
|
+
> {
|
|
91
|
+
const res = await fetch(`${baseUrl}${API}/materializations`);
|
|
92
|
+
expect(res.status).toBe(200);
|
|
93
|
+
return (await res.json()) as Array<Record<string, unknown>>;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function triggerOf(m: Record<string, unknown>): string | undefined {
|
|
97
|
+
return (m.metadata as { trigger?: string } | null)?.trigger;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
async function pollTerminal(id: string): Promise<Record<string, unknown>> {
|
|
101
|
+
const deadline = Date.now() + 60_000;
|
|
102
|
+
while (Date.now() < deadline) {
|
|
103
|
+
const res = await fetch(`${baseUrl}${API}/materializations/${id}`);
|
|
104
|
+
expect(res.status).toBe(200);
|
|
105
|
+
const m = (await res.json()) as Record<string, unknown>;
|
|
106
|
+
if (TERMINAL.includes(m.status as string)) return m;
|
|
107
|
+
await new Promise((r) => setTimeout(r, 250));
|
|
108
|
+
}
|
|
109
|
+
throw new Error(`materialization ${id} did not reach a terminal state`);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
it(
|
|
113
|
+
"fires a due scheduled package as a SCHEDULER build that completes",
|
|
114
|
+
async () => {
|
|
115
|
+
// Arm on the first tick (nextFire is strictly future, so no fire yet).
|
|
116
|
+
await scheduler.tick(T0);
|
|
117
|
+
expect(
|
|
118
|
+
(await listMaterializations()).filter(
|
|
119
|
+
(m) => triggerOf(m) === "SCHEDULER",
|
|
120
|
+
).length,
|
|
121
|
+
).toBe(0);
|
|
122
|
+
|
|
123
|
+
// Advance well past the daily cron: the sweep fires one SCHEDULER run.
|
|
124
|
+
await scheduler.tick(T0 + THREE_DAYS_MS);
|
|
125
|
+
|
|
126
|
+
const scheduled = (await listMaterializations()).filter(
|
|
127
|
+
(m) => triggerOf(m) === "SCHEDULER",
|
|
128
|
+
);
|
|
129
|
+
expect(scheduled.length).toBe(1);
|
|
130
|
+
|
|
131
|
+
// The real build settles at MANIFEST_FILE_READY, producing the table.
|
|
132
|
+
const built = await pollTerminal(scheduled[0].id as string);
|
|
133
|
+
expect(built.status).toBe("MANIFEST_FILE_READY");
|
|
134
|
+
const entries = (
|
|
135
|
+
built.manifest as { entries: Record<string, unknown> }
|
|
136
|
+
).entries;
|
|
137
|
+
expect(Object.keys(entries).length).toBe(1);
|
|
138
|
+
// The SCHEDULER trigger must survive the build commit (not just exist
|
|
139
|
+
// while the run is in-flight) — otherwise a completed scheduled run is
|
|
140
|
+
// indistinguishable from a manual one.
|
|
141
|
+
expect(triggerOf(built)).toBe("SCHEDULER");
|
|
142
|
+
|
|
143
|
+
// Cleanup: drop the table + record so the negative case starts clean.
|
|
144
|
+
const del = await fetch(
|
|
145
|
+
`${baseUrl}${API}/materializations/${scheduled[0].id}?dropTables=true`,
|
|
146
|
+
{ method: "DELETE" },
|
|
147
|
+
);
|
|
148
|
+
expect(del.status).toBe(204);
|
|
149
|
+
},
|
|
150
|
+
{ timeout: 120_000 },
|
|
151
|
+
);
|
|
152
|
+
|
|
153
|
+
it(
|
|
154
|
+
"never fires a control-plane-driven package (manifestLocation set)",
|
|
155
|
+
async () => {
|
|
156
|
+
// Make the package orchestrated: point it at a (bind-clean) manifest.
|
|
157
|
+
const manifestFile = path.join(
|
|
158
|
+
os.tmpdir(),
|
|
159
|
+
`sched-empty-manifest-${T0}.json`,
|
|
160
|
+
);
|
|
161
|
+
await fsp.writeFile(
|
|
162
|
+
manifestFile,
|
|
163
|
+
JSON.stringify({
|
|
164
|
+
builtAt: new Date(T0).toISOString(),
|
|
165
|
+
strict: false,
|
|
166
|
+
entries: {},
|
|
167
|
+
}),
|
|
168
|
+
"utf8",
|
|
169
|
+
);
|
|
170
|
+
const patch = await fetch(`${baseUrl}${API}`, {
|
|
171
|
+
method: "PATCH",
|
|
172
|
+
headers: { "Content-Type": "application/json" },
|
|
173
|
+
body: JSON.stringify({
|
|
174
|
+
name: PACKAGE_NAME,
|
|
175
|
+
manifestLocation: manifestFile,
|
|
176
|
+
}),
|
|
177
|
+
});
|
|
178
|
+
expect(patch.ok).toBe(true);
|
|
179
|
+
|
|
180
|
+
const before = (await listMaterializations()).length;
|
|
181
|
+
|
|
182
|
+
// Even armed + well past the cron, an orchestrated package is skipped
|
|
183
|
+
// (Guard 2) — the control plane owns its refresh.
|
|
184
|
+
await scheduler.tick(T0);
|
|
185
|
+
await scheduler.tick(T0 + THREE_DAYS_MS);
|
|
186
|
+
|
|
187
|
+
expect((await listMaterializations()).length).toBe(before);
|
|
188
|
+
|
|
189
|
+
await fetch(`${baseUrl}${API}`, {
|
|
190
|
+
method: "PATCH",
|
|
191
|
+
headers: { "Content-Type": "application/json" },
|
|
192
|
+
body: JSON.stringify({
|
|
193
|
+
name: PACKAGE_NAME,
|
|
194
|
+
manifestLocation: null,
|
|
195
|
+
}),
|
|
196
|
+
});
|
|
197
|
+
},
|
|
198
|
+
{ timeout: 60_000 },
|
|
199
|
+
);
|
|
200
|
+
});
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
/// <reference types="bun-types" />
|
|
2
|
+
|
|
3
|
+
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
|
4
|
+
import path from "path";
|
|
5
|
+
import { fileURLToPath } from "url";
|
|
6
|
+
import { CronEvaluator } from "../../../src/service/cron_evaluator";
|
|
7
|
+
import { MaterializationScheduler } from "../../../src/service/materialization_scheduler";
|
|
8
|
+
import { MaterializationService } from "../../../src/service/materialization_service";
|
|
9
|
+
import { environmentStore } from "../../../src/server";
|
|
10
|
+
import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
|
|
11
|
+
|
|
12
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
13
|
+
const __dirname = path.dirname(__filename);
|
|
14
|
+
|
|
15
|
+
const PROJECT_NAME = "sched-recovery-project";
|
|
16
|
+
const PACKAGE_NAME = "persist-schedule-test";
|
|
17
|
+
const CRON = "0 6 * * *"; // the fixture's daily schedule
|
|
18
|
+
const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
|
|
19
|
+
const CONFIG = { tickIntervalMs: 60_000, maxFiresPerTick: 10 };
|
|
20
|
+
const TERMINAL = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"];
|
|
21
|
+
|
|
22
|
+
// The restart-recovery anchor is only unit-tested against a stubbed
|
|
23
|
+
// getLatestScheduledFireAt. This drives the real sequence end to end: a real
|
|
24
|
+
// SCHEDULER fire lands a row in the store, then a *second* scheduler instance
|
|
25
|
+
// (a simulated process restart — empty in-memory arming state) re-anchors from
|
|
26
|
+
// that row via the real repo query. We assert exactly one catch-up when an
|
|
27
|
+
// occurrence came due during "downtime", and no double-fire when it didn't.
|
|
28
|
+
describe("MaterializationScheduler restart recovery (E2E, real store)", () => {
|
|
29
|
+
let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
|
|
30
|
+
let baseUrl: string;
|
|
31
|
+
let service: MaterializationService;
|
|
32
|
+
// A fixed anchor for the *first* scheduler's arming clock; the real fire's
|
|
33
|
+
// created_at (wall clock) is what recovery actually re-anchors from, and we
|
|
34
|
+
// read it back from the store rather than assuming it.
|
|
35
|
+
const T0 = Date.parse("2026-01-01T00:00:00Z");
|
|
36
|
+
const THREE_DAYS_MS = 3 * 24 * 60 * 60 * 1000;
|
|
37
|
+
|
|
38
|
+
beforeAll(async () => {
|
|
39
|
+
env = await startRestE2E();
|
|
40
|
+
baseUrl = env.baseUrl;
|
|
41
|
+
service = new MaterializationService(environmentStore);
|
|
42
|
+
|
|
43
|
+
const fixtureDir = path.resolve(
|
|
44
|
+
__dirname,
|
|
45
|
+
"../../fixtures/persist-schedule-test",
|
|
46
|
+
);
|
|
47
|
+
const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
|
|
48
|
+
method: "POST",
|
|
49
|
+
headers: { "Content-Type": "application/json" },
|
|
50
|
+
body: JSON.stringify({
|
|
51
|
+
name: PROJECT_NAME,
|
|
52
|
+
packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
|
|
53
|
+
connections: [],
|
|
54
|
+
}),
|
|
55
|
+
});
|
|
56
|
+
if (!createRes.ok) {
|
|
57
|
+
throw new Error(
|
|
58
|
+
`Failed to create test project (${createRes.status}): ${await createRes.text()}`,
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Wait for the package to load into the store the scheduler sweeps.
|
|
63
|
+
const deadline = Date.now() + 30_000;
|
|
64
|
+
while (Date.now() < deadline) {
|
|
65
|
+
const res = await fetch(`${baseUrl}${API}`);
|
|
66
|
+
if (res.ok) break;
|
|
67
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
afterAll(async () => {
|
|
72
|
+
if (baseUrl) {
|
|
73
|
+
try {
|
|
74
|
+
await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
|
|
75
|
+
method: "DELETE",
|
|
76
|
+
});
|
|
77
|
+
} catch {
|
|
78
|
+
// best-effort
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
await env?.stop();
|
|
82
|
+
env = null;
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
function newScheduler(): MaterializationScheduler {
|
|
86
|
+
// A fresh instance == a process restart: no in-memory arming state, so the
|
|
87
|
+
// first arm re-anchors from the store.
|
|
88
|
+
return new MaterializationScheduler(environmentStore, service, CONFIG);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async function scheduledCount(): Promise<number> {
|
|
92
|
+
const res = await fetch(`${baseUrl}${API}/materializations`);
|
|
93
|
+
expect(res.status).toBe(200);
|
|
94
|
+
const rows = (await res.json()) as Array<Record<string, unknown>>;
|
|
95
|
+
return rows.filter(
|
|
96
|
+
(m) =>
|
|
97
|
+
(m.metadata as { trigger?: string } | null)?.trigger ===
|
|
98
|
+
"SCHEDULER",
|
|
99
|
+
).length;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async function pollAllTerminal(): Promise<void> {
|
|
103
|
+
const deadline = Date.now() + 90_000;
|
|
104
|
+
while (Date.now() < deadline) {
|
|
105
|
+
const res = await fetch(`${baseUrl}${API}/materializations`);
|
|
106
|
+
const rows = (await res.json()) as Array<Record<string, unknown>>;
|
|
107
|
+
if (rows.every((m) => TERMINAL.includes(m.status as string))) return;
|
|
108
|
+
await new Promise((r) => setTimeout(r, 250));
|
|
109
|
+
}
|
|
110
|
+
throw new Error("materializations did not all reach a terminal state");
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
it(
|
|
114
|
+
"recovers exactly one missed occurrence across a restart, then jumps forward",
|
|
115
|
+
async () => {
|
|
116
|
+
const cron = new CronEvaluator();
|
|
117
|
+
|
|
118
|
+
// ── Scheduler #1: produce a real baseline SCHEDULER fire. ──
|
|
119
|
+
const s1 = newScheduler();
|
|
120
|
+
await s1.tick(T0); // arm (nextFire strictly future -> no fire)
|
|
121
|
+
expect(await scheduledCount()).toBe(0);
|
|
122
|
+
await s1.tick(T0 + THREE_DAYS_MS); // an occurrence is due -> one fire
|
|
123
|
+
expect(await scheduledCount()).toBe(1);
|
|
124
|
+
await pollAllTerminal();
|
|
125
|
+
|
|
126
|
+
// The instant recovery re-anchors from is the newest SCHEDULER row's
|
|
127
|
+
// created_at (wall clock), read through the same path the scheduler uses.
|
|
128
|
+
const lastFire = await service.getLatestScheduledFireAt(
|
|
129
|
+
PROJECT_NAME,
|
|
130
|
+
PACKAGE_NAME,
|
|
131
|
+
);
|
|
132
|
+
expect(lastFire).not.toBeNull();
|
|
133
|
+
const missedOccurrence = cron.nextAfter(CRON, lastFire!).getTime();
|
|
134
|
+
|
|
135
|
+
// ── Scheduler #2: simulated restart with an occurrence missed during
|
|
136
|
+
// downtime (now is just past the first occurrence after the last fire). ──
|
|
137
|
+
const s2 = newScheduler();
|
|
138
|
+
await s2.tick(missedOccurrence + 60_000); // due on first arm -> catch up
|
|
139
|
+
expect(await scheduledCount()).toBe(2); // exactly one catch-up
|
|
140
|
+
await pollAllTerminal();
|
|
141
|
+
// Advanced forward: the same instance does not fire again next tick.
|
|
142
|
+
await s2.tick(missedOccurrence + 120_000);
|
|
143
|
+
expect(await scheduledCount()).toBe(2);
|
|
144
|
+
},
|
|
145
|
+
{ timeout: 200_000 },
|
|
146
|
+
);
|
|
147
|
+
|
|
148
|
+
it(
|
|
149
|
+
"does not double-fire on a restart when no occurrence was missed",
|
|
150
|
+
async () => {
|
|
151
|
+
const cron = new CronEvaluator();
|
|
152
|
+
const before = await scheduledCount();
|
|
153
|
+
|
|
154
|
+
// Latest SCHEDULER fire is the catch-up above (created_at ~ now). The
|
|
155
|
+
// next occurrence after it is the next 06:00 UTC, hours away.
|
|
156
|
+
const lastFire = await service.getLatestScheduledFireAt(
|
|
157
|
+
PROJECT_NAME,
|
|
158
|
+
PACKAGE_NAME,
|
|
159
|
+
);
|
|
160
|
+
expect(lastFire).not.toBeNull();
|
|
161
|
+
const nextOccurrence = cron.nextAfter(CRON, lastFire!).getTime();
|
|
162
|
+
|
|
163
|
+
// ── Scheduler #3: simulated restart shortly after the last fire, before
|
|
164
|
+
// the next occurrence is due -> arms into the future, no catch-up. ──
|
|
165
|
+
const s3 = newScheduler();
|
|
166
|
+
await s3.tick(lastFire!.getTime() + 1_000);
|
|
167
|
+
expect(await scheduledCount()).toBe(before);
|
|
168
|
+
expect(lastFire!.getTime() + 1_000).toBeLessThan(nextOccurrence);
|
|
169
|
+
},
|
|
170
|
+
{ timeout: 120_000 },
|
|
171
|
+
);
|
|
172
|
+
});
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/// <reference types="bun-types" />
|
|
2
|
+
|
|
3
|
+
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
|
4
|
+
import path from "path";
|
|
5
|
+
import { fileURLToPath } from "url";
|
|
6
|
+
import type { MaterializationScheduler } from "../../../src/service/materialization_scheduler";
|
|
7
|
+
import { MaterializationService } from "../../../src/service/materialization_service";
|
|
8
|
+
import {
|
|
9
|
+
environmentStore,
|
|
10
|
+
startMaterializationSchedulerFromEnv,
|
|
11
|
+
} from "../../../src/server";
|
|
12
|
+
import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
|
|
13
|
+
|
|
14
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
15
|
+
const __dirname = path.dirname(__filename);
|
|
16
|
+
|
|
17
|
+
const PROJECT_NAME = "sched-wiring-project";
|
|
18
|
+
const PACKAGE_NAME = "persist-schedule-test";
|
|
19
|
+
const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
|
|
20
|
+
|
|
21
|
+
const ENABLE = "PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER";
|
|
22
|
+
const INTERVAL = "PUBLISHER_MATERIALIZATION_SCHEDULER_INTERVAL_MS";
|
|
23
|
+
|
|
24
|
+
// The other scheduler tests hand-construct MaterializationScheduler and call
|
|
25
|
+
// tick() by hand, bypassing the env-var → getMaterializationSchedulerConfig →
|
|
26
|
+
// construct → start()/unref() → real-timer path an operator actually runs. This
|
|
27
|
+
// exercises that wiring: set the flag + a short interval, start the scheduler
|
|
28
|
+
// through the same seam server.ts uses at boot, and observe a build fire through
|
|
29
|
+
// the real timer (not a hand-driven tick).
|
|
30
|
+
describe("MaterializationScheduler wiring (E2E, real env-var timer)", () => {
|
|
31
|
+
let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
|
|
32
|
+
let baseUrl: string;
|
|
33
|
+
let scheduler: MaterializationScheduler | null = null;
|
|
34
|
+
const prev: Record<string, string | undefined> = {};
|
|
35
|
+
|
|
36
|
+
beforeAll(async () => {
|
|
37
|
+
env = await startRestE2E();
|
|
38
|
+
baseUrl = env.baseUrl;
|
|
39
|
+
|
|
40
|
+
const fixtureDir = path.resolve(
|
|
41
|
+
__dirname,
|
|
42
|
+
"../../fixtures/persist-schedule-test",
|
|
43
|
+
);
|
|
44
|
+
const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
|
|
45
|
+
method: "POST",
|
|
46
|
+
headers: { "Content-Type": "application/json" },
|
|
47
|
+
body: JSON.stringify({
|
|
48
|
+
name: PROJECT_NAME,
|
|
49
|
+
packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
|
|
50
|
+
connections: [],
|
|
51
|
+
}),
|
|
52
|
+
});
|
|
53
|
+
if (!createRes.ok) {
|
|
54
|
+
throw new Error(
|
|
55
|
+
`Failed to create test project (${createRes.status}): ${await createRes.text()}`,
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// Wait for the package to load, then switch it to an every-minute cron so
|
|
60
|
+
// the real timer fires within the test window (the fixture ships a daily
|
|
61
|
+
// cron, which a wall-clock test can't wait on).
|
|
62
|
+
const deadline = Date.now() + 30_000;
|
|
63
|
+
while (Date.now() < deadline) {
|
|
64
|
+
const res = await fetch(`${baseUrl}${API}`);
|
|
65
|
+
if (res.ok) break;
|
|
66
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
67
|
+
}
|
|
68
|
+
const patch = await fetch(`${baseUrl}${API}`, {
|
|
69
|
+
method: "PATCH",
|
|
70
|
+
headers: { "Content-Type": "application/json" },
|
|
71
|
+
body: JSON.stringify({
|
|
72
|
+
name: PACKAGE_NAME,
|
|
73
|
+
materialization: { schedule: "* * * * *" },
|
|
74
|
+
}),
|
|
75
|
+
});
|
|
76
|
+
expect(patch.ok).toBe(true);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
afterAll(async () => {
|
|
80
|
+
scheduler?.stop();
|
|
81
|
+
// Restore env so the shared test process isn't left with the flag set.
|
|
82
|
+
for (const key of [ENABLE, INTERVAL]) {
|
|
83
|
+
if (prev[key] === undefined) delete process.env[key];
|
|
84
|
+
else process.env[key] = prev[key];
|
|
85
|
+
}
|
|
86
|
+
if (baseUrl) {
|
|
87
|
+
try {
|
|
88
|
+
await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
|
|
89
|
+
method: "DELETE",
|
|
90
|
+
});
|
|
91
|
+
} catch {
|
|
92
|
+
// best-effort
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
await env?.stop();
|
|
96
|
+
env = null;
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
it(
|
|
100
|
+
"boots the scheduler from env config and fires a build through the real timer",
|
|
101
|
+
async () => {
|
|
102
|
+
// Enable via the real env-var path and use the minimum 1s sweep so the
|
|
103
|
+
// every-minute cron is picked up promptly once its boundary passes.
|
|
104
|
+
prev[ENABLE] = process.env[ENABLE];
|
|
105
|
+
prev[INTERVAL] = process.env[INTERVAL];
|
|
106
|
+
process.env[ENABLE] = "true";
|
|
107
|
+
process.env[INTERVAL] = "1000";
|
|
108
|
+
|
|
109
|
+
scheduler = startMaterializationSchedulerFromEnv(
|
|
110
|
+
environmentStore,
|
|
111
|
+
new MaterializationService(environmentStore),
|
|
112
|
+
);
|
|
113
|
+
// The config path must have constructed + started a scheduler (a null
|
|
114
|
+
// here would mean the enable flag wasn't honored).
|
|
115
|
+
expect(scheduler).not.toBeNull();
|
|
116
|
+
|
|
117
|
+
// Wait for the real timer to fire a SCHEDULER build and for it to build.
|
|
118
|
+
const deadline = Date.now() + 150_000;
|
|
119
|
+
let built: Record<string, unknown> | null = null;
|
|
120
|
+
while (Date.now() < deadline && !built) {
|
|
121
|
+
const res = await fetch(`${baseUrl}${API}/materializations`);
|
|
122
|
+
const rows = (await res.json()) as Array<Record<string, unknown>>;
|
|
123
|
+
built =
|
|
124
|
+
rows.find(
|
|
125
|
+
(m) =>
|
|
126
|
+
(m.metadata as { trigger?: string } | null)?.trigger ===
|
|
127
|
+
"SCHEDULER" && m.status === "MANIFEST_FILE_READY",
|
|
128
|
+
) ?? null;
|
|
129
|
+
if (!built) await new Promise((r) => setTimeout(r, 1_000));
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
expect(built).not.toBeNull();
|
|
133
|
+
const entries = (
|
|
134
|
+
built!.manifest as { entries: Record<string, unknown> }
|
|
135
|
+
).entries;
|
|
136
|
+
expect(Object.keys(entries).length).toBe(1);
|
|
137
|
+
|
|
138
|
+
// Stop the timer before the test ends so it can't fire into other files.
|
|
139
|
+
scheduler!.stop();
|
|
140
|
+
},
|
|
141
|
+
{ timeout: 180_000 },
|
|
142
|
+
);
|
|
143
|
+
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{W as r,G as t,j as e,$ as i,a5 as o}from"./index-EqslXZ44.js";function m(){const s=r(),{environmentName:n}=t();if(n){const a=i({environmentName:n});return e.jsx(o,{onSelectPackage:s,resourceUri:a})}else return e.jsx("div",{children:e.jsx("h2",{children:"Missing environment name"})})}export{m as default};
|