@malloy-publisher/server 0.0.228 → 0.0.229

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/dist/app/api-doc.yaml +84 -1
  2. package/dist/app/assets/EnvironmentPage-QOoHiVeJ.js +1 -0
  3. package/dist/app/assets/{HomePage-Bkwc9Woc.js → HomePage-C71GOfVW.js} +1 -1
  4. package/dist/app/assets/{LightMode-Bum_KBpN.js → LightMode-CrgCAwLe.js} +1 -1
  5. package/dist/app/assets/MainPage-BG5__FN3.js +2 -0
  6. package/dist/app/assets/MaterializationsPage-DE6PnrDR.js +1 -0
  7. package/dist/app/assets/ModelPage-CcBjcbLm.js +1 -0
  8. package/dist/app/assets/PackagePage-JTy3ztkB.js +1 -0
  9. package/dist/app/assets/{RouteError-DmJLpLXm.js → RouteError-Cymbp47a.js} +1 -1
  10. package/dist/app/assets/ThemeEditorPage-C_nMnHr8.js +1 -0
  11. package/dist/app/assets/WorkbookPage-CPQu-DQx.js +1 -0
  12. package/dist/app/assets/{core-CEDZMHV1.es-_yGzNgNH.js → core-Coi3caGs.es-CSOmajHS.js} +10 -10
  13. package/dist/app/assets/{index-DDMrjIT3.js → index-CM2qhQCI.js} +1 -1
  14. package/dist/app/assets/{index-CE9xhdra.js → index-CcuuST2X.js} +1 -1
  15. package/dist/app/assets/{index-OEjKNSYb.js → index-CkmABCAw.js} +388 -241
  16. package/dist/app/assets/{index-CdmFub34.js → index-DlWCXghy.js} +1 -1
  17. package/dist/app/assets/index-DxArlgRD.js +529 -0
  18. package/dist/app/index.html +1 -1
  19. package/dist/instrumentation.mjs +1 -1
  20. package/dist/package_load_worker.mjs +1 -1
  21. package/dist/server.mjs +6934 -36
  22. package/package.json +4 -2
  23. package/src/config.spec.ts +87 -1
  24. package/src/config.ts +102 -1
  25. package/src/controller/materialization.controller.spec.ts +9 -0
  26. package/src/controller/materialization.controller.ts +15 -0
  27. package/src/materialization_metrics.ts +33 -0
  28. package/src/mcp/skills/build_skills_bundle.ts +29 -9
  29. package/src/mcp/skills/skills_bundle.json +1 -1
  30. package/src/mcp/skills/skills_bundle.spec.ts +31 -2
  31. package/src/query_param_utils.ts +11 -0
  32. package/src/server.ts +67 -9
  33. package/src/service/connection.spec.ts +8 -7
  34. package/src/service/cron_evaluator.spec.ts +79 -0
  35. package/src/service/cron_evaluator.ts +105 -0
  36. package/src/service/environment.ts +118 -13
  37. package/src/service/environment_store.spec.ts +372 -2
  38. package/src/service/environment_store.ts +149 -17
  39. package/src/service/environment_store_anchoring.spec.ts +107 -0
  40. package/src/service/manifest_loader.spec.ts +3 -1
  41. package/src/service/manifest_loader.ts +9 -5
  42. package/src/service/materialization_scheduler.spec.ts +260 -0
  43. package/src/service/materialization_scheduler.ts +304 -0
  44. package/src/service/materialization_service.spec.ts +167 -4
  45. package/src/service/materialization_service.ts +121 -8
  46. package/src/service/materialization_test_fixtures.ts +10 -1
  47. package/src/service/package.ts +90 -1
  48. package/src/service/package_quote_bound_tables.spec.ts +171 -0
  49. package/src/service/persistence_policy.spec.ts +31 -0
  50. package/src/service/quoting.spec.ts +57 -1
  51. package/src/service/quoting.ts +33 -0
  52. package/src/storage/DatabaseInterface.ts +15 -0
  53. package/src/storage/duckdb/DuckDBRepository.ts +17 -0
  54. package/src/storage/duckdb/MaterializationRepository.spec.ts +154 -0
  55. package/src/storage/duckdb/MaterializationRepository.ts +56 -0
  56. package/tests/fixtures/persist-schedule-test/data/orders.csv +5 -0
  57. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +11 -0
  58. package/tests/fixtures/persist-schedule-test/publisher.json +7 -0
  59. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +5 -0
  60. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +11 -0
  61. package/tests/fixtures/persist-schedule-warn-test/publisher.json +6 -0
  62. package/tests/integration/materialization/environment_list.integration.spec.ts +133 -0
  63. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +7 -2
  64. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +150 -0
  65. package/tests/integration/materialization/schedule_edit.integration.spec.ts +126 -0
  66. package/tests/integration/materialization/scheduler.integration.spec.ts +200 -0
  67. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +172 -0
  68. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +143 -0
  69. package/dist/app/assets/EnvironmentPage-EW2lbGvb.js +0 -1
  70. package/dist/app/assets/MainPage-oiEy7TNM.js +0 -2
  71. package/dist/app/assets/MaterializationsPage-C_VJsTgU.js +0 -1
  72. package/dist/app/assets/ModelPage-z8REqAmk.js +0 -1
  73. package/dist/app/assets/PackagePage-C2Vtt1Ln.js +0 -1
  74. package/dist/app/assets/ThemeEditorPage-BywFjC7A.js +0 -1
  75. package/dist/app/assets/WorkbookPage-DCMizDMR.js +0 -1
  76. package/dist/app/assets/index-EqslXZ44.js +0 -527
@@ -0,0 +1,126 @@
1
+ /// <reference types="bun-types" />
2
+
3
+ import { afterAll, beforeAll, describe, expect, it } from "bun:test";
4
+ import path from "path";
5
+ import { fileURLToPath } from "url";
6
+ import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
7
+
8
+ const __filename = fileURLToPath(import.meta.url);
9
+ const __dirname = path.dirname(__filename);
10
+
11
+ const PROJECT_NAME = "sched-edit-project";
12
+ const PACKAGE_NAME = "persist-schedule-test";
13
+ const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
14
+
15
+ // Exercises the editable-schedule PATCH path: materialization.schedule and
16
+ // scope are writable via updatePackage, validated with the same rules a publish
17
+ // enforces (schedule requires scope: version, valid 5-field cron), and
18
+ // persisted to publisher.json.
19
+ describe("Editable materialization schedule (PATCH)", () => {
20
+ let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
21
+ let baseUrl: string;
22
+
23
+ beforeAll(async () => {
24
+ env = await startRestE2E();
25
+ baseUrl = env.baseUrl;
26
+
27
+ const fixtureDir = path.resolve(
28
+ __dirname,
29
+ "../../fixtures/persist-schedule-test",
30
+ );
31
+ const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
32
+ method: "POST",
33
+ headers: { "Content-Type": "application/json" },
34
+ body: JSON.stringify({
35
+ name: PROJECT_NAME,
36
+ packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
37
+ connections: [],
38
+ }),
39
+ });
40
+ if (!createRes.ok) {
41
+ throw new Error(
42
+ `Failed to create test project (${createRes.status}): ${await createRes.text()}`,
43
+ );
44
+ }
45
+
46
+ const deadline = Date.now() + 30_000;
47
+ while (Date.now() < deadline) {
48
+ const res = await fetch(`${baseUrl}${API}`);
49
+ if (res.ok) break;
50
+ await new Promise((r) => setTimeout(r, 500));
51
+ }
52
+ });
53
+
54
+ afterAll(async () => {
55
+ if (baseUrl) {
56
+ try {
57
+ await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
58
+ method: "DELETE",
59
+ });
60
+ } catch {
61
+ // best-effort
62
+ }
63
+ }
64
+ await env?.stop();
65
+ env = null;
66
+ });
67
+
68
+ async function getPackage(): Promise<Record<string, unknown>> {
69
+ const res = await fetch(`${baseUrl}${API}`);
70
+ expect(res.status).toBe(200);
71
+ return (await res.json()) as Record<string, unknown>;
72
+ }
73
+
74
+ function scheduleOf(
75
+ pkg: Record<string, unknown>,
76
+ ): string | null | undefined {
77
+ return (pkg.materialization as { schedule?: string | null } | null)
78
+ ?.schedule;
79
+ }
80
+
81
+ async function patch(body: Record<string, unknown>): Promise<Response> {
82
+ return fetch(`${baseUrl}${API}`, {
83
+ method: "PATCH",
84
+ headers: { "Content-Type": "application/json" },
85
+ body: JSON.stringify({ name: PACKAGE_NAME, ...body }),
86
+ });
87
+ }
88
+
89
+ it("starts from the fixture's version-scoped schedule", async () => {
90
+ const pkg = await getPackage();
91
+ expect(pkg.scope).toBe("version");
92
+ expect(scheduleOf(pkg)).toBe("0 6 * * *");
93
+ });
94
+
95
+ it("edits and persists a new schedule", async () => {
96
+ const res = await patch({
97
+ materialization: { schedule: "*/30 * * * *" },
98
+ });
99
+ expect(res.ok).toBe(true);
100
+ expect(scheduleOf(await getPackage())).toBe("*/30 * * * *");
101
+ });
102
+
103
+ it("rejects an invalid cron (400) and leaves the schedule unchanged", async () => {
104
+ const res = await patch({ materialization: { schedule: "not a cron" } });
105
+ expect(res.status).toBe(400);
106
+ expect(scheduleOf(await getPackage())).toBe("*/30 * * * *");
107
+ });
108
+
109
+ it("rejects a schedule without version scope (400)", async () => {
110
+ const res = await patch({
111
+ scope: "package",
112
+ materialization: { schedule: "0 6 * * *" },
113
+ });
114
+ expect(res.status).toBe(400);
115
+ // Prior good state preserved (scope + schedule both unchanged).
116
+ const pkg = await getPackage();
117
+ expect(pkg.scope).toBe("version");
118
+ expect(scheduleOf(pkg)).toBe("*/30 * * * *");
119
+ });
120
+
121
+ it("clears the schedule with schedule: null", async () => {
122
+ const res = await patch({ materialization: { schedule: null } });
123
+ expect(res.ok).toBe(true);
124
+ expect(scheduleOf(await getPackage())).toBeNull();
125
+ });
126
+ });
@@ -0,0 +1,200 @@
1
+ /// <reference types="bun-types" />
2
+
3
+ import { afterAll, beforeAll, describe, expect, it } from "bun:test";
4
+ import fsp from "fs/promises";
5
+ import os from "os";
6
+ import path from "path";
7
+ import { fileURLToPath } from "url";
8
+ import { MaterializationScheduler } from "../../../src/service/materialization_scheduler";
9
+ import { MaterializationService } from "../../../src/service/materialization_service";
10
+ import { environmentStore } from "../../../src/server";
11
+ import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
12
+
13
+ const __filename = fileURLToPath(import.meta.url);
14
+ const __dirname = path.dirname(__filename);
15
+
16
+ const PROJECT_NAME = "sched-project";
17
+ const PACKAGE_NAME = "persist-schedule-test";
18
+ const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
19
+
20
+ const TERMINAL = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"];
21
+
22
+ // Deterministic clock for the scheduler: we call tick(nowMs) explicitly, so the
23
+ // fixture's daily cron ("0 6 * * *") fires without waiting on a real minute.
24
+ const T0 = Date.parse("2026-01-01T00:00:00Z");
25
+ const THREE_DAYS_MS = 3 * 24 * 60 * 60 * 1000;
26
+
27
+ describe("MaterializationScheduler (E2E, real build)", () => {
28
+ let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
29
+ let baseUrl: string;
30
+ let scheduler: MaterializationScheduler;
31
+
32
+ beforeAll(async () => {
33
+ env = await startRestE2E();
34
+ baseUrl = env.baseUrl;
35
+
36
+ const fixtureDir = path.resolve(
37
+ __dirname,
38
+ "../../fixtures/persist-schedule-test",
39
+ );
40
+ const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
41
+ method: "POST",
42
+ headers: { "Content-Type": "application/json" },
43
+ body: JSON.stringify({
44
+ name: PROJECT_NAME,
45
+ packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
46
+ connections: [],
47
+ }),
48
+ });
49
+ if (!createRes.ok) {
50
+ throw new Error(
51
+ `Failed to create test project (${createRes.status}): ${await createRes.text()}`,
52
+ );
53
+ }
54
+
55
+ // Wait for the package to load (so it's in the environmentStore the
56
+ // scheduler sweeps).
57
+ const deadline = Date.now() + 30_000;
58
+ while (Date.now() < deadline) {
59
+ const res = await fetch(`${baseUrl}${API}`);
60
+ if (res.ok) break;
61
+ await new Promise((r) => setTimeout(r, 500));
62
+ }
63
+
64
+ // Drive the real scheduler against the server's real environmentStore and
65
+ // a real MaterializationService (shares the same repository). We call
66
+ // tick() by hand with a controlled clock rather than relying on the timer.
67
+ scheduler = new MaterializationScheduler(
68
+ environmentStore,
69
+ new MaterializationService(environmentStore),
70
+ { tickIntervalMs: 60_000, maxFiresPerTick: 10 },
71
+ );
72
+ });
73
+
74
+ afterAll(async () => {
75
+ if (baseUrl) {
76
+ try {
77
+ await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
78
+ method: "DELETE",
79
+ });
80
+ } catch {
81
+ // best-effort
82
+ }
83
+ }
84
+ await env?.stop();
85
+ env = null;
86
+ });
87
+
88
+ async function listMaterializations(): Promise<
89
+ Array<Record<string, unknown>>
90
+ > {
91
+ const res = await fetch(`${baseUrl}${API}/materializations`);
92
+ expect(res.status).toBe(200);
93
+ return (await res.json()) as Array<Record<string, unknown>>;
94
+ }
95
+
96
+ function triggerOf(m: Record<string, unknown>): string | undefined {
97
+ return (m.metadata as { trigger?: string } | null)?.trigger;
98
+ }
99
+
100
+ async function pollTerminal(id: string): Promise<Record<string, unknown>> {
101
+ const deadline = Date.now() + 60_000;
102
+ while (Date.now() < deadline) {
103
+ const res = await fetch(`${baseUrl}${API}/materializations/${id}`);
104
+ expect(res.status).toBe(200);
105
+ const m = (await res.json()) as Record<string, unknown>;
106
+ if (TERMINAL.includes(m.status as string)) return m;
107
+ await new Promise((r) => setTimeout(r, 250));
108
+ }
109
+ throw new Error(`materialization ${id} did not reach a terminal state`);
110
+ }
111
+
112
+ it(
113
+ "fires a due scheduled package as a SCHEDULER build that completes",
114
+ async () => {
115
+ // Arm on the first tick (nextFire is strictly future, so no fire yet).
116
+ await scheduler.tick(T0);
117
+ expect(
118
+ (await listMaterializations()).filter(
119
+ (m) => triggerOf(m) === "SCHEDULER",
120
+ ).length,
121
+ ).toBe(0);
122
+
123
+ // Advance well past the daily cron: the sweep fires one SCHEDULER run.
124
+ await scheduler.tick(T0 + THREE_DAYS_MS);
125
+
126
+ const scheduled = (await listMaterializations()).filter(
127
+ (m) => triggerOf(m) === "SCHEDULER",
128
+ );
129
+ expect(scheduled.length).toBe(1);
130
+
131
+ // The real build settles at MANIFEST_FILE_READY, producing the table.
132
+ const built = await pollTerminal(scheduled[0].id as string);
133
+ expect(built.status).toBe("MANIFEST_FILE_READY");
134
+ const entries = (
135
+ built.manifest as { entries: Record<string, unknown> }
136
+ ).entries;
137
+ expect(Object.keys(entries).length).toBe(1);
138
+ // The SCHEDULER trigger must survive the build commit (not just exist
139
+ // while the run is in-flight) — otherwise a completed scheduled run is
140
+ // indistinguishable from a manual one.
141
+ expect(triggerOf(built)).toBe("SCHEDULER");
142
+
143
+ // Cleanup: drop the table + record so the negative case starts clean.
144
+ const del = await fetch(
145
+ `${baseUrl}${API}/materializations/${scheduled[0].id}?dropTables=true`,
146
+ { method: "DELETE" },
147
+ );
148
+ expect(del.status).toBe(204);
149
+ },
150
+ { timeout: 120_000 },
151
+ );
152
+
153
+ it(
154
+ "never fires a control-plane-driven package (manifestLocation set)",
155
+ async () => {
156
+ // Make the package orchestrated: point it at a (bind-clean) manifest.
157
+ const manifestFile = path.join(
158
+ os.tmpdir(),
159
+ `sched-empty-manifest-${T0}.json`,
160
+ );
161
+ await fsp.writeFile(
162
+ manifestFile,
163
+ JSON.stringify({
164
+ builtAt: new Date(T0).toISOString(),
165
+ strict: false,
166
+ entries: {},
167
+ }),
168
+ "utf8",
169
+ );
170
+ const patch = await fetch(`${baseUrl}${API}`, {
171
+ method: "PATCH",
172
+ headers: { "Content-Type": "application/json" },
173
+ body: JSON.stringify({
174
+ name: PACKAGE_NAME,
175
+ manifestLocation: manifestFile,
176
+ }),
177
+ });
178
+ expect(patch.ok).toBe(true);
179
+
180
+ const before = (await listMaterializations()).length;
181
+
182
+ // Even armed + well past the cron, an orchestrated package is skipped
183
+ // (Guard 2) — the control plane owns its refresh.
184
+ await scheduler.tick(T0);
185
+ await scheduler.tick(T0 + THREE_DAYS_MS);
186
+
187
+ expect((await listMaterializations()).length).toBe(before);
188
+
189
+ await fetch(`${baseUrl}${API}`, {
190
+ method: "PATCH",
191
+ headers: { "Content-Type": "application/json" },
192
+ body: JSON.stringify({
193
+ name: PACKAGE_NAME,
194
+ manifestLocation: null,
195
+ }),
196
+ });
197
+ },
198
+ { timeout: 60_000 },
199
+ );
200
+ });
@@ -0,0 +1,172 @@
1
+ /// <reference types="bun-types" />
2
+
3
+ import { afterAll, beforeAll, describe, expect, it } from "bun:test";
4
+ import path from "path";
5
+ import { fileURLToPath } from "url";
6
+ import { CronEvaluator } from "../../../src/service/cron_evaluator";
7
+ import { MaterializationScheduler } from "../../../src/service/materialization_scheduler";
8
+ import { MaterializationService } from "../../../src/service/materialization_service";
9
+ import { environmentStore } from "../../../src/server";
10
+ import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
11
+
12
+ const __filename = fileURLToPath(import.meta.url);
13
+ const __dirname = path.dirname(__filename);
14
+
15
+ const PROJECT_NAME = "sched-recovery-project";
16
+ const PACKAGE_NAME = "persist-schedule-test";
17
+ const CRON = "0 6 * * *"; // the fixture's daily schedule
18
+ const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
19
+ const CONFIG = { tickIntervalMs: 60_000, maxFiresPerTick: 10 };
20
+ const TERMINAL = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"];
21
+
22
+ // The restart-recovery anchor is only unit-tested against a stubbed
23
+ // getLatestScheduledFireAt. This drives the real sequence end to end: a real
24
+ // SCHEDULER fire lands a row in the store, then a *second* scheduler instance
25
+ // (a simulated process restart — empty in-memory arming state) re-anchors from
26
+ // that row via the real repo query. We assert exactly one catch-up when an
27
+ // occurrence came due during "downtime", and no double-fire when it didn't.
28
+ describe("MaterializationScheduler restart recovery (E2E, real store)", () => {
29
+ let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
30
+ let baseUrl: string;
31
+ let service: MaterializationService;
32
+ // A fixed anchor for the *first* scheduler's arming clock; the real fire's
33
+ // created_at (wall clock) is what recovery actually re-anchors from, and we
34
+ // read it back from the store rather than assuming it.
35
+ const T0 = Date.parse("2026-01-01T00:00:00Z");
36
+ const THREE_DAYS_MS = 3 * 24 * 60 * 60 * 1000;
37
+
38
+ beforeAll(async () => {
39
+ env = await startRestE2E();
40
+ baseUrl = env.baseUrl;
41
+ service = new MaterializationService(environmentStore);
42
+
43
+ const fixtureDir = path.resolve(
44
+ __dirname,
45
+ "../../fixtures/persist-schedule-test",
46
+ );
47
+ const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
48
+ method: "POST",
49
+ headers: { "Content-Type": "application/json" },
50
+ body: JSON.stringify({
51
+ name: PROJECT_NAME,
52
+ packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
53
+ connections: [],
54
+ }),
55
+ });
56
+ if (!createRes.ok) {
57
+ throw new Error(
58
+ `Failed to create test project (${createRes.status}): ${await createRes.text()}`,
59
+ );
60
+ }
61
+
62
+ // Wait for the package to load into the store the scheduler sweeps.
63
+ const deadline = Date.now() + 30_000;
64
+ while (Date.now() < deadline) {
65
+ const res = await fetch(`${baseUrl}${API}`);
66
+ if (res.ok) break;
67
+ await new Promise((r) => setTimeout(r, 500));
68
+ }
69
+ });
70
+
71
+ afterAll(async () => {
72
+ if (baseUrl) {
73
+ try {
74
+ await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
75
+ method: "DELETE",
76
+ });
77
+ } catch {
78
+ // best-effort
79
+ }
80
+ }
81
+ await env?.stop();
82
+ env = null;
83
+ });
84
+
85
+ function newScheduler(): MaterializationScheduler {
86
+ // A fresh instance == a process restart: no in-memory arming state, so the
87
+ // first arm re-anchors from the store.
88
+ return new MaterializationScheduler(environmentStore, service, CONFIG);
89
+ }
90
+
91
+ async function scheduledCount(): Promise<number> {
92
+ const res = await fetch(`${baseUrl}${API}/materializations`);
93
+ expect(res.status).toBe(200);
94
+ const rows = (await res.json()) as Array<Record<string, unknown>>;
95
+ return rows.filter(
96
+ (m) =>
97
+ (m.metadata as { trigger?: string } | null)?.trigger ===
98
+ "SCHEDULER",
99
+ ).length;
100
+ }
101
+
102
+ async function pollAllTerminal(): Promise<void> {
103
+ const deadline = Date.now() + 90_000;
104
+ while (Date.now() < deadline) {
105
+ const res = await fetch(`${baseUrl}${API}/materializations`);
106
+ const rows = (await res.json()) as Array<Record<string, unknown>>;
107
+ if (rows.every((m) => TERMINAL.includes(m.status as string))) return;
108
+ await new Promise((r) => setTimeout(r, 250));
109
+ }
110
+ throw new Error("materializations did not all reach a terminal state");
111
+ }
112
+
113
+ it(
114
+ "recovers exactly one missed occurrence across a restart, then jumps forward",
115
+ async () => {
116
+ const cron = new CronEvaluator();
117
+
118
+ // ── Scheduler #1: produce a real baseline SCHEDULER fire. ──
119
+ const s1 = newScheduler();
120
+ await s1.tick(T0); // arm (nextFire strictly future -> no fire)
121
+ expect(await scheduledCount()).toBe(0);
122
+ await s1.tick(T0 + THREE_DAYS_MS); // an occurrence is due -> one fire
123
+ expect(await scheduledCount()).toBe(1);
124
+ await pollAllTerminal();
125
+
126
+ // The instant recovery re-anchors from is the newest SCHEDULER row's
127
+ // created_at (wall clock), read through the same path the scheduler uses.
128
+ const lastFire = await service.getLatestScheduledFireAt(
129
+ PROJECT_NAME,
130
+ PACKAGE_NAME,
131
+ );
132
+ expect(lastFire).not.toBeNull();
133
+ const missedOccurrence = cron.nextAfter(CRON, lastFire!).getTime();
134
+
135
+ // ── Scheduler #2: simulated restart with an occurrence missed during
136
+ // downtime (now is just past the first occurrence after the last fire). ──
137
+ const s2 = newScheduler();
138
+ await s2.tick(missedOccurrence + 60_000); // due on first arm -> catch up
139
+ expect(await scheduledCount()).toBe(2); // exactly one catch-up
140
+ await pollAllTerminal();
141
+ // Advanced forward: the same instance does not fire again next tick.
142
+ await s2.tick(missedOccurrence + 120_000);
143
+ expect(await scheduledCount()).toBe(2);
144
+ },
145
+ { timeout: 200_000 },
146
+ );
147
+
148
+ it(
149
+ "does not double-fire on a restart when no occurrence was missed",
150
+ async () => {
151
+ const cron = new CronEvaluator();
152
+ const before = await scheduledCount();
153
+
154
+ // Latest SCHEDULER fire is the catch-up above (created_at ~ now). The
155
+ // next occurrence after it is the next 06:00 UTC, hours away.
156
+ const lastFire = await service.getLatestScheduledFireAt(
157
+ PROJECT_NAME,
158
+ PACKAGE_NAME,
159
+ );
160
+ expect(lastFire).not.toBeNull();
161
+ const nextOccurrence = cron.nextAfter(CRON, lastFire!).getTime();
162
+
163
+ // ── Scheduler #3: simulated restart shortly after the last fire, before
164
+ // the next occurrence is due -> arms into the future, no catch-up. ──
165
+ const s3 = newScheduler();
166
+ await s3.tick(lastFire!.getTime() + 1_000);
167
+ expect(await scheduledCount()).toBe(before);
168
+ expect(lastFire!.getTime() + 1_000).toBeLessThan(nextOccurrence);
169
+ },
170
+ { timeout: 120_000 },
171
+ );
172
+ });
@@ -0,0 +1,143 @@
1
+ /// <reference types="bun-types" />
2
+
3
+ import { afterAll, beforeAll, describe, expect, it } from "bun:test";
4
+ import path from "path";
5
+ import { fileURLToPath } from "url";
6
+ import type { MaterializationScheduler } from "../../../src/service/materialization_scheduler";
7
+ import { MaterializationService } from "../../../src/service/materialization_service";
8
+ import {
9
+ environmentStore,
10
+ startMaterializationSchedulerFromEnv,
11
+ } from "../../../src/server";
12
+ import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
13
+
14
+ const __filename = fileURLToPath(import.meta.url);
15
+ const __dirname = path.dirname(__filename);
16
+
17
+ const PROJECT_NAME = "sched-wiring-project";
18
+ const PACKAGE_NAME = "persist-schedule-test";
19
+ const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
20
+
21
+ const ENABLE = "PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER";
22
+ const INTERVAL = "PUBLISHER_MATERIALIZATION_SCHEDULER_INTERVAL_MS";
23
+
24
+ // The other scheduler tests hand-construct MaterializationScheduler and call
25
+ // tick() by hand, bypassing the env-var → getMaterializationSchedulerConfig →
26
+ // construct → start()/unref() → real-timer path an operator actually runs. This
27
+ // exercises that wiring: set the flag + a short interval, start the scheduler
28
+ // through the same seam server.ts uses at boot, and observe a build fire through
29
+ // the real timer (not a hand-driven tick).
30
+ describe("MaterializationScheduler wiring (E2E, real env-var timer)", () => {
31
+ let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
32
+ let baseUrl: string;
33
+ let scheduler: MaterializationScheduler | null = null;
34
+ const prev: Record<string, string | undefined> = {};
35
+
36
+ beforeAll(async () => {
37
+ env = await startRestE2E();
38
+ baseUrl = env.baseUrl;
39
+
40
+ const fixtureDir = path.resolve(
41
+ __dirname,
42
+ "../../fixtures/persist-schedule-test",
43
+ );
44
+ const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
45
+ method: "POST",
46
+ headers: { "Content-Type": "application/json" },
47
+ body: JSON.stringify({
48
+ name: PROJECT_NAME,
49
+ packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
50
+ connections: [],
51
+ }),
52
+ });
53
+ if (!createRes.ok) {
54
+ throw new Error(
55
+ `Failed to create test project (${createRes.status}): ${await createRes.text()}`,
56
+ );
57
+ }
58
+
59
+ // Wait for the package to load, then switch it to an every-minute cron so
60
+ // the real timer fires within the test window (the fixture ships a daily
61
+ // cron, which a wall-clock test can't wait on).
62
+ const deadline = Date.now() + 30_000;
63
+ while (Date.now() < deadline) {
64
+ const res = await fetch(`${baseUrl}${API}`);
65
+ if (res.ok) break;
66
+ await new Promise((r) => setTimeout(r, 500));
67
+ }
68
+ const patch = await fetch(`${baseUrl}${API}`, {
69
+ method: "PATCH",
70
+ headers: { "Content-Type": "application/json" },
71
+ body: JSON.stringify({
72
+ name: PACKAGE_NAME,
73
+ materialization: { schedule: "* * * * *" },
74
+ }),
75
+ });
76
+ expect(patch.ok).toBe(true);
77
+ });
78
+
79
+ afterAll(async () => {
80
+ scheduler?.stop();
81
+ // Restore env so the shared test process isn't left with the flag set.
82
+ for (const key of [ENABLE, INTERVAL]) {
83
+ if (prev[key] === undefined) delete process.env[key];
84
+ else process.env[key] = prev[key];
85
+ }
86
+ if (baseUrl) {
87
+ try {
88
+ await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
89
+ method: "DELETE",
90
+ });
91
+ } catch {
92
+ // best-effort
93
+ }
94
+ }
95
+ await env?.stop();
96
+ env = null;
97
+ });
98
+
99
+ it(
100
+ "boots the scheduler from env config and fires a build through the real timer",
101
+ async () => {
102
+ // Enable via the real env-var path and use the minimum 1s sweep so the
103
+ // every-minute cron is picked up promptly once its boundary passes.
104
+ prev[ENABLE] = process.env[ENABLE];
105
+ prev[INTERVAL] = process.env[INTERVAL];
106
+ process.env[ENABLE] = "true";
107
+ process.env[INTERVAL] = "1000";
108
+
109
+ scheduler = startMaterializationSchedulerFromEnv(
110
+ environmentStore,
111
+ new MaterializationService(environmentStore),
112
+ );
113
+ // The config path must have constructed + started a scheduler (a null
114
+ // here would mean the enable flag wasn't honored).
115
+ expect(scheduler).not.toBeNull();
116
+
117
+ // Wait for the real timer to fire a SCHEDULER build and for it to build.
118
+ const deadline = Date.now() + 150_000;
119
+ let built: Record<string, unknown> | null = null;
120
+ while (Date.now() < deadline && !built) {
121
+ const res = await fetch(`${baseUrl}${API}/materializations`);
122
+ const rows = (await res.json()) as Array<Record<string, unknown>>;
123
+ built =
124
+ rows.find(
125
+ (m) =>
126
+ (m.metadata as { trigger?: string } | null)?.trigger ===
127
+ "SCHEDULER" && m.status === "MANIFEST_FILE_READY",
128
+ ) ?? null;
129
+ if (!built) await new Promise((r) => setTimeout(r, 1_000));
130
+ }
131
+
132
+ expect(built).not.toBeNull();
133
+ const entries = (
134
+ built!.manifest as { entries: Record<string, unknown> }
135
+ ).entries;
136
+ expect(Object.keys(entries).length).toBe(1);
137
+
138
+ // Stop the timer before the test ends so it can't fire into other files.
139
+ scheduler!.stop();
140
+ },
141
+ { timeout: 180_000 },
142
+ );
143
+ });
@@ -1 +0,0 @@
1
- import{W as r,G as t,j as e,$ as i,a5 as o}from"./index-EqslXZ44.js";function m(){const s=r(),{environmentName:n}=t();if(n){const a=i({environmentName:n});return e.jsx(o,{onSelectPackage:s,resourceUri:a})}else return e.jsx("div",{children:e.jsx("h2",{children:"Missing environment name"})})}export{m as default};