@malloy-publisher/server 0.0.228 → 0.0.230
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +102 -10
- package/dist/app/assets/EnvironmentPage-wa_EPkwK.js +1 -0
- package/dist/app/assets/{HomePage-Bkwc9Woc.js → HomePage-jnCrupQp.js} +1 -1
- package/dist/app/assets/{LightMode-Bum_KBpN.js → LightMode-DYbwNULZ.js} +1 -1
- package/dist/app/assets/MainPage-CuJLrPNI.js +2 -0
- package/dist/app/assets/MaterializationsPage-D_67x2ee.js +1 -0
- package/dist/app/assets/ModelPage-D5JtAWqR.js +1 -0
- package/dist/app/assets/PackagePage-BRwtqUSG.js +1 -0
- package/dist/app/assets/{RouteError-DmJLpLXm.js → RouteError-CBNNrnSD.js} +1 -1
- package/dist/app/assets/ThemeEditorPage-CTCeBneA.js +1 -0
- package/dist/app/assets/WorkbookPage-SN6f1RBm.js +1 -0
- package/dist/app/assets/{core-CEDZMHV1.es-_yGzNgNH.js → core-Dp3q5Ieu.es-CD5FvM2s.js} +10 -10
- package/dist/app/assets/{index-CdmFub34.js → index-B3Nn8Vm2.js} +459 -446
- package/dist/app/assets/index-BLCx1EdC.js +18 -0
- package/dist/app/assets/index-C_tJstcx.js +529 -0
- package/dist/app/assets/{index-CE9xhdra.js → index-CfmBVB6M.js} +1 -1
- package/dist/app/assets/{index-OEjKNSYb.js → index-DU4r7GdU.js} +573 -413
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +1 -1
- package/dist/package_load_worker.mjs +109 -8
- package/dist/server.mjs +7504 -80
- package/package.json +15 -13
- package/scripts/bake-duckdb-extensions.js +5 -2
- package/src/config.spec.ts +87 -1
- package/src/config.ts +142 -1
- package/src/controller/materialization.controller.spec.ts +9 -0
- package/src/controller/materialization.controller.ts +15 -0
- package/src/ducklake_version.spec.ts +163 -0
- package/src/ducklake_version.ts +153 -0
- package/src/errors.ts +12 -0
- package/src/malloy_pin_prereqs.spec.ts +25 -0
- package/src/materialization_metrics.ts +33 -0
- package/src/mcp/skills/build_skills_bundle.ts +29 -9
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/skills/skills_bundle.spec.ts +31 -2
- package/src/query_param_utils.ts +11 -0
- package/src/server.ts +74 -9
- package/src/service/authorize.spec.ts +22 -0
- package/src/service/authorize.ts +267 -10
- package/src/service/authorize_integration.spec.ts +1068 -26
- package/src/service/connection.spec.ts +79 -7
- package/src/service/connection.ts +312 -25
- package/src/service/connection_config.spec.ts +21 -0
- package/src/service/connection_config.ts +15 -1
- package/src/service/cron_evaluator.spec.ts +79 -0
- package/src/service/cron_evaluator.ts +105 -0
- package/src/service/ducklake_lazy_attach.spec.ts +110 -0
- package/src/service/environment.ts +156 -17
- package/src/service/environment_store.spec.ts +569 -14
- package/src/service/environment_store.ts +237 -31
- package/src/service/environment_store_anchoring.spec.ts +107 -0
- package/src/service/extension_fetch_policy.spec.ts +256 -0
- package/src/service/manifest_loader.spec.ts +3 -1
- package/src/service/manifest_loader.ts +9 -5
- package/src/service/materialization_scheduler.spec.ts +289 -0
- package/src/service/materialization_scheduler.ts +304 -0
- package/src/service/materialization_service.spec.ts +286 -5
- package/src/service/materialization_service.ts +121 -8
- package/src/service/materialization_test_fixtures.ts +10 -1
- package/src/service/model.spec.ts +67 -0
- package/src/service/model.ts +656 -31
- package/src/service/package.spec.ts +38 -0
- package/src/service/package.ts +102 -2
- package/src/service/package_quote_bound_tables.spec.ts +171 -0
- package/src/service/persistence_policy.spec.ts +31 -0
- package/src/service/quoting.spec.ts +57 -1
- package/src/service/quoting.ts +33 -0
- package/src/storage/DatabaseInterface.ts +15 -0
- package/src/storage/duckdb/DuckDBRepository.ts +17 -0
- package/src/storage/duckdb/MaterializationRepository.spec.ts +191 -0
- package/src/storage/duckdb/MaterializationRepository.ts +68 -0
- package/tests/fixtures/persist-schedule-test/data/orders.csv +5 -0
- package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +11 -0
- package/tests/fixtures/persist-schedule-test/publisher.json +7 -0
- package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +5 -0
- package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +11 -0
- package/tests/fixtures/persist-schedule-warn-test/publisher.json +6 -0
- package/tests/integration/materialization/environment_list.integration.spec.ts +133 -0
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +7 -2
- package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +150 -0
- package/tests/integration/materialization/schedule_edit.integration.spec.ts +126 -0
- package/tests/integration/materialization/scheduler.integration.spec.ts +200 -0
- package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +172 -0
- package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +256 -0
- package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +143 -0
- package/dist/app/assets/EnvironmentPage-EW2lbGvb.js +0 -1
- package/dist/app/assets/MainPage-oiEy7TNM.js +0 -2
- package/dist/app/assets/MaterializationsPage-C_VJsTgU.js +0 -1
- package/dist/app/assets/ModelPage-z8REqAmk.js +0 -1
- package/dist/app/assets/PackagePage-C2Vtt1Ln.js +0 -1
- package/dist/app/assets/ThemeEditorPage-BywFjC7A.js +0 -1
- package/dist/app/assets/WorkbookPage-DCMizDMR.js +0 -1
- package/dist/app/assets/index-DDMrjIT3.js +0 -18
- package/dist/app/assets/index-EqslXZ44.js +0 -527
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
import type { MaterializationSchedulerConfig } from "../config";
|
|
2
|
+
import { MaterializationConflictError } from "../errors";
|
|
3
|
+
import { logger } from "../logger";
|
|
4
|
+
import { recordScheduledFire } from "../materialization_metrics";
|
|
5
|
+
import { CronEvaluator } from "./cron_evaluator";
|
|
6
|
+
import type { EnvironmentStore } from "./environment_store";
|
|
7
|
+
import type { MaterializationService } from "./materialization_service";
|
|
8
|
+
|
|
9
|
+
/** Per-package arming state, keyed by `${environmentName}::${packageName}`. */
|
|
10
|
+
interface ScheduleState {
|
|
11
|
+
/** The cron the state was armed from; a change re-arms `nextFireAtMs`. */
|
|
12
|
+
schedule: string;
|
|
13
|
+
/** Epoch ms of the next due fire (`cron.nextAfter`, strictly future when armed). */
|
|
14
|
+
nextFireAtMs: number;
|
|
15
|
+
/** Epoch ms of the last fire attempt, or null if never fired. */
|
|
16
|
+
lastFiredAtMs: number | null;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Standalone materialization scheduler: fires a loaded package's
|
|
21
|
+
* `materialization.schedule` cron so an open-source publisher (no control plane)
|
|
22
|
+
* can rebuild on a cadence. Modeled on {@link PackageMemoryGovernor} — a
|
|
23
|
+
* config-driven `setInterval` with `start`/`stop` and one `tick`.
|
|
24
|
+
*
|
|
25
|
+
* **Orchestration safety (two independent guards):**
|
|
26
|
+
* 1. Disabled by default — only constructed when
|
|
27
|
+
* {@link getMaterializationSchedulerConfig} returns non-null
|
|
28
|
+
* (`PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER`), which an orchestrated
|
|
29
|
+
* deployment never sets, so the scheduler never runs there.
|
|
30
|
+
* 2. Even when enabled, a package with a configured `manifestLocation` (i.e.
|
|
31
|
+
* control-plane-driven) is skipped — the CP already fires it, so we never
|
|
32
|
+
* double-drive.
|
|
33
|
+
*
|
|
34
|
+
* These guards are **independent and both load-bearing — never collapse them to
|
|
35
|
+
* just the `manifestLocation` check.** A control-plane-loaded package that is
|
|
36
|
+
* *serving live* has `manifestLocation === null`, so Guard 2 does not cover it;
|
|
37
|
+
* only Guard 1 (the opt-in flag, which an orchestrated worker must never set —
|
|
38
|
+
* see {@link getMaterializationSchedulerConfig}) keeps the scheduler off there.
|
|
39
|
+
*
|
|
40
|
+
* It only sweeps **already-loaded** packages (never forces a load), fires the
|
|
41
|
+
* existing `createMaterialization` auto-run path tagged `trigger=SCHEDULER`, and
|
|
42
|
+
* only fires packages whose persistence policy is valid (`scope: version`, no
|
|
43
|
+
* `freshness` — the same rule a publish enforces). Arming state is in memory,
|
|
44
|
+
* but on the first arm after a (re)start it re-anchors from the newest recorded
|
|
45
|
+
* `SCHEDULER` fire (see {@link arm}), so an occurrence that came due during
|
|
46
|
+
* downtime fires exactly once and then jumps forward — recovery matches the
|
|
47
|
+
* control plane's persisted-`next_fire_at` behavior rather than silently
|
|
48
|
+
* skipping. A never-fired schedule still does not fire on the arming tick
|
|
49
|
+
* (`nextAfter(now)` is strictly future).
|
|
50
|
+
*/
|
|
51
|
+
export class MaterializationScheduler {
|
|
52
|
+
private timer: ReturnType<typeof setInterval> | null = null;
|
|
53
|
+
private readonly cron: CronEvaluator;
|
|
54
|
+
private readonly state = new Map<string, ScheduleState>();
|
|
55
|
+
|
|
56
|
+
constructor(
|
|
57
|
+
private readonly environmentStore: EnvironmentStore,
|
|
58
|
+
private readonly materializationService: MaterializationService,
|
|
59
|
+
private readonly config: MaterializationSchedulerConfig,
|
|
60
|
+
cron?: CronEvaluator,
|
|
61
|
+
) {
|
|
62
|
+
this.cron = cron ?? new CronEvaluator();
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Begin the periodic sweep. Idempotent; the interval is `unref`'d. */
|
|
66
|
+
public start(): void {
|
|
67
|
+
if (this.timer !== null) return;
|
|
68
|
+
this.timer = setInterval(() => {
|
|
69
|
+
void this.tick();
|
|
70
|
+
}, this.config.tickIntervalMs);
|
|
71
|
+
(
|
|
72
|
+
this.timer as ReturnType<typeof setInterval> & { unref?: () => void }
|
|
73
|
+
).unref?.();
|
|
74
|
+
logger.info(
|
|
75
|
+
`MaterializationScheduler started (interval=${this.config.tickIntervalMs}ms, maxFiresPerTick=${this.config.maxFiresPerTick})`,
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Stop the periodic sweep. Idempotent. */
|
|
80
|
+
public stop(): void {
|
|
81
|
+
if (this.timer !== null) {
|
|
82
|
+
clearInterval(this.timer);
|
|
83
|
+
this.timer = null;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* One sweep: arm newly-seen schedules, fire due ones (capped per tick), and
|
|
89
|
+
* prune state for packages no longer loaded/scheduled. Exposed so tests can
|
|
90
|
+
* drive it synchronously. Never throws — per-package errors are isolated.
|
|
91
|
+
*/
|
|
92
|
+
public async tick(nowMs: number = Date.now()): Promise<void> {
|
|
93
|
+
const seen = new Set<string>();
|
|
94
|
+
let fired = 0;
|
|
95
|
+
|
|
96
|
+
for (const env of this.environmentStore.getLoadedEnvironments()) {
|
|
97
|
+
const environmentName = env.getEnvironmentName();
|
|
98
|
+
for (const pkg of env.getLoadedPackages()) {
|
|
99
|
+
const packageName = pkg.getPackageName();
|
|
100
|
+
const key = `${environmentName}::${packageName}`;
|
|
101
|
+
try {
|
|
102
|
+
const schedule = this.schedulableCron(pkg);
|
|
103
|
+
if (schedule === null) {
|
|
104
|
+
// Not a standalone-schedulable package (no cron, orchestrated,
|
|
105
|
+
// or invalid policy/cron) — drop any stale arming state.
|
|
106
|
+
this.state.delete(key);
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
seen.add(key);
|
|
110
|
+
|
|
111
|
+
const armed = await this.arm(
|
|
112
|
+
environmentName,
|
|
113
|
+
packageName,
|
|
114
|
+
key,
|
|
115
|
+
schedule,
|
|
116
|
+
nowMs,
|
|
117
|
+
);
|
|
118
|
+
if (nowMs < armed.nextFireAtMs) continue; // not due yet
|
|
119
|
+
|
|
120
|
+
if (fired >= this.config.maxFiresPerTick) {
|
|
121
|
+
// Stampede guard: leave it due; it fires on a later tick.
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
await this.fire(environmentName, packageName);
|
|
125
|
+
fired++;
|
|
126
|
+
// Advance regardless of outcome so a persistent failure retries on
|
|
127
|
+
// the next cron occurrence (cron cadence), never every tick.
|
|
128
|
+
armed.nextFireAtMs = this.cron
|
|
129
|
+
.nextAfter(schedule, new Date(nowMs))
|
|
130
|
+
.getTime();
|
|
131
|
+
armed.lastFiredAtMs = nowMs;
|
|
132
|
+
} catch (err) {
|
|
133
|
+
logger.warn("MaterializationScheduler: package sweep failed", {
|
|
134
|
+
environmentName,
|
|
135
|
+
packageName,
|
|
136
|
+
error: err instanceof Error ? err.message : String(err),
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
for (const key of [...this.state.keys()]) {
|
|
143
|
+
if (!seen.has(key)) this.state.delete(key);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* The package's schedulable cron, or null when it must be skipped: no
|
|
149
|
+
* `materialization.schedule`; a `manifestLocation` (control-plane-driven —
|
|
150
|
+
* Guard 2); an invalid persistence policy (e.g. schedule without
|
|
151
|
+
* `scope: version`, or schedule + freshness — the publish-gate rules); or an
|
|
152
|
+
* unparseable cron. Skips are debug-logged (the policy issues are already
|
|
153
|
+
* warned at package load), not warned each tick.
|
|
154
|
+
*/
|
|
155
|
+
private schedulableCron(pkg: {
|
|
156
|
+
getPackageMetadata(): {
|
|
157
|
+
materialization?: { schedule?: string | null } | null;
|
|
158
|
+
manifestLocation?: string | null;
|
|
159
|
+
};
|
|
160
|
+
persistencePolicyWarnings(): string[];
|
|
161
|
+
getPackageName(): string;
|
|
162
|
+
}): string | null {
|
|
163
|
+
const metadata = pkg.getPackageMetadata();
|
|
164
|
+
const schedule = metadata.materialization?.schedule ?? null;
|
|
165
|
+
if (!schedule) return null;
|
|
166
|
+
if (metadata.manifestLocation) return null; // orchestrated — CP fires it
|
|
167
|
+
if (pkg.persistencePolicyWarnings().length > 0) {
|
|
168
|
+
logger.debug(
|
|
169
|
+
"MaterializationScheduler: skipping package with invalid persistence policy",
|
|
170
|
+
{ packageName: pkg.getPackageName() },
|
|
171
|
+
);
|
|
172
|
+
return null;
|
|
173
|
+
}
|
|
174
|
+
if (!this.cron.isValid(schedule)) {
|
|
175
|
+
logger.debug("MaterializationScheduler: skipping invalid cron", {
|
|
176
|
+
packageName: pkg.getPackageName(),
|
|
177
|
+
schedule,
|
|
178
|
+
});
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
return schedule;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Return the package's arming state, (re)arming when it is new or its cron
|
|
186
|
+
* changed.
|
|
187
|
+
*
|
|
188
|
+
* **First arm of a package this process** (no in-memory state — a fresh start
|
|
189
|
+
* or a newly-loaded package): anchor `nextFireAt` to the newest recorded
|
|
190
|
+
* `SCHEDULER` fire, `nextAfter(lastScheduledFire)`. If that instant is already
|
|
191
|
+
* past (an occurrence came due while the publisher was down), it fires once
|
|
192
|
+
* on this tick and then jumps forward to `nextAfter(now)` — exactly one
|
|
193
|
+
* catch-up, matching the control plane's persisted-`next_fire_at` recovery
|
|
194
|
+
* contract, never a burst. With no prior `SCHEDULER` fire on record we fall
|
|
195
|
+
* back to `nextAfter(now)` (strictly future), so a never-fired schedule does
|
|
196
|
+
* not fire on the arming tick. (Residual divergence: a schedule set while the
|
|
197
|
+
* scheduler was disabled has no prior fire, so it is not caught up on first
|
|
198
|
+
* enable.)
|
|
199
|
+
*
|
|
200
|
+
* **Cron changed**: re-arm from `nextAfter(now)` — an edited cadence starts
|
|
201
|
+
* fresh, with no catch-up for the old cron.
|
|
202
|
+
*/
|
|
203
|
+
private async arm(
|
|
204
|
+
environmentName: string,
|
|
205
|
+
packageName: string,
|
|
206
|
+
key: string,
|
|
207
|
+
schedule: string,
|
|
208
|
+
nowMs: number,
|
|
209
|
+
): Promise<ScheduleState> {
|
|
210
|
+
const existing = this.state.get(key);
|
|
211
|
+
if (existing && existing.schedule === schedule) {
|
|
212
|
+
return existing;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
const anchor =
|
|
216
|
+
existing === undefined
|
|
217
|
+
? await this.recoveryAnchor(environmentName, packageName, nowMs)
|
|
218
|
+
: new Date(nowMs);
|
|
219
|
+
|
|
220
|
+
const armed: ScheduleState = {
|
|
221
|
+
schedule,
|
|
222
|
+
nextFireAtMs: this.cron.nextAfter(schedule, anchor).getTime(),
|
|
223
|
+
lastFiredAtMs: existing?.lastFiredAtMs ?? null,
|
|
224
|
+
};
|
|
225
|
+
this.state.set(key, armed);
|
|
226
|
+
return armed;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* The instant to compute the first `nextFireAt` from after a (re)start: the
|
|
231
|
+
* newest recorded `SCHEDULER` fire if any, else `now`. Any lookup failure
|
|
232
|
+
* degrades to `now` (no catch-up) so arming always succeeds.
|
|
233
|
+
*/
|
|
234
|
+
private async recoveryAnchor(
|
|
235
|
+
environmentName: string,
|
|
236
|
+
packageName: string,
|
|
237
|
+
nowMs: number,
|
|
238
|
+
): Promise<Date> {
|
|
239
|
+
try {
|
|
240
|
+
const last =
|
|
241
|
+
await this.materializationService.getLatestScheduledFireAt(
|
|
242
|
+
environmentName,
|
|
243
|
+
packageName,
|
|
244
|
+
);
|
|
245
|
+
return last ?? new Date(nowMs);
|
|
246
|
+
} catch (err) {
|
|
247
|
+
logger.warn(
|
|
248
|
+
"MaterializationScheduler: could not read last scheduled fire; arming from now",
|
|
249
|
+
{
|
|
250
|
+
environmentName,
|
|
251
|
+
packageName,
|
|
252
|
+
error: err instanceof Error ? err.message : String(err),
|
|
253
|
+
},
|
|
254
|
+
);
|
|
255
|
+
return new Date(nowMs);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* Fire a whole-package scheduled rebuild via the existing auto-run path. An
|
|
261
|
+
* already-active materialization coalesces (the in-flight build covers this
|
|
262
|
+
* occurrence); any other error is logged and metered but never propagated.
|
|
263
|
+
*
|
|
264
|
+
* **Standalone fires are tables-only.** This path runs a single-phase build
|
|
265
|
+
* that materializes the package's persist sources into tables. A hosted
|
|
266
|
+
* (control-plane-driven) deployment runs a two-phase job — tables, then a
|
|
267
|
+
* second pass over indexed dimensions — so index behavior is a hosted-only
|
|
268
|
+
* concern that cannot be exercised by the standalone scheduler. Anything a
|
|
269
|
+
* schedule fires locally is exactly what the `createMaterialization`
|
|
270
|
+
* auto-run produces on demand; only the `trigger` label differs.
|
|
271
|
+
*/
|
|
272
|
+
private async fire(
|
|
273
|
+
environmentName: string,
|
|
274
|
+
packageName: string,
|
|
275
|
+
): Promise<void> {
|
|
276
|
+
try {
|
|
277
|
+
await this.materializationService.createMaterialization(
|
|
278
|
+
environmentName,
|
|
279
|
+
packageName,
|
|
280
|
+
{ forceRefresh: true, trigger: "SCHEDULER" },
|
|
281
|
+
);
|
|
282
|
+
recordScheduledFire("fired");
|
|
283
|
+
logger.info("MaterializationScheduler: fired scheduled rebuild", {
|
|
284
|
+
environmentName,
|
|
285
|
+
packageName,
|
|
286
|
+
});
|
|
287
|
+
} catch (err) {
|
|
288
|
+
if (err instanceof MaterializationConflictError) {
|
|
289
|
+
recordScheduledFire("conflict");
|
|
290
|
+
logger.debug(
|
|
291
|
+
"MaterializationScheduler: skipped fire; a materialization is already active",
|
|
292
|
+
{ environmentName, packageName },
|
|
293
|
+
);
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
296
|
+
recordScheduledFire("error");
|
|
297
|
+
logger.warn("MaterializationScheduler: scheduled fire failed", {
|
|
298
|
+
environmentName,
|
|
299
|
+
packageName,
|
|
300
|
+
error: err instanceof Error ? err.message : String(err),
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Connection as MalloyConnection } from "@malloydata/malloy";
|
|
2
2
|
import { Manifest } from "@malloydata/malloy";
|
|
3
|
-
import { beforeEach, describe, expect, it } from "bun:test";
|
|
3
|
+
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
|
4
4
|
import * as sinon from "sinon";
|
|
5
5
|
import {
|
|
6
6
|
BadRequestError,
|
|
@@ -27,6 +27,11 @@ import {
|
|
|
27
27
|
MaterializationService,
|
|
28
28
|
stagingSuffix,
|
|
29
29
|
} from "./materialization_service";
|
|
30
|
+
import { resetMaterializationTelemetryForTesting } from "../materialization_metrics";
|
|
31
|
+
import {
|
|
32
|
+
startMetricsHarness,
|
|
33
|
+
type MetricsHarness,
|
|
34
|
+
} from "../test_helpers/metrics_harness";
|
|
30
35
|
|
|
31
36
|
type MockRepo = sinon.SinonStubbedInstance<ResourceRepository>;
|
|
32
37
|
|
|
@@ -166,6 +171,7 @@ describe("MaterializationService", () => {
|
|
|
166
171
|
forceRefresh: true,
|
|
167
172
|
sourceNames: ["orders"],
|
|
168
173
|
mode: "auto",
|
|
174
|
+
trigger: "ON_DEMAND",
|
|
169
175
|
},
|
|
170
176
|
]);
|
|
171
177
|
});
|
|
@@ -183,10 +189,27 @@ describe("MaterializationService", () => {
|
|
|
183
189
|
forceRefresh: false,
|
|
184
190
|
sourceNames: null,
|
|
185
191
|
mode: "auto",
|
|
192
|
+
trigger: "ON_DEMAND",
|
|
186
193
|
},
|
|
187
194
|
);
|
|
188
195
|
});
|
|
189
196
|
|
|
197
|
+
it("records trigger=SCHEDULER when the scheduler fires the run", async () => {
|
|
198
|
+
ctx.repository.getActiveMaterialization.resolves(null);
|
|
199
|
+
ctx.repository.createMaterialization.resolves(
|
|
200
|
+
makeMaterialization({ status: "PENDING" }),
|
|
201
|
+
);
|
|
202
|
+
|
|
203
|
+
await ctx.service.createMaterialization("my-env", "pkg", {
|
|
204
|
+
forceRefresh: true,
|
|
205
|
+
trigger: "SCHEDULER",
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
expect(
|
|
209
|
+
ctx.repository.createMaterialization.firstCall.args[3],
|
|
210
|
+
).toMatchObject({ mode: "auto", trigger: "SCHEDULER" });
|
|
211
|
+
});
|
|
212
|
+
|
|
190
213
|
it("rejects when an active materialization already exists", async () => {
|
|
191
214
|
ctx.repository.getActiveMaterialization.resolves(
|
|
192
215
|
makeMaterialization({
|
|
@@ -482,6 +505,86 @@ describe("MaterializationService", () => {
|
|
|
482
505
|
});
|
|
483
506
|
});
|
|
484
507
|
|
|
508
|
+
describe("deleteMaterialization telemetry", () => {
|
|
509
|
+
let ctx: ReturnType<typeof createMocks>;
|
|
510
|
+
let harness: MetricsHarness;
|
|
511
|
+
|
|
512
|
+
beforeEach(async () => {
|
|
513
|
+
ctx = createMocks();
|
|
514
|
+
harness = await startMetricsHarness();
|
|
515
|
+
// Drop cached instruments so recordDropTables re-binds to this provider.
|
|
516
|
+
resetMaterializationTelemetryForTesting();
|
|
517
|
+
});
|
|
518
|
+
|
|
519
|
+
afterEach(async () => {
|
|
520
|
+
resetMaterializationTelemetryForTesting();
|
|
521
|
+
await harness.shutdown();
|
|
522
|
+
});
|
|
523
|
+
|
|
524
|
+
const DROP_COUNTER = "publisher_materialization_drop_tables_total";
|
|
525
|
+
|
|
526
|
+
// A terminal materialization whose manifest names one physical table, so
|
|
527
|
+
// dropMaterializedTables issues exactly one DROP — the emission under test.
|
|
528
|
+
function dropTablesFixture(runSQL: sinon.SinonStub) {
|
|
529
|
+
const getMalloyConnection = sinon
|
|
530
|
+
.stub()
|
|
531
|
+
.resolves({ runSQL, dialectName: "duckdb" });
|
|
532
|
+
(ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
|
|
533
|
+
getPackage: sinon.stub().resolves({ getMalloyConnection }),
|
|
534
|
+
withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
|
|
535
|
+
fn(),
|
|
536
|
+
});
|
|
537
|
+
ctx.repository.getMaterializationById.resolves(
|
|
538
|
+
makeMaterialization({
|
|
539
|
+
status: "MANIFEST_FILE_READY",
|
|
540
|
+
manifest: {
|
|
541
|
+
builtAt: new Date().toISOString(),
|
|
542
|
+
strict: false,
|
|
543
|
+
entries: {
|
|
544
|
+
b1: {
|
|
545
|
+
sourceEntityId: "b1",
|
|
546
|
+
physicalTableName: "orders_mz",
|
|
547
|
+
connectionName: "duckdb",
|
|
548
|
+
},
|
|
549
|
+
},
|
|
550
|
+
},
|
|
551
|
+
}),
|
|
552
|
+
);
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
it("meters a successful physical-table drop as outcome=success", async () => {
|
|
556
|
+
dropTablesFixture(sinon.stub().resolves());
|
|
557
|
+
|
|
558
|
+
await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
|
|
559
|
+
dropTables: true,
|
|
560
|
+
});
|
|
561
|
+
|
|
562
|
+
expect(
|
|
563
|
+
await harness.collectCounter(DROP_COUNTER, { outcome: "success" }),
|
|
564
|
+
).toBe(1);
|
|
565
|
+
expect(
|
|
566
|
+
await harness.collectCounter(DROP_COUNTER, { outcome: "failure" }),
|
|
567
|
+
).toBe(0);
|
|
568
|
+
});
|
|
569
|
+
|
|
570
|
+
it("meters a failed drop as outcome=failure and still deletes the record", async () => {
|
|
571
|
+
dropTablesFixture(sinon.stub().rejects(new Error("drop boom")));
|
|
572
|
+
|
|
573
|
+
// The drop is best-effort: a failure is metered but never surfaces to the
|
|
574
|
+
// caller, and the record is still removed.
|
|
575
|
+
await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
|
|
576
|
+
dropTables: true,
|
|
577
|
+
});
|
|
578
|
+
|
|
579
|
+
expect(
|
|
580
|
+
await harness.collectCounter(DROP_COUNTER, { outcome: "failure" }),
|
|
581
|
+
).toBe(1);
|
|
582
|
+
expect(ctx.repository.deleteMaterialization.calledOnceWith("mat-1")).toBe(
|
|
583
|
+
true,
|
|
584
|
+
);
|
|
585
|
+
});
|
|
586
|
+
});
|
|
587
|
+
|
|
485
588
|
describe("stagingSuffix", () => {
|
|
486
589
|
it("derives a short, stable suffix from the sourceEntityId", () => {
|
|
487
590
|
expect(stagingSuffix("abcdef1234567890")).toBe("_abcdef123456");
|
|
@@ -788,6 +891,110 @@ describe("executeInstructedBuild", () => {
|
|
|
788
891
|
expect(rootIdx).toBeGreaterThan(midIdx);
|
|
789
892
|
});
|
|
790
893
|
|
|
894
|
+
it("seeds a downstream build with the QUOTED upstream reference (case-folding dialect)", async () => {
|
|
895
|
+
// A carried/seeded upstream (built in a prior run or unit, e.g. via
|
|
896
|
+
// referenceManifest) must reach the downstream's SQL generation as the
|
|
897
|
+
// SAME quoted path the builder CREATEd it with; otherwise the downstream
|
|
898
|
+
// CREATE misses the case-preserved table on a case-folding engine
|
|
899
|
+
// (Snowflake). Capture the buildManifest Malloy is handed for the
|
|
900
|
+
// downstream to prove the seed was quoted for its connection's dialect
|
|
901
|
+
// before it becomes a FROM.
|
|
902
|
+
let seen:
|
|
903
|
+
| {
|
|
904
|
+
buildManifest?: {
|
|
905
|
+
entries?: Record<string, { tableName?: string }>;
|
|
906
|
+
};
|
|
907
|
+
}
|
|
908
|
+
| undefined;
|
|
909
|
+
const runSQL = sinon.stub().resolves();
|
|
910
|
+
const connection = {
|
|
911
|
+
runSQL,
|
|
912
|
+
dialectName: "snowflake",
|
|
913
|
+
} as unknown as MalloyConnection;
|
|
914
|
+
const down = fakeSource({
|
|
915
|
+
name: "down",
|
|
916
|
+
sourceEntityId: "bdownaaaaaaaaaa",
|
|
917
|
+
dialectName: "snowflake",
|
|
918
|
+
onGetSQL: (o) => {
|
|
919
|
+
seen = o as typeof seen;
|
|
920
|
+
},
|
|
921
|
+
});
|
|
922
|
+
const compiled = compiledWith(
|
|
923
|
+
{ down },
|
|
924
|
+
[["down"]],
|
|
925
|
+
new Map([["duckdb", connection]]),
|
|
926
|
+
);
|
|
927
|
+
|
|
928
|
+
await callExecute(
|
|
929
|
+
compiled,
|
|
930
|
+
[
|
|
931
|
+
{
|
|
932
|
+
sourceEntityId: "bdownaaaaaaaaaa",
|
|
933
|
+
materializedTableId: "mt-d",
|
|
934
|
+
physicalTableName: "down_v1",
|
|
935
|
+
realization: "COPY",
|
|
936
|
+
},
|
|
937
|
+
],
|
|
938
|
+
{
|
|
939
|
+
up: {
|
|
940
|
+
sourceEntityId: "up",
|
|
941
|
+
physicalTableName: "schema.orders_base",
|
|
942
|
+
connectionName: "duckdb",
|
|
943
|
+
},
|
|
944
|
+
},
|
|
945
|
+
);
|
|
946
|
+
|
|
947
|
+
expect(seen?.buildManifest?.entries?.up?.tableName).toBe(
|
|
948
|
+
'"schema"."orders_base"',
|
|
949
|
+
);
|
|
950
|
+
});
|
|
951
|
+
|
|
952
|
+
it("seeds unquoted when the upstream's connection is absent from the build (older CP / cross-connection)", async () => {
|
|
953
|
+
// No connectionName on the seed (an older control plane) -> bind verbatim,
|
|
954
|
+
// exactly the pre-change behavior. Non-case-folding engines are unaffected;
|
|
955
|
+
// a case-folding engine self-signals by failing the downstream CREATE.
|
|
956
|
+
let seen:
|
|
957
|
+
| {
|
|
958
|
+
buildManifest?: {
|
|
959
|
+
entries?: Record<string, { tableName?: string }>;
|
|
960
|
+
};
|
|
961
|
+
}
|
|
962
|
+
| undefined;
|
|
963
|
+
const runSQL = sinon.stub().resolves();
|
|
964
|
+
const connection = {
|
|
965
|
+
runSQL,
|
|
966
|
+
dialectName: "snowflake",
|
|
967
|
+
} as unknown as MalloyConnection;
|
|
968
|
+
const down = fakeSource({
|
|
969
|
+
name: "down",
|
|
970
|
+
sourceEntityId: "bdownaaaaaaaaaa",
|
|
971
|
+
dialectName: "snowflake",
|
|
972
|
+
onGetSQL: (o) => {
|
|
973
|
+
seen = o as typeof seen;
|
|
974
|
+
},
|
|
975
|
+
});
|
|
976
|
+
const compiled = compiledWith(
|
|
977
|
+
{ down },
|
|
978
|
+
[["down"]],
|
|
979
|
+
new Map([["duckdb", connection]]),
|
|
980
|
+
);
|
|
981
|
+
|
|
982
|
+
await callExecute(
|
|
983
|
+
compiled,
|
|
984
|
+
[
|
|
985
|
+
{
|
|
986
|
+
sourceEntityId: "bdownaaaaaaaaaa",
|
|
987
|
+
materializedTableId: "mt-d",
|
|
988
|
+
physicalTableName: "down_v1",
|
|
989
|
+
realization: "COPY",
|
|
990
|
+
},
|
|
991
|
+
],
|
|
992
|
+
{ up: { sourceEntityId: "up", physicalTableName: "orders_base" } },
|
|
993
|
+
);
|
|
994
|
+
|
|
995
|
+
expect(seen?.buildManifest?.entries?.up?.tableName).toBe("orders_base");
|
|
996
|
+
});
|
|
997
|
+
|
|
791
998
|
it("throws when an instructed graph's connection is missing", async () => {
|
|
792
999
|
const s1 = fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" });
|
|
793
1000
|
const compiled = compiledWith({ s1 }, [["s1"]], new Map());
|
|
@@ -819,6 +1026,7 @@ describe("buildOneSource", () => {
|
|
|
819
1026
|
connection: { runSQL: sinon.SinonStub },
|
|
820
1027
|
physicalTableName: string,
|
|
821
1028
|
dialectName?: string,
|
|
1029
|
+
manifest: Manifest = new Manifest(),
|
|
822
1030
|
): Promise<{ sourceEntityId: string; physicalTableName: string }> {
|
|
823
1031
|
const source = fakeSource({
|
|
824
1032
|
name: "orders",
|
|
@@ -847,7 +1055,7 @@ describe("buildOneSource", () => {
|
|
|
847
1055
|
instruction,
|
|
848
1056
|
connection,
|
|
849
1057
|
{ duckdb: "dig" },
|
|
850
|
-
|
|
1058
|
+
manifest,
|
|
851
1059
|
);
|
|
852
1060
|
}
|
|
853
1061
|
|
|
@@ -879,6 +1087,39 @@ describe("buildOneSource", () => {
|
|
|
879
1087
|
);
|
|
880
1088
|
});
|
|
881
1089
|
|
|
1090
|
+
it("rethrows the original build error when staging cleanup also fails", async () => {
|
|
1091
|
+
const runSQL = sinon.stub();
|
|
1092
|
+
runSQL.onCall(0).resolves(); // initial staging drop
|
|
1093
|
+
runSQL.onCall(1).rejects(new Error("create boom")); // CREATE TABLE AS
|
|
1094
|
+
runSQL.onCall(2).rejects(new Error("cleanup boom")); // cleanup drop fails
|
|
1095
|
+
// The cleanup failure is swallowed (logged as a physical leak); the
|
|
1096
|
+
// original build error is what propagates, so the run is classified by its
|
|
1097
|
+
// real cause rather than the cleanup noise.
|
|
1098
|
+
await expect(callBuildOneSource({ runSQL }, "orders_v1")).rejects.toThrow(
|
|
1099
|
+
"create boom",
|
|
1100
|
+
);
|
|
1101
|
+
expect(runSQL.callCount).toBe(3);
|
|
1102
|
+
expect(runSQL.lastCall.args[0]).toBe(
|
|
1103
|
+
'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
|
|
1104
|
+
);
|
|
1105
|
+
});
|
|
1106
|
+
|
|
1107
|
+
it("drops staging when the build is cancelled mid-run (no _staging orphan)", async () => {
|
|
1108
|
+
// buildOneSource is cancellation-agnostic: a cancel surfaces as a thrown
|
|
1109
|
+
// build SQL and takes the same staging-cleanup path as a failure, so a
|
|
1110
|
+
// cancelled build leaves no orphaned _staging table behind.
|
|
1111
|
+
const runSQL = sinon.stub();
|
|
1112
|
+
runSQL.onCall(0).resolves(); // initial staging drop
|
|
1113
|
+
runSQL.onCall(1).rejects(new Error("Build cancelled")); // aborted mid-CREATE
|
|
1114
|
+
runSQL.onCall(2).resolves(); // cleanup staging drop
|
|
1115
|
+
await expect(callBuildOneSource({ runSQL }, "orders_v1")).rejects.toThrow(
|
|
1116
|
+
"Build cancelled",
|
|
1117
|
+
);
|
|
1118
|
+
expect(runSQL.lastCall.args[0]).toBe(
|
|
1119
|
+
'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
|
|
1120
|
+
);
|
|
1121
|
+
});
|
|
1122
|
+
|
|
882
1123
|
it("quotes each segment of a container path for a backtick dialect", async () => {
|
|
883
1124
|
const runSQL = sinon.stub().resolves();
|
|
884
1125
|
// Container path (dataset.table) on a backtick dialect (BigQuery): each
|
|
@@ -898,6 +1139,33 @@ describe("buildOneSource", () => {
|
|
|
898
1139
|
]);
|
|
899
1140
|
expect(entry.physicalTableName).toBe("ds.orders_v1");
|
|
900
1141
|
});
|
|
1142
|
+
|
|
1143
|
+
it("records the QUOTED just-built name in the build manifest (matches the CREATE)", async () => {
|
|
1144
|
+
// The build manifest feeds a downstream persist's FROM verbatim, so the
|
|
1145
|
+
// recorded name must be the SAME quoted path the CREATE used — else a
|
|
1146
|
+
// downstream built later in this run misses the case-preserved table on a
|
|
1147
|
+
// case-folding engine. buildManifest also passes through Malloy's canonical
|
|
1148
|
+
// path validation (requireCanonicalTablePathAnyDialect), so this proves the
|
|
1149
|
+
// quoted form is accepted, not just produced.
|
|
1150
|
+
const runSQL = sinon.stub().resolves();
|
|
1151
|
+
const manifest = new Manifest();
|
|
1152
|
+
await callBuildOneSource(
|
|
1153
|
+
{ runSQL },
|
|
1154
|
+
"schema.orders_base",
|
|
1155
|
+
"snowflake",
|
|
1156
|
+
manifest,
|
|
1157
|
+
);
|
|
1158
|
+
expect(manifest.buildManifest.entries["abcdef1234567890"].tableName).toBe(
|
|
1159
|
+
'"schema"."orders_base"',
|
|
1160
|
+
);
|
|
1161
|
+
// The logical (unquoted) name is what the CREATE was told to produce, and
|
|
1162
|
+
// is exactly what quoteTablePath quoted for the DDL — read mirrors write.
|
|
1163
|
+
const created = runSQL
|
|
1164
|
+
.getCalls()
|
|
1165
|
+
.map((c) => c.args[0] as string)
|
|
1166
|
+
.find((s) => s.startsWith("CREATE TABLE"))!;
|
|
1167
|
+
expect(created).toContain('"schema"."orders_base_abcdef123456"');
|
|
1168
|
+
});
|
|
901
1169
|
});
|
|
902
1170
|
|
|
903
1171
|
describe("runBuild (branch behavior)", () => {
|
|
@@ -922,7 +1190,11 @@ describe("runBuild (branch behavior)", () => {
|
|
|
922
1190
|
forceRefresh: boolean;
|
|
923
1191
|
buildInstructions: BuildInstruction[] | undefined;
|
|
924
1192
|
referenceManifest?:
|
|
925
|
-
| {
|
|
1193
|
+
| {
|
|
1194
|
+
sourceEntityId: string;
|
|
1195
|
+
physicalTableName: string;
|
|
1196
|
+
connectionName?: string;
|
|
1197
|
+
}[]
|
|
926
1198
|
| undefined;
|
|
927
1199
|
strictUpstreams?: boolean | undefined;
|
|
928
1200
|
},
|
|
@@ -989,7 +1261,11 @@ describe("runBuild (branch behavior)", () => {
|
|
|
989
1261
|
forceRefresh: false,
|
|
990
1262
|
buildInstructions: instructions,
|
|
991
1263
|
referenceManifest: [
|
|
992
|
-
{
|
|
1264
|
+
{
|
|
1265
|
+
sourceEntityId: "up-1",
|
|
1266
|
+
physicalTableName: "upstream_tbl",
|
|
1267
|
+
connectionName: "sf",
|
|
1268
|
+
},
|
|
993
1269
|
],
|
|
994
1270
|
strictUpstreams: true,
|
|
995
1271
|
},
|
|
@@ -998,10 +1274,13 @@ describe("runBuild (branch behavior)", () => {
|
|
|
998
1274
|
|
|
999
1275
|
// The reference manifest becomes the seed (carried) entries so a
|
|
1000
1276
|
// downstream build resolves its upstream to the existing physical table.
|
|
1277
|
+
// connectionName is carried through so the seed can be quoted for the
|
|
1278
|
+
// upstream's dialect (see quoteSeedTablePath).
|
|
1001
1279
|
expect(svc.executeInstructedBuild.firstCall.args[2]).toEqual({
|
|
1002
1280
|
"up-1": {
|
|
1003
1281
|
sourceEntityId: "up-1",
|
|
1004
1282
|
physicalTableName: "upstream_tbl",
|
|
1283
|
+
connectionName: "sf",
|
|
1005
1284
|
},
|
|
1006
1285
|
});
|
|
1007
1286
|
// strictUpstreams flows through as the strict flag (5th arg).
|
|
@@ -1065,9 +1344,11 @@ describe("autoLoadManifest", () => {
|
|
|
1065
1344
|
).autoLoadManifest({ reloadAllModelsForPackage }, "pkg", entries);
|
|
1066
1345
|
|
|
1067
1346
|
expect(reloadAllModelsForPackage.calledOnce).toBe(true);
|
|
1347
|
+
// connectionName is carried alongside tableName so the bind step
|
|
1348
|
+
// (Package.quoteBoundTableNames) can quote the path for its dialect.
|
|
1068
1349
|
expect(reloadAllModelsForPackage.firstCall.args).toEqual([
|
|
1069
1350
|
"pkg",
|
|
1070
|
-
{ b1: { tableName: "orders_v1" } },
|
|
1351
|
+
{ b1: { tableName: "orders_v1", connectionName: "duckdb" } },
|
|
1071
1352
|
]);
|
|
1072
1353
|
});
|
|
1073
1354
|
|