@malloy-publisher/server 0.0.228 → 0.0.230

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/dist/app/api-doc.yaml +102 -10
  2. package/dist/app/assets/EnvironmentPage-wa_EPkwK.js +1 -0
  3. package/dist/app/assets/{HomePage-Bkwc9Woc.js → HomePage-jnCrupQp.js} +1 -1
  4. package/dist/app/assets/{LightMode-Bum_KBpN.js → LightMode-DYbwNULZ.js} +1 -1
  5. package/dist/app/assets/MainPage-CuJLrPNI.js +2 -0
  6. package/dist/app/assets/MaterializationsPage-D_67x2ee.js +1 -0
  7. package/dist/app/assets/ModelPage-D5JtAWqR.js +1 -0
  8. package/dist/app/assets/PackagePage-BRwtqUSG.js +1 -0
  9. package/dist/app/assets/{RouteError-DmJLpLXm.js → RouteError-CBNNrnSD.js} +1 -1
  10. package/dist/app/assets/ThemeEditorPage-CTCeBneA.js +1 -0
  11. package/dist/app/assets/WorkbookPage-SN6f1RBm.js +1 -0
  12. package/dist/app/assets/{core-CEDZMHV1.es-_yGzNgNH.js → core-Dp3q5Ieu.es-CD5FvM2s.js} +10 -10
  13. package/dist/app/assets/{index-CdmFub34.js → index-B3Nn8Vm2.js} +459 -446
  14. package/dist/app/assets/index-BLCx1EdC.js +18 -0
  15. package/dist/app/assets/index-C_tJstcx.js +529 -0
  16. package/dist/app/assets/{index-CE9xhdra.js → index-CfmBVB6M.js} +1 -1
  17. package/dist/app/assets/{index-OEjKNSYb.js → index-DU4r7GdU.js} +573 -413
  18. package/dist/app/index.html +1 -1
  19. package/dist/instrumentation.mjs +1 -1
  20. package/dist/package_load_worker.mjs +109 -8
  21. package/dist/server.mjs +7504 -80
  22. package/package.json +15 -13
  23. package/scripts/bake-duckdb-extensions.js +5 -2
  24. package/src/config.spec.ts +87 -1
  25. package/src/config.ts +142 -1
  26. package/src/controller/materialization.controller.spec.ts +9 -0
  27. package/src/controller/materialization.controller.ts +15 -0
  28. package/src/ducklake_version.spec.ts +163 -0
  29. package/src/ducklake_version.ts +153 -0
  30. package/src/errors.ts +12 -0
  31. package/src/malloy_pin_prereqs.spec.ts +25 -0
  32. package/src/materialization_metrics.ts +33 -0
  33. package/src/mcp/skills/build_skills_bundle.ts +29 -9
  34. package/src/mcp/skills/skills_bundle.json +1 -1
  35. package/src/mcp/skills/skills_bundle.spec.ts +31 -2
  36. package/src/query_param_utils.ts +11 -0
  37. package/src/server.ts +74 -9
  38. package/src/service/authorize.spec.ts +22 -0
  39. package/src/service/authorize.ts +267 -10
  40. package/src/service/authorize_integration.spec.ts +1068 -26
  41. package/src/service/connection.spec.ts +79 -7
  42. package/src/service/connection.ts +312 -25
  43. package/src/service/connection_config.spec.ts +21 -0
  44. package/src/service/connection_config.ts +15 -1
  45. package/src/service/cron_evaluator.spec.ts +79 -0
  46. package/src/service/cron_evaluator.ts +105 -0
  47. package/src/service/ducklake_lazy_attach.spec.ts +110 -0
  48. package/src/service/environment.ts +156 -17
  49. package/src/service/environment_store.spec.ts +569 -14
  50. package/src/service/environment_store.ts +237 -31
  51. package/src/service/environment_store_anchoring.spec.ts +107 -0
  52. package/src/service/extension_fetch_policy.spec.ts +256 -0
  53. package/src/service/manifest_loader.spec.ts +3 -1
  54. package/src/service/manifest_loader.ts +9 -5
  55. package/src/service/materialization_scheduler.spec.ts +289 -0
  56. package/src/service/materialization_scheduler.ts +304 -0
  57. package/src/service/materialization_service.spec.ts +286 -5
  58. package/src/service/materialization_service.ts +121 -8
  59. package/src/service/materialization_test_fixtures.ts +10 -1
  60. package/src/service/model.spec.ts +67 -0
  61. package/src/service/model.ts +656 -31
  62. package/src/service/package.spec.ts +38 -0
  63. package/src/service/package.ts +102 -2
  64. package/src/service/package_quote_bound_tables.spec.ts +171 -0
  65. package/src/service/persistence_policy.spec.ts +31 -0
  66. package/src/service/quoting.spec.ts +57 -1
  67. package/src/service/quoting.ts +33 -0
  68. package/src/storage/DatabaseInterface.ts +15 -0
  69. package/src/storage/duckdb/DuckDBRepository.ts +17 -0
  70. package/src/storage/duckdb/MaterializationRepository.spec.ts +191 -0
  71. package/src/storage/duckdb/MaterializationRepository.ts +68 -0
  72. package/tests/fixtures/persist-schedule-test/data/orders.csv +5 -0
  73. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +11 -0
  74. package/tests/fixtures/persist-schedule-test/publisher.json +7 -0
  75. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +5 -0
  76. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +11 -0
  77. package/tests/fixtures/persist-schedule-warn-test/publisher.json +6 -0
  78. package/tests/integration/materialization/environment_list.integration.spec.ts +133 -0
  79. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +7 -2
  80. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +150 -0
  81. package/tests/integration/materialization/schedule_edit.integration.spec.ts +126 -0
  82. package/tests/integration/materialization/scheduler.integration.spec.ts +200 -0
  83. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +172 -0
  84. package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +256 -0
  85. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +143 -0
  86. package/dist/app/assets/EnvironmentPage-EW2lbGvb.js +0 -1
  87. package/dist/app/assets/MainPage-oiEy7TNM.js +0 -2
  88. package/dist/app/assets/MaterializationsPage-C_VJsTgU.js +0 -1
  89. package/dist/app/assets/ModelPage-z8REqAmk.js +0 -1
  90. package/dist/app/assets/PackagePage-C2Vtt1Ln.js +0 -1
  91. package/dist/app/assets/ThemeEditorPage-BywFjC7A.js +0 -1
  92. package/dist/app/assets/WorkbookPage-DCMizDMR.js +0 -1
  93. package/dist/app/assets/index-DDMrjIT3.js +0 -18
  94. package/dist/app/assets/index-EqslXZ44.js +0 -527
@@ -0,0 +1,304 @@
1
+ import type { MaterializationSchedulerConfig } from "../config";
2
+ import { MaterializationConflictError } from "../errors";
3
+ import { logger } from "../logger";
4
+ import { recordScheduledFire } from "../materialization_metrics";
5
+ import { CronEvaluator } from "./cron_evaluator";
6
+ import type { EnvironmentStore } from "./environment_store";
7
+ import type { MaterializationService } from "./materialization_service";
8
+
9
+ /** Per-package arming state, keyed by `${environmentName}::${packageName}`. */
10
+ interface ScheduleState {
11
+ /** The cron the state was armed from; a change re-arms `nextFireAtMs`. */
12
+ schedule: string;
13
+ /** Epoch ms of the next due fire (`cron.nextAfter`, strictly future when armed). */
14
+ nextFireAtMs: number;
15
+ /** Epoch ms of the last fire attempt, or null if never fired. */
16
+ lastFiredAtMs: number | null;
17
+ }
18
+
19
+ /**
20
+ * Standalone materialization scheduler: fires a loaded package's
21
+ * `materialization.schedule` cron so an open-source publisher (no control plane)
22
+ * can rebuild on a cadence. Modeled on {@link PackageMemoryGovernor} — a
23
+ * config-driven `setInterval` with `start`/`stop` and one `tick`.
24
+ *
25
+ * **Orchestration safety (two independent guards):**
26
+ * 1. Disabled by default — only constructed when
27
+ * {@link getMaterializationSchedulerConfig} returns non-null
28
+ * (`PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER`), which an orchestrated
29
+ * deployment never sets, so the scheduler never runs there.
30
+ * 2. Even when enabled, a package with a configured `manifestLocation` (i.e.
31
+ * control-plane-driven) is skipped — the CP already fires it, so we never
32
+ * double-drive.
33
+ *
34
+ * These guards are **independent and both load-bearing — never collapse them to
35
+ * just the `manifestLocation` check.** A control-plane-loaded package that is
36
+ * *serving live* has `manifestLocation === null`, so Guard 2 does not cover it;
37
+ * only Guard 1 (the opt-in flag, which an orchestrated worker must never set —
38
+ * see {@link getMaterializationSchedulerConfig}) keeps the scheduler off there.
39
+ *
40
+ * It only sweeps **already-loaded** packages (never forces a load), fires the
41
+ * existing `createMaterialization` auto-run path tagged `trigger=SCHEDULER`, and
42
+ * only fires packages whose persistence policy is valid (`scope: version`, no
43
+ * `freshness` — the same rule a publish enforces). Arming state is in memory,
44
+ * but on the first arm after a (re)start it re-anchors from the newest recorded
45
+ * `SCHEDULER` fire (see {@link arm}), so an occurrence that came due during
46
+ * downtime fires exactly once and then jumps forward — recovery matches the
47
+ * control plane's persisted-`next_fire_at` behavior rather than silently
48
+ * skipping. A never-fired schedule still does not fire on the arming tick
49
+ * (`nextAfter(now)` is strictly future).
50
+ */
51
+ export class MaterializationScheduler {
52
+ private timer: ReturnType<typeof setInterval> | null = null;
53
+ private readonly cron: CronEvaluator;
54
+ private readonly state = new Map<string, ScheduleState>();
55
+
56
+ constructor(
57
+ private readonly environmentStore: EnvironmentStore,
58
+ private readonly materializationService: MaterializationService,
59
+ private readonly config: MaterializationSchedulerConfig,
60
+ cron?: CronEvaluator,
61
+ ) {
62
+ this.cron = cron ?? new CronEvaluator();
63
+ }
64
+
65
+ /** Begin the periodic sweep. Idempotent; the interval is `unref`'d. */
66
+ public start(): void {
67
+ if (this.timer !== null) return;
68
+ this.timer = setInterval(() => {
69
+ void this.tick();
70
+ }, this.config.tickIntervalMs);
71
+ (
72
+ this.timer as ReturnType<typeof setInterval> & { unref?: () => void }
73
+ ).unref?.();
74
+ logger.info(
75
+ `MaterializationScheduler started (interval=${this.config.tickIntervalMs}ms, maxFiresPerTick=${this.config.maxFiresPerTick})`,
76
+ );
77
+ }
78
+
79
+ /** Stop the periodic sweep. Idempotent. */
80
+ public stop(): void {
81
+ if (this.timer !== null) {
82
+ clearInterval(this.timer);
83
+ this.timer = null;
84
+ }
85
+ }
86
+
87
+ /**
88
+ * One sweep: arm newly-seen schedules, fire due ones (capped per tick), and
89
+ * prune state for packages no longer loaded/scheduled. Exposed so tests can
90
+ * drive it synchronously. Never throws — per-package errors are isolated.
91
+ */
92
+ public async tick(nowMs: number = Date.now()): Promise<void> {
93
+ const seen = new Set<string>();
94
+ let fired = 0;
95
+
96
+ for (const env of this.environmentStore.getLoadedEnvironments()) {
97
+ const environmentName = env.getEnvironmentName();
98
+ for (const pkg of env.getLoadedPackages()) {
99
+ const packageName = pkg.getPackageName();
100
+ const key = `${environmentName}::${packageName}`;
101
+ try {
102
+ const schedule = this.schedulableCron(pkg);
103
+ if (schedule === null) {
104
+ // Not a standalone-schedulable package (no cron, orchestrated,
105
+ // or invalid policy/cron) — drop any stale arming state.
106
+ this.state.delete(key);
107
+ continue;
108
+ }
109
+ seen.add(key);
110
+
111
+ const armed = await this.arm(
112
+ environmentName,
113
+ packageName,
114
+ key,
115
+ schedule,
116
+ nowMs,
117
+ );
118
+ if (nowMs < armed.nextFireAtMs) continue; // not due yet
119
+
120
+ if (fired >= this.config.maxFiresPerTick) {
121
+ // Stampede guard: leave it due; it fires on a later tick.
122
+ continue;
123
+ }
124
+ await this.fire(environmentName, packageName);
125
+ fired++;
126
+ // Advance regardless of outcome so a persistent failure retries on
127
+ // the next cron occurrence (cron cadence), never every tick.
128
+ armed.nextFireAtMs = this.cron
129
+ .nextAfter(schedule, new Date(nowMs))
130
+ .getTime();
131
+ armed.lastFiredAtMs = nowMs;
132
+ } catch (err) {
133
+ logger.warn("MaterializationScheduler: package sweep failed", {
134
+ environmentName,
135
+ packageName,
136
+ error: err instanceof Error ? err.message : String(err),
137
+ });
138
+ }
139
+ }
140
+ }
141
+
142
+ for (const key of [...this.state.keys()]) {
143
+ if (!seen.has(key)) this.state.delete(key);
144
+ }
145
+ }
146
+
147
+ /**
148
+ * The package's schedulable cron, or null when it must be skipped: no
149
+ * `materialization.schedule`; a `manifestLocation` (control-plane-driven —
150
+ * Guard 2); an invalid persistence policy (e.g. schedule without
151
+ * `scope: version`, or schedule + freshness — the publish-gate rules); or an
152
+ * unparseable cron. Skips are debug-logged (the policy issues are already
153
+ * warned at package load), not warned each tick.
154
+ */
155
+ private schedulableCron(pkg: {
156
+ getPackageMetadata(): {
157
+ materialization?: { schedule?: string | null } | null;
158
+ manifestLocation?: string | null;
159
+ };
160
+ persistencePolicyWarnings(): string[];
161
+ getPackageName(): string;
162
+ }): string | null {
163
+ const metadata = pkg.getPackageMetadata();
164
+ const schedule = metadata.materialization?.schedule ?? null;
165
+ if (!schedule) return null;
166
+ if (metadata.manifestLocation) return null; // orchestrated — CP fires it
167
+ if (pkg.persistencePolicyWarnings().length > 0) {
168
+ logger.debug(
169
+ "MaterializationScheduler: skipping package with invalid persistence policy",
170
+ { packageName: pkg.getPackageName() },
171
+ );
172
+ return null;
173
+ }
174
+ if (!this.cron.isValid(schedule)) {
175
+ logger.debug("MaterializationScheduler: skipping invalid cron", {
176
+ packageName: pkg.getPackageName(),
177
+ schedule,
178
+ });
179
+ return null;
180
+ }
181
+ return schedule;
182
+ }
183
+
184
+ /**
185
+ * Return the package's arming state, (re)arming when it is new or its cron
186
+ * changed.
187
+ *
188
+ * **First arm of a package this process** (no in-memory state — a fresh start
189
+ * or a newly-loaded package): anchor `nextFireAt` to the newest recorded
190
+ * `SCHEDULER` fire, `nextAfter(lastScheduledFire)`. If that instant is already
191
+ * past (an occurrence came due while the publisher was down), it fires once
192
+ * on this tick and then jumps forward to `nextAfter(now)` — exactly one
193
+ * catch-up, matching the control plane's persisted-`next_fire_at` recovery
194
+ * contract, never a burst. With no prior `SCHEDULER` fire on record we fall
195
+ * back to `nextAfter(now)` (strictly future), so a never-fired schedule does
196
+ * not fire on the arming tick. (Residual divergence: a schedule set while the
197
+ * scheduler was disabled has no prior fire, so it is not caught up on first
198
+ * enable.)
199
+ *
200
+ * **Cron changed**: re-arm from `nextAfter(now)` — an edited cadence starts
201
+ * fresh, with no catch-up for the old cron.
202
+ */
203
+ private async arm(
204
+ environmentName: string,
205
+ packageName: string,
206
+ key: string,
207
+ schedule: string,
208
+ nowMs: number,
209
+ ): Promise<ScheduleState> {
210
+ const existing = this.state.get(key);
211
+ if (existing && existing.schedule === schedule) {
212
+ return existing;
213
+ }
214
+
215
+ const anchor =
216
+ existing === undefined
217
+ ? await this.recoveryAnchor(environmentName, packageName, nowMs)
218
+ : new Date(nowMs);
219
+
220
+ const armed: ScheduleState = {
221
+ schedule,
222
+ nextFireAtMs: this.cron.nextAfter(schedule, anchor).getTime(),
223
+ lastFiredAtMs: existing?.lastFiredAtMs ?? null,
224
+ };
225
+ this.state.set(key, armed);
226
+ return armed;
227
+ }
228
+
229
+ /**
230
+ * The instant to compute the first `nextFireAt` from after a (re)start: the
231
+ * newest recorded `SCHEDULER` fire if any, else `now`. Any lookup failure
232
+ * degrades to `now` (no catch-up) so arming always succeeds.
233
+ */
234
+ private async recoveryAnchor(
235
+ environmentName: string,
236
+ packageName: string,
237
+ nowMs: number,
238
+ ): Promise<Date> {
239
+ try {
240
+ const last =
241
+ await this.materializationService.getLatestScheduledFireAt(
242
+ environmentName,
243
+ packageName,
244
+ );
245
+ return last ?? new Date(nowMs);
246
+ } catch (err) {
247
+ logger.warn(
248
+ "MaterializationScheduler: could not read last scheduled fire; arming from now",
249
+ {
250
+ environmentName,
251
+ packageName,
252
+ error: err instanceof Error ? err.message : String(err),
253
+ },
254
+ );
255
+ return new Date(nowMs);
256
+ }
257
+ }
258
+
259
+ /**
260
+ * Fire a whole-package scheduled rebuild via the existing auto-run path. An
261
+ * already-active materialization coalesces (the in-flight build covers this
262
+ * occurrence); any other error is logged and metered but never propagated.
263
+ *
264
+ * **Standalone fires are tables-only.** This path runs a single-phase build
265
+ * that materializes the package's persist sources into tables. A hosted
266
+ * (control-plane-driven) deployment runs a two-phase job — tables, then a
267
+ * second pass over indexed dimensions — so index behavior is a hosted-only
268
+ * concern that cannot be exercised by the standalone scheduler. Anything a
269
+ * schedule fires locally is exactly what the `createMaterialization`
270
+ * auto-run produces on demand; only the `trigger` label differs.
271
+ */
272
+ private async fire(
273
+ environmentName: string,
274
+ packageName: string,
275
+ ): Promise<void> {
276
+ try {
277
+ await this.materializationService.createMaterialization(
278
+ environmentName,
279
+ packageName,
280
+ { forceRefresh: true, trigger: "SCHEDULER" },
281
+ );
282
+ recordScheduledFire("fired");
283
+ logger.info("MaterializationScheduler: fired scheduled rebuild", {
284
+ environmentName,
285
+ packageName,
286
+ });
287
+ } catch (err) {
288
+ if (err instanceof MaterializationConflictError) {
289
+ recordScheduledFire("conflict");
290
+ logger.debug(
291
+ "MaterializationScheduler: skipped fire; a materialization is already active",
292
+ { environmentName, packageName },
293
+ );
294
+ return;
295
+ }
296
+ recordScheduledFire("error");
297
+ logger.warn("MaterializationScheduler: scheduled fire failed", {
298
+ environmentName,
299
+ packageName,
300
+ error: err instanceof Error ? err.message : String(err),
301
+ });
302
+ }
303
+ }
304
+ }
@@ -1,6 +1,6 @@
1
1
  import type { Connection as MalloyConnection } from "@malloydata/malloy";
2
2
  import { Manifest } from "@malloydata/malloy";
3
- import { beforeEach, describe, expect, it } from "bun:test";
3
+ import { afterEach, beforeEach, describe, expect, it } from "bun:test";
4
4
  import * as sinon from "sinon";
5
5
  import {
6
6
  BadRequestError,
@@ -27,6 +27,11 @@ import {
27
27
  MaterializationService,
28
28
  stagingSuffix,
29
29
  } from "./materialization_service";
30
+ import { resetMaterializationTelemetryForTesting } from "../materialization_metrics";
31
+ import {
32
+ startMetricsHarness,
33
+ type MetricsHarness,
34
+ } from "../test_helpers/metrics_harness";
30
35
 
31
36
  type MockRepo = sinon.SinonStubbedInstance<ResourceRepository>;
32
37
 
@@ -166,6 +171,7 @@ describe("MaterializationService", () => {
166
171
  forceRefresh: true,
167
172
  sourceNames: ["orders"],
168
173
  mode: "auto",
174
+ trigger: "ON_DEMAND",
169
175
  },
170
176
  ]);
171
177
  });
@@ -183,10 +189,27 @@ describe("MaterializationService", () => {
183
189
  forceRefresh: false,
184
190
  sourceNames: null,
185
191
  mode: "auto",
192
+ trigger: "ON_DEMAND",
186
193
  },
187
194
  );
188
195
  });
189
196
 
197
+ it("records trigger=SCHEDULER when the scheduler fires the run", async () => {
198
+ ctx.repository.getActiveMaterialization.resolves(null);
199
+ ctx.repository.createMaterialization.resolves(
200
+ makeMaterialization({ status: "PENDING" }),
201
+ );
202
+
203
+ await ctx.service.createMaterialization("my-env", "pkg", {
204
+ forceRefresh: true,
205
+ trigger: "SCHEDULER",
206
+ });
207
+
208
+ expect(
209
+ ctx.repository.createMaterialization.firstCall.args[3],
210
+ ).toMatchObject({ mode: "auto", trigger: "SCHEDULER" });
211
+ });
212
+
190
213
  it("rejects when an active materialization already exists", async () => {
191
214
  ctx.repository.getActiveMaterialization.resolves(
192
215
  makeMaterialization({
@@ -482,6 +505,86 @@ describe("MaterializationService", () => {
482
505
  });
483
506
  });
484
507
 
508
+ describe("deleteMaterialization telemetry", () => {
509
+ let ctx: ReturnType<typeof createMocks>;
510
+ let harness: MetricsHarness;
511
+
512
+ beforeEach(async () => {
513
+ ctx = createMocks();
514
+ harness = await startMetricsHarness();
515
+ // Drop cached instruments so recordDropTables re-binds to this provider.
516
+ resetMaterializationTelemetryForTesting();
517
+ });
518
+
519
+ afterEach(async () => {
520
+ resetMaterializationTelemetryForTesting();
521
+ await harness.shutdown();
522
+ });
523
+
524
+ const DROP_COUNTER = "publisher_materialization_drop_tables_total";
525
+
526
+ // A terminal materialization whose manifest names one physical table, so
527
+ // dropMaterializedTables issues exactly one DROP — the emission under test.
528
+ function dropTablesFixture(runSQL: sinon.SinonStub) {
529
+ const getMalloyConnection = sinon
530
+ .stub()
531
+ .resolves({ runSQL, dialectName: "duckdb" });
532
+ (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
533
+ getPackage: sinon.stub().resolves({ getMalloyConnection }),
534
+ withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
535
+ fn(),
536
+ });
537
+ ctx.repository.getMaterializationById.resolves(
538
+ makeMaterialization({
539
+ status: "MANIFEST_FILE_READY",
540
+ manifest: {
541
+ builtAt: new Date().toISOString(),
542
+ strict: false,
543
+ entries: {
544
+ b1: {
545
+ sourceEntityId: "b1",
546
+ physicalTableName: "orders_mz",
547
+ connectionName: "duckdb",
548
+ },
549
+ },
550
+ },
551
+ }),
552
+ );
553
+ }
554
+
555
+ it("meters a successful physical-table drop as outcome=success", async () => {
556
+ dropTablesFixture(sinon.stub().resolves());
557
+
558
+ await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
559
+ dropTables: true,
560
+ });
561
+
562
+ expect(
563
+ await harness.collectCounter(DROP_COUNTER, { outcome: "success" }),
564
+ ).toBe(1);
565
+ expect(
566
+ await harness.collectCounter(DROP_COUNTER, { outcome: "failure" }),
567
+ ).toBe(0);
568
+ });
569
+
570
+ it("meters a failed drop as outcome=failure and still deletes the record", async () => {
571
+ dropTablesFixture(sinon.stub().rejects(new Error("drop boom")));
572
+
573
+ // The drop is best-effort: a failure is metered but never surfaces to the
574
+ // caller, and the record is still removed.
575
+ await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
576
+ dropTables: true,
577
+ });
578
+
579
+ expect(
580
+ await harness.collectCounter(DROP_COUNTER, { outcome: "failure" }),
581
+ ).toBe(1);
582
+ expect(ctx.repository.deleteMaterialization.calledOnceWith("mat-1")).toBe(
583
+ true,
584
+ );
585
+ });
586
+ });
587
+
485
588
  describe("stagingSuffix", () => {
486
589
  it("derives a short, stable suffix from the sourceEntityId", () => {
487
590
  expect(stagingSuffix("abcdef1234567890")).toBe("_abcdef123456");
@@ -788,6 +891,110 @@ describe("executeInstructedBuild", () => {
788
891
  expect(rootIdx).toBeGreaterThan(midIdx);
789
892
  });
790
893
 
894
+ it("seeds a downstream build with the QUOTED upstream reference (case-folding dialect)", async () => {
895
+ // A carried/seeded upstream (built in a prior run or unit, e.g. via
896
+ // referenceManifest) must reach the downstream's SQL generation as the
897
+ // SAME quoted path the builder CREATEd it with; otherwise the downstream
898
+ // CREATE misses the case-preserved table on a case-folding engine
899
+ // (Snowflake). Capture the buildManifest Malloy is handed for the
900
+ // downstream to prove the seed was quoted for its connection's dialect
901
+ // before it becomes a FROM.
902
+ let seen:
903
+ | {
904
+ buildManifest?: {
905
+ entries?: Record<string, { tableName?: string }>;
906
+ };
907
+ }
908
+ | undefined;
909
+ const runSQL = sinon.stub().resolves();
910
+ const connection = {
911
+ runSQL,
912
+ dialectName: "snowflake",
913
+ } as unknown as MalloyConnection;
914
+ const down = fakeSource({
915
+ name: "down",
916
+ sourceEntityId: "bdownaaaaaaaaaa",
917
+ dialectName: "snowflake",
918
+ onGetSQL: (o) => {
919
+ seen = o as typeof seen;
920
+ },
921
+ });
922
+ const compiled = compiledWith(
923
+ { down },
924
+ [["down"]],
925
+ new Map([["duckdb", connection]]),
926
+ );
927
+
928
+ await callExecute(
929
+ compiled,
930
+ [
931
+ {
932
+ sourceEntityId: "bdownaaaaaaaaaa",
933
+ materializedTableId: "mt-d",
934
+ physicalTableName: "down_v1",
935
+ realization: "COPY",
936
+ },
937
+ ],
938
+ {
939
+ up: {
940
+ sourceEntityId: "up",
941
+ physicalTableName: "schema.orders_base",
942
+ connectionName: "duckdb",
943
+ },
944
+ },
945
+ );
946
+
947
+ expect(seen?.buildManifest?.entries?.up?.tableName).toBe(
948
+ '"schema"."orders_base"',
949
+ );
950
+ });
951
+
952
+ it("seeds unquoted when the upstream's connection is absent from the build (older CP / cross-connection)", async () => {
953
+ // No connectionName on the seed (an older control plane) -> bind verbatim,
954
+ // exactly the pre-change behavior. Non-case-folding engines are unaffected;
955
+ // a case-folding engine self-signals by failing the downstream CREATE.
956
+ let seen:
957
+ | {
958
+ buildManifest?: {
959
+ entries?: Record<string, { tableName?: string }>;
960
+ };
961
+ }
962
+ | undefined;
963
+ const runSQL = sinon.stub().resolves();
964
+ const connection = {
965
+ runSQL,
966
+ dialectName: "snowflake",
967
+ } as unknown as MalloyConnection;
968
+ const down = fakeSource({
969
+ name: "down",
970
+ sourceEntityId: "bdownaaaaaaaaaa",
971
+ dialectName: "snowflake",
972
+ onGetSQL: (o) => {
973
+ seen = o as typeof seen;
974
+ },
975
+ });
976
+ const compiled = compiledWith(
977
+ { down },
978
+ [["down"]],
979
+ new Map([["duckdb", connection]]),
980
+ );
981
+
982
+ await callExecute(
983
+ compiled,
984
+ [
985
+ {
986
+ sourceEntityId: "bdownaaaaaaaaaa",
987
+ materializedTableId: "mt-d",
988
+ physicalTableName: "down_v1",
989
+ realization: "COPY",
990
+ },
991
+ ],
992
+ { up: { sourceEntityId: "up", physicalTableName: "orders_base" } },
993
+ );
994
+
995
+ expect(seen?.buildManifest?.entries?.up?.tableName).toBe("orders_base");
996
+ });
997
+
791
998
  it("throws when an instructed graph's connection is missing", async () => {
792
999
  const s1 = fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" });
793
1000
  const compiled = compiledWith({ s1 }, [["s1"]], new Map());
@@ -819,6 +1026,7 @@ describe("buildOneSource", () => {
819
1026
  connection: { runSQL: sinon.SinonStub },
820
1027
  physicalTableName: string,
821
1028
  dialectName?: string,
1029
+ manifest: Manifest = new Manifest(),
822
1030
  ): Promise<{ sourceEntityId: string; physicalTableName: string }> {
823
1031
  const source = fakeSource({
824
1032
  name: "orders",
@@ -847,7 +1055,7 @@ describe("buildOneSource", () => {
847
1055
  instruction,
848
1056
  connection,
849
1057
  { duckdb: "dig" },
850
- new Manifest(),
1058
+ manifest,
851
1059
  );
852
1060
  }
853
1061
 
@@ -879,6 +1087,39 @@ describe("buildOneSource", () => {
879
1087
  );
880
1088
  });
881
1089
 
1090
+ it("rethrows the original build error when staging cleanup also fails", async () => {
1091
+ const runSQL = sinon.stub();
1092
+ runSQL.onCall(0).resolves(); // initial staging drop
1093
+ runSQL.onCall(1).rejects(new Error("create boom")); // CREATE TABLE AS
1094
+ runSQL.onCall(2).rejects(new Error("cleanup boom")); // cleanup drop fails
1095
+ // The cleanup failure is swallowed (logged as a physical leak); the
1096
+ // original build error is what propagates, so the run is classified by its
1097
+ // real cause rather than the cleanup noise.
1098
+ await expect(callBuildOneSource({ runSQL }, "orders_v1")).rejects.toThrow(
1099
+ "create boom",
1100
+ );
1101
+ expect(runSQL.callCount).toBe(3);
1102
+ expect(runSQL.lastCall.args[0]).toBe(
1103
+ 'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
1104
+ );
1105
+ });
1106
+
1107
+ it("drops staging when the build is cancelled mid-run (no _staging orphan)", async () => {
1108
+ // buildOneSource is cancellation-agnostic: a cancel surfaces as a thrown
1109
+ // build SQL and takes the same staging-cleanup path as a failure, so a
1110
+ // cancelled build leaves no orphaned _staging table behind.
1111
+ const runSQL = sinon.stub();
1112
+ runSQL.onCall(0).resolves(); // initial staging drop
1113
+ runSQL.onCall(1).rejects(new Error("Build cancelled")); // aborted mid-CREATE
1114
+ runSQL.onCall(2).resolves(); // cleanup staging drop
1115
+ await expect(callBuildOneSource({ runSQL }, "orders_v1")).rejects.toThrow(
1116
+ "Build cancelled",
1117
+ );
1118
+ expect(runSQL.lastCall.args[0]).toBe(
1119
+ 'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
1120
+ );
1121
+ });
1122
+
882
1123
  it("quotes each segment of a container path for a backtick dialect", async () => {
883
1124
  const runSQL = sinon.stub().resolves();
884
1125
  // Container path (dataset.table) on a backtick dialect (BigQuery): each
@@ -898,6 +1139,33 @@ describe("buildOneSource", () => {
898
1139
  ]);
899
1140
  expect(entry.physicalTableName).toBe("ds.orders_v1");
900
1141
  });
1142
+
1143
+ it("records the QUOTED just-built name in the build manifest (matches the CREATE)", async () => {
1144
+ // The build manifest feeds a downstream persist's FROM verbatim, so the
1145
+ // recorded name must be the SAME quoted path the CREATE used — else a
1146
+ // downstream built later in this run misses the case-preserved table on a
1147
+ // case-folding engine. buildManifest also passes through Malloy's canonical
1148
+ // path validation (requireCanonicalTablePathAnyDialect), so this proves the
1149
+ // quoted form is accepted, not just produced.
1150
+ const runSQL = sinon.stub().resolves();
1151
+ const manifest = new Manifest();
1152
+ await callBuildOneSource(
1153
+ { runSQL },
1154
+ "schema.orders_base",
1155
+ "snowflake",
1156
+ manifest,
1157
+ );
1158
+ expect(manifest.buildManifest.entries["abcdef1234567890"].tableName).toBe(
1159
+ '"schema"."orders_base"',
1160
+ );
1161
+ // The logical (unquoted) name is what the CREATE was told to produce, and
1162
+ // is exactly what quoteTablePath quoted for the DDL — read mirrors write.
1163
+ const created = runSQL
1164
+ .getCalls()
1165
+ .map((c) => c.args[0] as string)
1166
+ .find((s) => s.startsWith("CREATE TABLE"))!;
1167
+ expect(created).toContain('"schema"."orders_base_abcdef123456"');
1168
+ });
901
1169
  });
902
1170
 
903
1171
  describe("runBuild (branch behavior)", () => {
@@ -922,7 +1190,11 @@ describe("runBuild (branch behavior)", () => {
922
1190
  forceRefresh: boolean;
923
1191
  buildInstructions: BuildInstruction[] | undefined;
924
1192
  referenceManifest?:
925
- | { sourceEntityId: string; physicalTableName: string }[]
1193
+ | {
1194
+ sourceEntityId: string;
1195
+ physicalTableName: string;
1196
+ connectionName?: string;
1197
+ }[]
926
1198
  | undefined;
927
1199
  strictUpstreams?: boolean | undefined;
928
1200
  },
@@ -989,7 +1261,11 @@ describe("runBuild (branch behavior)", () => {
989
1261
  forceRefresh: false,
990
1262
  buildInstructions: instructions,
991
1263
  referenceManifest: [
992
- { sourceEntityId: "up-1", physicalTableName: "upstream_tbl" },
1264
+ {
1265
+ sourceEntityId: "up-1",
1266
+ physicalTableName: "upstream_tbl",
1267
+ connectionName: "sf",
1268
+ },
993
1269
  ],
994
1270
  strictUpstreams: true,
995
1271
  },
@@ -998,10 +1274,13 @@ describe("runBuild (branch behavior)", () => {
998
1274
 
999
1275
  // The reference manifest becomes the seed (carried) entries so a
1000
1276
  // downstream build resolves its upstream to the existing physical table.
1277
+ // connectionName is carried through so the seed can be quoted for the
1278
+ // upstream's dialect (see quoteSeedTablePath).
1001
1279
  expect(svc.executeInstructedBuild.firstCall.args[2]).toEqual({
1002
1280
  "up-1": {
1003
1281
  sourceEntityId: "up-1",
1004
1282
  physicalTableName: "upstream_tbl",
1283
+ connectionName: "sf",
1005
1284
  },
1006
1285
  });
1007
1286
  // strictUpstreams flows through as the strict flag (5th arg).
@@ -1065,9 +1344,11 @@ describe("autoLoadManifest", () => {
1065
1344
  ).autoLoadManifest({ reloadAllModelsForPackage }, "pkg", entries);
1066
1345
 
1067
1346
  expect(reloadAllModelsForPackage.calledOnce).toBe(true);
1347
+ // connectionName is carried alongside tableName so the bind step
1348
+ // (Package.quoteBoundTableNames) can quote the path for its dialect.
1068
1349
  expect(reloadAllModelsForPackage.firstCall.args).toEqual([
1069
1350
  "pkg",
1070
- { b1: { tableName: "orders_v1" } },
1351
+ { b1: { tableName: "orders_v1", connectionName: "duckdb" } },
1071
1352
  ]);
1072
1353
  });
1073
1354