@malloy-publisher/server 0.0.235 → 0.0.236

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (301) hide show
  1. package/dist/app/api-doc.yaml +47 -17
  2. package/dist/package_load_worker.mjs +8 -1
  3. package/dist/server.mjs +471 -54
  4. package/package.json +5 -2
  5. package/.eslintrc.json +0 -44
  6. package/.prettierignore +0 -1
  7. package/.prettierrc +0 -4
  8. package/build.ts +0 -97
  9. package/eslint.config.mjs +0 -76
  10. package/k6-tests/README.md +0 -144
  11. package/k6-tests/bun.lock +0 -797
  12. package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
  13. package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
  14. package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
  15. package/k6-tests/load-test/load-test-crud.ts +0 -142
  16. package/k6-tests/load-test/load-test.ts +0 -784
  17. package/k6-tests/package.json +0 -27
  18. package/k6-tests/scripts/run-with-otel.sh +0 -50
  19. package/k6-tests/smoke-test/smoke-test.ts +0 -158
  20. package/k6-tests/tsconfig.json +0 -29
  21. package/k6-tests/utils/client_factory.ts +0 -128
  22. package/k6-tests/utils/common.ts +0 -489
  23. package/k6-tests/utils/logger.ts +0 -62
  24. package/k6-tests/utils/types.d.ts +0 -35
  25. package/malloy_mcp.dxt +0 -0
  26. package/publisher.config.example.bigquery.json +0 -33
  27. package/publisher.config.example.duckdb.json +0 -23
  28. package/publisher.config.json +0 -23
  29. package/scripts/bake-duckdb-extensions.js +0 -110
  30. package/src/config.spec.ts +0 -1535
  31. package/src/config.theme.spec.ts +0 -178
  32. package/src/config.ts +0 -1142
  33. package/src/constants.ts +0 -105
  34. package/src/controller/compile.controller.ts +0 -40
  35. package/src/controller/connection.controller.spec.ts +0 -849
  36. package/src/controller/connection.controller.ts +0 -901
  37. package/src/controller/database.controller.ts +0 -24
  38. package/src/controller/materialization.controller.spec.ts +0 -293
  39. package/src/controller/materialization.controller.ts +0 -319
  40. package/src/controller/model.controller.ts +0 -164
  41. package/src/controller/package.controller.spec.ts +0 -273
  42. package/src/controller/package.controller.ts +0 -306
  43. package/src/controller/query.controller.ts +0 -152
  44. package/src/controller/theme.controller.ts +0 -83
  45. package/src/controller/watch-mode.controller.ts +0 -243
  46. package/src/data_styles.ts +0 -71
  47. package/src/default-publisher.config.json +0 -23
  48. package/src/ducklake_version.spec.ts +0 -163
  49. package/src/ducklake_version.ts +0 -153
  50. package/src/errors.spec.ts +0 -119
  51. package/src/errors.ts +0 -237
  52. package/src/filter_deprecation.spec.ts +0 -64
  53. package/src/filter_deprecation.ts +0 -42
  54. package/src/health.spec.ts +0 -123
  55. package/src/health.ts +0 -233
  56. package/src/heap_check.spec.ts +0 -144
  57. package/src/heap_check.ts +0 -144
  58. package/src/instrumentation.ts +0 -214
  59. package/src/json_utils.spec.ts +0 -51
  60. package/src/json_utils.ts +0 -33
  61. package/src/logger.spec.ts +0 -210
  62. package/src/logger.ts +0 -251
  63. package/src/malloy_pin_prereqs.spec.ts +0 -25
  64. package/src/materialization_metrics.spec.ts +0 -169
  65. package/src/materialization_metrics.ts +0 -346
  66. package/src/mcp/error_messages.spec.ts +0 -35
  67. package/src/mcp/error_messages.ts +0 -183
  68. package/src/mcp/handler_utils.spec.ts +0 -108
  69. package/src/mcp/handler_utils.ts +0 -251
  70. package/src/mcp/mcp_constants.ts +0 -6
  71. package/src/mcp/query_envelope.spec.ts +0 -229
  72. package/src/mcp/query_envelope.ts +0 -240
  73. package/src/mcp/server.protocol.spec.ts +0 -250
  74. package/src/mcp/server.ts +0 -87
  75. package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
  76. package/src/mcp/skills/build_skills_bundle.ts +0 -176
  77. package/src/mcp/skills/skills_bundle.json +0 -1
  78. package/src/mcp/skills/skills_bundle.spec.ts +0 -171
  79. package/src/mcp/tool_response.spec.ts +0 -108
  80. package/src/mcp/tool_response.ts +0 -138
  81. package/src/mcp/tools/compile_tool.spec.ts +0 -315
  82. package/src/mcp/tools/compile_tool.ts +0 -208
  83. package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
  84. package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
  85. package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
  86. package/src/mcp/tools/docs_search_tool.ts +0 -128
  87. package/src/mcp/tools/embedding_index.spec.ts +0 -1236
  88. package/src/mcp/tools/embedding_index.ts +0 -808
  89. package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
  90. package/src/mcp/tools/execute_query_tool.ts +0 -316
  91. package/src/mcp/tools/get_context_eval.ts +0 -275
  92. package/src/mcp/tools/get_context_tool.spec.ts +0 -592
  93. package/src/mcp/tools/get_context_tool.ts +0 -585
  94. package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
  95. package/src/mcp/tools/reload_package_tool.ts +0 -132
  96. package/src/mcp_config.spec.ts +0 -919
  97. package/src/mcp_config.ts +0 -425
  98. package/src/oom_guards.integration.spec.ts +0 -269
  99. package/src/package_load/package_load_pool.spec.ts +0 -252
  100. package/src/package_load/package_load_pool.ts +0 -931
  101. package/src/package_load/package_load_worker.ts +0 -1034
  102. package/src/package_load/protocol.ts +0 -364
  103. package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
  104. package/src/package_load/rpc_wait_accountant.ts +0 -76
  105. package/src/package_load_metrics.spec.ts +0 -114
  106. package/src/package_load_metrics.ts +0 -127
  107. package/src/path_safety.spec.ts +0 -158
  108. package/src/path_safety.ts +0 -146
  109. package/src/pg_helpers.spec.ts +0 -223
  110. package/src/pg_helpers.ts +0 -52
  111. package/src/query_cap_metrics.spec.ts +0 -89
  112. package/src/query_cap_metrics.ts +0 -116
  113. package/src/query_concurrency.spec.ts +0 -247
  114. package/src/query_concurrency.ts +0 -237
  115. package/src/query_metadata_metrics.ts +0 -49
  116. package/src/query_param_utils.ts +0 -29
  117. package/src/query_timeout.spec.ts +0 -224
  118. package/src/query_timeout.ts +0 -179
  119. package/src/runtime/publisher.js +0 -328
  120. package/src/server-old.ts +0 -988
  121. package/src/server.ts +0 -2059
  122. package/src/service/annotations.spec.ts +0 -118
  123. package/src/service/annotations.ts +0 -91
  124. package/src/service/authorize.spec.ts +0 -154
  125. package/src/service/authorize.ts +0 -498
  126. package/src/service/authorize_integration.spec.ts +0 -2041
  127. package/src/service/build_plan.spec.ts +0 -685
  128. package/src/service/build_plan.ts +0 -797
  129. package/src/service/compile_authorize.spec.ts +0 -85
  130. package/src/service/compile_fragment_techniques.spec.ts +0 -156
  131. package/src/service/connection.spec.ts +0 -2360
  132. package/src/service/connection.ts +0 -2187
  133. package/src/service/connection_config.spec.ts +0 -734
  134. package/src/service/connection_config.ts +0 -899
  135. package/src/service/connection_federation.spec.ts +0 -184
  136. package/src/service/connection_fingerprint.spec.ts +0 -102
  137. package/src/service/connection_service.spec.ts +0 -711
  138. package/src/service/connection_service.ts +0 -291
  139. package/src/service/cron_evaluator.spec.ts +0 -79
  140. package/src/service/cron_evaluator.ts +0 -105
  141. package/src/service/db_utils.spec.ts +0 -1085
  142. package/src/service/db_utils.ts +0 -1502
  143. package/src/service/duckdb_instance_isolation.spec.ts +0 -137
  144. package/src/service/ducklake_lazy_attach.spec.ts +0 -110
  145. package/src/service/embedding_provider.spec.ts +0 -329
  146. package/src/service/embedding_provider.ts +0 -236
  147. package/src/service/environment.ts +0 -2191
  148. package/src/service/environment_admission.spec.ts +0 -344
  149. package/src/service/environment_compile.spec.ts +0 -197
  150. package/src/service/environment_store.spec.ts +0 -2274
  151. package/src/service/environment_store.ts +0 -2434
  152. package/src/service/environment_store_anchoring.spec.ts +0 -107
  153. package/src/service/environment_store_clone.spec.ts +0 -350
  154. package/src/service/explore_visibility.spec.ts +0 -434
  155. package/src/service/exports_probe.spec.ts +0 -107
  156. package/src/service/extension_fetch_policy.spec.ts +0 -256
  157. package/src/service/filter.spec.ts +0 -458
  158. package/src/service/filter.ts +0 -341
  159. package/src/service/filter_bypass.spec.ts +0 -418
  160. package/src/service/filter_integration.spec.ts +0 -1004
  161. package/src/service/freshness.spec.ts +0 -183
  162. package/src/service/freshness.ts +0 -112
  163. package/src/service/gcs_s3_utils.ts +0 -427
  164. package/src/service/given.ts +0 -105
  165. package/src/service/givens_integration.spec.ts +0 -219
  166. package/src/service/manifest_loader.spec.ts +0 -189
  167. package/src/service/manifest_loader.ts +0 -156
  168. package/src/service/materialization_build_session.spec.ts +0 -435
  169. package/src/service/materialization_build_session.ts +0 -681
  170. package/src/service/materialization_config_validation.spec.ts +0 -99
  171. package/src/service/materialization_config_validation.ts +0 -120
  172. package/src/service/materialization_eligibility.spec.ts +0 -158
  173. package/src/service/materialization_eligibility.ts +0 -305
  174. package/src/service/materialization_schedule_surface.spec.ts +0 -256
  175. package/src/service/materialization_scheduler.spec.ts +0 -289
  176. package/src/service/materialization_scheduler.ts +0 -304
  177. package/src/service/materialization_serve_transform.spec.ts +0 -1003
  178. package/src/service/materialization_serve_transform.ts +0 -779
  179. package/src/service/materialization_service.spec.ts +0 -2373
  180. package/src/service/materialization_service.ts +0 -2342
  181. package/src/service/materialization_test_fixtures.ts +0 -249
  182. package/src/service/model.spec.ts +0 -1157
  183. package/src/service/model.ts +0 -3370
  184. package/src/service/model_limits.spec.ts +0 -209
  185. package/src/service/model_limits.ts +0 -131
  186. package/src/service/model_storage_serve.spec.ts +0 -193
  187. package/src/service/model_storage_serve_joins.spec.ts +0 -193
  188. package/src/service/package.spec.ts +0 -612
  189. package/src/service/package.ts +0 -1799
  190. package/src/service/package_manifest.spec.ts +0 -230
  191. package/src/service/package_manifest.ts +0 -241
  192. package/src/service/package_memory_governor.spec.ts +0 -173
  193. package/src/service/package_memory_governor.ts +0 -233
  194. package/src/service/package_quote_bound_tables.spec.ts +0 -171
  195. package/src/service/package_race.spec.ts +0 -208
  196. package/src/service/package_reload_safety.spec.ts +0 -193
  197. package/src/service/package_rollback.spec.ts +0 -190
  198. package/src/service/package_worker_path.spec.ts +0 -645
  199. package/src/service/path_injection.spec.ts +0 -39
  200. package/src/service/persist_annotation_validation.spec.ts +0 -89
  201. package/src/service/persist_annotation_validation.ts +0 -59
  202. package/src/service/persistence_policy.spec.ts +0 -374
  203. package/src/service/proxy.spec.ts +0 -414
  204. package/src/service/proxy.ts +0 -248
  205. package/src/service/query_boundary.spec.ts +0 -470
  206. package/src/service/query_metadata.spec.ts +0 -408
  207. package/src/service/query_metadata.ts +0 -492
  208. package/src/service/query_metadata_identity.spec.ts +0 -149
  209. package/src/service/query_text.spec.ts +0 -79
  210. package/src/service/query_text.ts +0 -43
  211. package/src/service/quoting.spec.ts +0 -135
  212. package/src/service/quoting.ts +0 -78
  213. package/src/service/resolve_environment.ts +0 -15
  214. package/src/service/restricted_mode.spec.ts +0 -299
  215. package/src/service/source_extraction.ts +0 -225
  216. package/src/service/theme_store.ts +0 -199
  217. package/src/storage/DatabaseInterface.ts +0 -198
  218. package/src/storage/StorageManager.ts +0 -113
  219. package/src/storage/duckdb/ConnectionRepository.ts +0 -143
  220. package/src/storage/duckdb/DuckDBConnection.ts +0 -123
  221. package/src/storage/duckdb/DuckDBRepository.ts +0 -208
  222. package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
  223. package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
  224. package/src/storage/duckdb/MaterializationRepository.ts +0 -336
  225. package/src/storage/duckdb/PackageRepository.ts +0 -135
  226. package/src/storage/duckdb/schema.ts +0 -229
  227. package/src/stream_helpers.spec.ts +0 -280
  228. package/src/stream_helpers.ts +0 -162
  229. package/src/telemetry.ts +0 -20
  230. package/src/test_helpers/metrics_harness.ts +0 -166
  231. package/src/theme_key_parity.spec.ts +0 -57
  232. package/src/utils.ts +0 -29
  233. package/tests/fixtures/authorize-compile/model.malloy +0 -9
  234. package/tests/fixtures/authorize-compile/publisher.json +0 -4
  235. package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
  236. package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
  237. package/tests/fixtures/html-pages-test/data.csv +0 -3
  238. package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
  239. package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
  240. package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
  241. package/tests/fixtures/html-pages-test/public/data.json +0 -1
  242. package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
  243. package/tests/fixtures/html-pages-test/public/index.html +0 -9
  244. package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
  245. package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
  246. package/tests/fixtures/html-pages-test/public/slides.html +0 -12
  247. package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
  248. package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
  249. package/tests/fixtures/html-pages-test/publisher.json +0 -5
  250. package/tests/fixtures/html-pages-test/report.malloy +0 -1
  251. package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
  252. package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
  253. package/tests/fixtures/persist-multi-level/publisher.json +0 -5
  254. package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
  255. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
  256. package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
  257. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
  258. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
  259. package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
  260. package/tests/fixtures/persist-test/data/orders.csv +0 -5
  261. package/tests/fixtures/persist-test/persist_test.malloy +0 -11
  262. package/tests/fixtures/persist-test/publisher.json +0 -5
  263. package/tests/fixtures/publisher.config.json +0 -15
  264. package/tests/fixtures/query-givens/data/orders.csv +0 -7
  265. package/tests/fixtures/query-givens/model.malloy +0 -34
  266. package/tests/fixtures/query-givens/publisher.json +0 -5
  267. package/tests/fixtures/xlsx/database.xlsx +0 -0
  268. package/tests/harness/mcp_test_setup.ts +0 -332
  269. package/tests/harness/rest_e2e.ts +0 -68
  270. package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
  271. package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
  272. package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
  273. package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
  274. package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
  275. package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
  276. package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
  277. package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
  278. package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
  279. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
  280. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
  281. package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
  282. package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
  283. package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
  284. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
  285. package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
  286. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
  287. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
  288. package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
  289. package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
  290. package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
  291. package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
  292. package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
  293. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
  294. package/tests/unit/controller/theme.controller.spec.ts +0 -141
  295. package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
  296. package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
  297. package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
  298. package/tests/unit/duckdb/repositories.test.ts +0 -208
  299. package/tests/unit/ducklake/ducklake.test.ts +0 -952
  300. package/tests/unit/service/theme_store.spec.ts +0 -139
  301. package/tsconfig.json +0 -32
@@ -1,2373 +0,0 @@
1
- import type { Connection as MalloyConnection } from "@malloydata/malloy";
2
- import { Manifest } from "@malloydata/malloy";
3
- import { afterEach, beforeEach, describe, expect, it } from "bun:test";
4
- import * as sinon from "sinon";
5
- import {
6
- MaterializationEligibilityError,
7
- BadRequestError,
8
- EnvironmentNotFoundError,
9
- InvalidStateTransitionError,
10
- MaterializationConflictError,
11
- MaterializationNotFoundError,
12
- } from "../errors";
13
- import {
14
- BuildInstruction,
15
- MaterializationStatus,
16
- ResourceRepository,
17
- } from "../storage/DatabaseInterface";
18
- import { DuplicateActiveMaterializationError } from "../storage/duckdb/MaterializationRepository";
19
- import { EnvironmentStore } from "./environment_store";
20
- import {
21
- compiledWith,
22
- fakeSource,
23
- makeBuildPlan,
24
- makeInstruction,
25
- makeMaterialization,
26
- } from "./materialization_test_fixtures";
27
- import {
28
- manifestExcludingStorage,
29
- MaterializationService,
30
- redactConnectionSecrets,
31
- stagingSuffix,
32
- } from "./materialization_service";
33
- import { logger } from "../logger";
34
- import { resetMaterializationTelemetryForTesting } from "../materialization_metrics";
35
- import {
36
- startMetricsHarness,
37
- type MetricsHarness,
38
- } from "../test_helpers/metrics_harness";
39
-
40
- type MockRepo = sinon.SinonStubbedInstance<ResourceRepository>;
41
-
42
- /**
43
- * Turn per-query metadata on for one test and put the environment back after.
44
- * The feature ships dark, so a test that asserts what a statement carries has
45
- * to enable it — and saying so per test keeps the default honest here: a test
46
- * that forgets is testing the shipped default, which is "nothing attached".
47
- */
48
- function withQueryMetadataOn(): void {
49
- const original = process.env.PUBLISHER_QUERY_METADATA;
50
- beforeEach(() => {
51
- process.env.PUBLISHER_QUERY_METADATA = "on";
52
- });
53
- afterEach(() => {
54
- if (original === undefined) delete process.env.PUBLISHER_QUERY_METADATA;
55
- else process.env.PUBLISHER_QUERY_METADATA = original;
56
- });
57
- }
58
-
59
- describe("redactConnectionSecrets", () => {
60
- it("strips credential values but keeps the message legible", () => {
61
- const source = {
62
- name: "orders_pg",
63
- type: "postgres",
64
- postgresConnection: {
65
- host: "db.internal",
66
- userName: "svc",
67
- password: "sup3r-secret-pw",
68
- },
69
- };
70
- const msg =
71
- "IO Error: could not connect password=sup3r-secret-pw to db.internal";
72
- const out = redactConnectionSecrets(msg, source);
73
- expect(out).not.toContain("sup3r-secret-pw");
74
- expect(out).toContain("could not connect");
75
- expect(out).toContain("db.internal"); // host isn't a secret — stays
76
- });
77
-
78
- it("passes a credential-free error (e.g. schema not found) through verbatim", () => {
79
- const source = {
80
- name: "lake",
81
- type: "ducklake",
82
- ducklakeConnection: {
83
- catalog: { postgresConnection: { password: "catpw123" } },
84
- },
85
- };
86
- const msg =
87
- "Catalog Error: Schema 'analytics' not found in DuckLake 'lake'";
88
- expect(redactConnectionSecrets(msg, source)).toBe(msg);
89
- });
90
-
91
- it("redacts a secret in its SQL-escaped form (single quotes doubled)", () => {
92
- // A DuckDB error echoes the offending statement, in which a secret with a
93
- // single quote appears escaped (`'` -> `''`), so the raw value won't match.
94
- const source = {
95
- name: "pg",
96
- type: "postgres",
97
- postgresConnection: { password: "pa'ss'word" },
98
- };
99
- const msg =
100
- "Parser Error: syntax error near 'password=pa''ss''word host=h'";
101
- const out = redactConnectionSecrets(msg, source);
102
- expect(out).not.toContain("pa''ss''word");
103
- expect(out).toContain("***");
104
- });
105
-
106
- // Stubbed at the seam because the leak needs an ATTACH (not CTAS) failure:
107
- // bad catalog creds fast-fail at connection validation, so no black-box test
108
- // can reach this branch.
109
- it("redacts an INFRA chained-build failure, and does not call it a refusal", async () => {
110
- const sandbox = sinon.createSandbox();
111
- try {
112
- const destinationConnection = {
113
- name: "lake",
114
- type: "ducklake",
115
- ducklakeConnection: {
116
- catalog: {
117
- postgresConnection: {
118
- host: "catalog.internal",
119
- userName: "lake_rw",
120
- password: "c4talog-pw",
121
- },
122
- },
123
- },
124
- };
125
- const sourceConnection = {
126
- name: "orders_pg",
127
- type: "postgres",
128
- postgresConnection: { host: "db.internal", password: "src-pw-99" },
129
- };
130
-
131
- // A failed RW ATTACH echoes the statement, connstring included.
132
- sandbox
133
- .stub(
134
- MaterializationService.prototype as unknown as Record<
135
- string,
136
- unknown
137
- >,
138
- "buildDownstreamViaParents" as never,
139
- )
140
- .rejects(
141
- new Error(
142
- "IO Error: failed to ATTACH 'ducklake:postgres:host=catalog.internal " +
143
- "user=lake_rw password=c4talog-pw' (db lake)",
144
- ),
145
- );
146
-
147
- const { service } = createMocks();
148
- const environment = {
149
- getApiConnection: (name: string) =>
150
- name === "lake" ? destinationConnection : sourceConnection,
151
- getEnvironmentPath: () => "/tmp/env",
152
- };
153
-
154
- const call = (
155
- service as unknown as {
156
- buildOneSourceIntoStorage: (...a: unknown[]) => Promise<unknown>;
157
- }
158
- ).buildOneSourceIntoStorage(
159
- { name: "orders_by_month", connectionName: "orders_pg" },
160
- {
161
- sourceEntityId: "sid-1",
162
- physicalTableName: "mz_orders_by_month",
163
- destination: "lake",
164
- },
165
- // strict: throw instead of falling through to recompute-from-raw.
166
- { strict: true, update: () => {} },
167
- environment,
168
- "SELECT 1",
169
- {},
170
- true, // dependsOnStorageUpstream — take the chained path
171
- );
172
-
173
- // A failed ATTACH is infrastructure, not a shape limit: it must NOT
174
- // present as the strict-upstreams refusal (that message means "we could
175
- // have recomputed from raw but you forbade it", which is not what
176
- // happened here).
177
- await expect(call).rejects.toThrow(
178
- /Failed to materialize chained source/i,
179
- );
180
- await expect(call).rejects.not.toThrow(/strict upstreams forbid/i);
181
- const message = await call.then(
182
- () => "",
183
- (e: unknown) => (e instanceof Error ? e.message : String(e)),
184
- );
185
- expect(message).not.toContain("c4talog-pw");
186
- expect(message).toContain("***");
187
- // Redacted but still legible.
188
- expect(message).toContain("failed to ATTACH");
189
- expect(message).toContain("catalog.internal");
190
- expect(message).toContain("orders_by_month");
191
- } finally {
192
- sandbox.restore();
193
- }
194
- });
195
- });
196
-
197
- function createMocks() {
198
- const sandbox = sinon.createSandbox();
199
-
200
- const repository = {
201
- getEnvironmentByName: sandbox.stub(),
202
- listMaterializations: sandbox.stub(),
203
- getMaterializationById: sandbox.stub(),
204
- getActiveMaterialization: sandbox.stub(),
205
- createMaterialization: sandbox.stub(),
206
- updateMaterialization: sandbox.stub(),
207
- deleteMaterialization: sandbox.stub(),
208
- } as unknown as MockRepo;
209
-
210
- const storageManager = { getRepository: () => repository };
211
-
212
- const environmentStore = {
213
- storageManager,
214
- getEnvironment: sandbox.stub(),
215
- } as unknown as EnvironmentStore;
216
-
217
- // Default: environment resolves and yields a package whose build plan is
218
- // empty (no persist sources). Individual tests override getBuildPlan to
219
- // exercise the orchestrated path. getModelPaths()=>[] keeps the background
220
- // build from touching the Malloy runtime.
221
- repository.getEnvironmentByName.resolves({
222
- id: "env-1",
223
- name: "my-env",
224
- path: "/test",
225
- createdAt: new Date(),
226
- updatedAt: new Date(),
227
- });
228
- setPackage(environmentStore, { getBuildPlan: () => null });
229
-
230
- const service = new MaterializationService(environmentStore);
231
-
232
- return { sandbox, repository, environmentStore, service };
233
- }
234
-
235
- /** Point environmentStore.getEnvironment at a package with the given overrides. */
236
- function setPackage(
237
- environmentStore: EnvironmentStore,
238
- pkgOverrides: Record<string, unknown>,
239
- ): void {
240
- const pkg = {
241
- getBuildPlan: () => null,
242
- getModelPaths: () => [],
243
- getPackagePath: () => "/test",
244
- getMalloyConfig: () => ({}),
245
- getMalloyConnection: async () => ({}),
246
- // No bound manifestLocation by default, so seedFromBoundManifest no-ops.
247
- getPackageMetadata: () => ({}),
248
- ...pkgOverrides,
249
- };
250
- (environmentStore.getEnvironment as sinon.SinonStub).resolves({
251
- getPackage: sinon.stub().resolves(pkg),
252
- withPackageLock: async (_name: string, fn: () => Promise<unknown>) =>
253
- fn(),
254
- reloadAllModelsForPackage: sinon.stub().resolves(),
255
- });
256
- }
257
-
258
- describe("MaterializationService", () => {
259
- let ctx: ReturnType<typeof createMocks>;
260
-
261
- beforeEach(() => {
262
- ctx = createMocks();
263
- });
264
-
265
- describe("resolveEnvironmentId", () => {
266
- it("throws EnvironmentNotFoundError when the environment is missing", async () => {
267
- ctx.repository.getEnvironmentByName.resolves(null);
268
- await expect(
269
- ctx.service.listMaterializations("unknown", "pkg"),
270
- ).rejects.toThrow(EnvironmentNotFoundError);
271
- });
272
- });
273
-
274
- describe("deleteMaterialization drop of a storage= table", () => {
275
- it("routes a storage entry to a destination-aware RW drop (best-effort, delete still completes)", async () => {
276
- // getApiConnection returns a non-storage type so the RW drop refuses
277
- // at its destination gate (pre-session), exercising the routing + the
278
- // best-effort failure handling without needing a live DuckDB attach.
279
- const getApiConnection = sinon
280
- .stub()
281
- .returns({ name: "lake", type: "postgres" });
282
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
283
- getPackage: sinon
284
- .stub()
285
- .resolves({ getMalloyConnection: async () => ({}) }),
286
- getApiConnection,
287
- getEnvironmentPath: () => "/test",
288
- });
289
- ctx.repository.getMaterializationById.resolves(
290
- makeMaterialization({
291
- status: "MANIFEST_FILE_READY",
292
- manifest: {
293
- entries: {
294
- se1: {
295
- sourceEntityId: "se1",
296
- sourceName: "daily",
297
- physicalTableName: "daily__mabc123",
298
- connectionName: "wh",
299
- storageConnectionName: "lake",
300
- },
301
- },
302
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
303
- } as any,
304
- }),
305
- );
306
-
307
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
308
- dropTables: true,
309
- });
310
-
311
- // Routed to the storage path (resolved the destination connection for
312
- // an RW drop, not the old skip), and the delete completed despite the
313
- // drop's best-effort failure.
314
- expect(getApiConnection.calledWith("lake")).toBe(true);
315
- expect(
316
- (
317
- ctx.repository.deleteMaterialization as sinon.SinonStub
318
- ).calledWith("mat-1"),
319
- ).toBe(true);
320
- });
321
-
322
- it("skips the drop when another MANIFEST_FILE_READY run still references the table (MED-3)", async () => {
323
- // name=-verbatim: generations of a source share one physical name, so
324
- // dropping a superseded record's table must NOT take out the table a
325
- // remaining run still serves.
326
- const getApiConnection = sinon
327
- .stub()
328
- .returns({ name: "lake", type: "duckdb" });
329
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
330
- getPackage: sinon
331
- .stub()
332
- .resolves({ getMalloyConnection: async () => ({}) }),
333
- getApiConnection,
334
- getEnvironmentPath: () => "/test",
335
- });
336
- const entries = {
337
- se1: {
338
- sourceEntityId: "se1",
339
- sourceName: "daily",
340
- physicalTableName: "daily", // shared logical name
341
- connectionName: "wh",
342
- storageConnectionName: "lake",
343
- },
344
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
345
- } as any;
346
- ctx.repository.getMaterializationById.resolves(
347
- makeMaterialization({
348
- id: "mat-1",
349
- status: "MANIFEST_FILE_READY",
350
- manifest: { entries },
351
- }),
352
- );
353
- // Another remaining successful run references the SAME lake.daily table.
354
- ctx.repository.listMaterializations.resolves([
355
- makeMaterialization({ id: "mat-1", manifest: { entries } }),
356
- makeMaterialization({
357
- id: "mat-2",
358
- status: "MANIFEST_FILE_READY",
359
- manifest: { entries },
360
- }),
361
- ]);
362
-
363
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
364
- dropTables: true,
365
- });
366
-
367
- // The drop was skipped (never resolved the destination connection), and
368
- // the record was still deleted.
369
- expect(getApiConnection.called).toBe(false);
370
- expect(
371
- (
372
- ctx.repository.deleteMaterialization as sinon.SinonStub
373
- ).calledWith("mat-1"),
374
- ).toBe(true);
375
- });
376
- });
377
-
378
- describe("deleteMaterialization rebinds serve bindings (both tiers)", () => {
379
- const storageEntry = {
380
- se1: {
381
- sourceEntityId: "se1",
382
- sourceName: "daily",
383
- physicalTableName: "daily__mabc",
384
- connectionName: "wh",
385
- storageConnectionName: "lake",
386
- schema: [{ name: "a", type: "BIGINT" }],
387
- },
388
- };
389
- // A colocated entry carries NO storageConnectionName.
390
- const colocatedEntry = {
391
- ce1: {
392
- sourceEntityId: "ce1",
393
- sourceName: "daily",
394
- physicalTableName: "daily__cabc",
395
- connectionName: "wh",
396
- },
397
- };
398
-
399
- function setupEnv(): {
400
- storageSpy: sinon.SinonStub;
401
- colocatedSpy: sinon.SinonStub;
402
- } {
403
- const storageSpy = sinon.stub().resolves();
404
- const colocatedSpy = sinon.stub().resolves();
405
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
406
- getPackage: sinon
407
- .stub()
408
- .resolves({ getMalloyConnection: async () => ({}) }),
409
- getApiConnection: () => ({ name: "lake", type: "duckdb" }),
410
- getEnvironmentPath: () => "/test",
411
- bindPackageStorageServeBindings: storageSpy,
412
- bindPackageColocatedServeManifest: colocatedSpy,
413
- });
414
- ctx.repository.getMaterializationById.resolves(
415
- makeMaterialization({
416
- status: "MANIFEST_FILE_READY",
417
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
418
- manifest: { entries: {} } as any,
419
- }),
420
- );
421
- return { storageSpy, colocatedSpy };
422
- }
423
-
424
- it("re-derives storage bindings from the next-latest materialization (on)", async () => {
425
- process.env.PERSIST_STORAGE_MODE = "on";
426
- try {
427
- const { storageSpy, colocatedSpy } = setupEnv();
428
- // The latest REMAINING successful run carries a storage entry.
429
- ctx.repository.listMaterializations.resolves([
430
- makeMaterialization({
431
- id: "m-prev",
432
- status: "MANIFEST_FILE_READY",
433
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
434
- manifest: { entries: storageEntry } as any,
435
- }),
436
- ]);
437
-
438
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
439
-
440
- expect(storageSpy.calledOnce).toBe(true);
441
- expect(Object.keys(storageSpy.firstCall.args[1])).toContain("se1");
442
- // Colocated is re-derived too (with no colocated entries ⇒ cleared).
443
- expect(colocatedSpy.calledOnce).toBe(true);
444
- expect(colocatedSpy.firstCall.args[1]).toEqual({});
445
- } finally {
446
- delete process.env.PERSIST_STORAGE_MODE;
447
- }
448
- });
449
-
450
- it("re-derives colocated bindings regardless of tier mode (off)", async () => {
451
- // PERSIST_STORAGE_MODE unset ⇒ off. Colocated is the v0 path, not gated
452
- // by the storage kill switch, so it is still re-derived; storage is not.
453
- const { storageSpy, colocatedSpy } = setupEnv();
454
- ctx.repository.listMaterializations.resolves([
455
- makeMaterialization({
456
- id: "m-prev",
457
- status: "MANIFEST_FILE_READY",
458
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
459
- manifest: { entries: colocatedEntry } as any,
460
- }),
461
- ]);
462
-
463
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
464
-
465
- expect(colocatedSpy.calledOnce).toBe(true);
466
- expect(Object.keys(colocatedSpy.firstCall.args[1])).toContain("ce1");
467
- // Storage rebind is skipped when the tier is off.
468
- expect(storageSpy.called).toBe(false);
469
- });
470
-
471
- it("clears both tiers' bindings when no successful materialization remains (on)", async () => {
472
- process.env.PERSIST_STORAGE_MODE = "on";
473
- try {
474
- const { storageSpy, colocatedSpy } = setupEnv();
475
- ctx.repository.listMaterializations.resolves([]); // none remain
476
-
477
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
478
-
479
- expect(storageSpy.calledOnce).toBe(true);
480
- expect(storageSpy.firstCall.args[1]).toEqual({}); // empty ⇒ serve live
481
- expect(colocatedSpy.calledOnce).toBe(true);
482
- expect(colocatedSpy.firstCall.args[1]).toEqual({});
483
- } finally {
484
- delete process.env.PERSIST_STORAGE_MODE;
485
- }
486
- });
487
-
488
- it("skips only the storage rebind when the tier is off (dark-ship)", async () => {
489
- const { storageSpy, colocatedSpy } = setupEnv(); // off
490
- ctx.repository.listMaterializations.resolves([]); // none remain
491
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
492
- // Storage is skipped when off; colocated still re-derives (to clear).
493
- expect(storageSpy.called).toBe(false);
494
- expect(colocatedSpy.calledOnce).toBe(true);
495
- expect(colocatedSpy.firstCall.args[1]).toEqual({});
496
- });
497
- });
498
-
499
- describe("queries", () => {
500
- it("lists materializations for a package", async () => {
501
- const rows = [
502
- makeMaterialization(),
503
- makeMaterialization({ id: "m2" }),
504
- ];
505
- ctx.repository.listMaterializations.resolves(rows);
506
- expect(
507
- await ctx.service.listMaterializations("my-env", "pkg"),
508
- ).toEqual(rows);
509
- });
510
-
511
- it("returns a specific materialization", async () => {
512
- const m = makeMaterialization();
513
- ctx.repository.getMaterializationById.resolves(m);
514
- expect(
515
- await ctx.service.getMaterialization("my-env", "pkg", "mat-1"),
516
- ).toEqual(m);
517
- });
518
-
519
- it("throws when the materialization is missing", async () => {
520
- ctx.repository.getMaterializationById.resolves(null);
521
- await expect(
522
- ctx.service.getMaterialization("my-env", "pkg", "nope"),
523
- ).rejects.toThrow(MaterializationNotFoundError);
524
- });
525
-
526
- it("throws when the materialization belongs to another package", async () => {
527
- ctx.repository.getMaterializationById.resolves(
528
- makeMaterialization({ packageName: "other" }),
529
- );
530
- await expect(
531
- ctx.service.getMaterialization("my-env", "pkg", "mat-1"),
532
- ).rejects.toThrow(MaterializationNotFoundError);
533
- });
534
- });
535
-
536
- describe("createMaterialization (auto-run)", () => {
537
- it("creates a PENDING record and persists options with mode=auto", async () => {
538
- ctx.repository.getActiveMaterialization.resolves(null);
539
- const pending = makeMaterialization({ status: "PENDING" });
540
- ctx.repository.createMaterialization.resolves(pending);
541
-
542
- const result = await ctx.service.createMaterialization(
543
- "my-env",
544
- "pkg",
545
- {
546
- forceRefresh: true,
547
- sourceNames: ["orders"],
548
- },
549
- );
550
-
551
- expect(result.status).toBe("PENDING");
552
- expect(ctx.repository.createMaterialization.firstCall.args).toEqual([
553
- "env-1",
554
- "pkg",
555
- "PENDING",
556
- {
557
- forceRefresh: true,
558
- sourceNames: ["orders"],
559
- mode: "auto",
560
- trigger: "ON_DEMAND",
561
- },
562
- ]);
563
- });
564
-
565
- it("defaults to auto mode in metadata", async () => {
566
- ctx.repository.getActiveMaterialization.resolves(null);
567
- ctx.repository.createMaterialization.resolves(
568
- makeMaterialization({ status: "PENDING" }),
569
- );
570
-
571
- await ctx.service.createMaterialization("my-env", "pkg", {});
572
-
573
- expect(ctx.repository.createMaterialization.firstCall.args[3]).toEqual(
574
- {
575
- forceRefresh: false,
576
- sourceNames: null,
577
- mode: "auto",
578
- trigger: "ON_DEMAND",
579
- },
580
- );
581
- });
582
-
583
- it("records trigger=SCHEDULER when the scheduler fires the run", async () => {
584
- ctx.repository.getActiveMaterialization.resolves(null);
585
- ctx.repository.createMaterialization.resolves(
586
- makeMaterialization({ status: "PENDING" }),
587
- );
588
-
589
- await ctx.service.createMaterialization("my-env", "pkg", {
590
- forceRefresh: true,
591
- trigger: "SCHEDULER",
592
- });
593
-
594
- expect(
595
- ctx.repository.createMaterialization.firstCall.args[3],
596
- ).toMatchObject({ mode: "auto", trigger: "SCHEDULER" });
597
- });
598
-
599
- it("rejects when an active materialization already exists", async () => {
600
- ctx.repository.getActiveMaterialization.resolves(
601
- makeMaterialization({
602
- id: "existing",
603
- status: "MANIFEST_ROWS_READY",
604
- }),
605
- );
606
- await expect(
607
- ctx.service.createMaterialization("my-env", "pkg"),
608
- ).rejects.toThrow(MaterializationConflictError);
609
- });
610
-
611
- it("translates a lost create race into a conflict", async () => {
612
- ctx.repository.getActiveMaterialization
613
- .onFirstCall()
614
- .resolves(null)
615
- .onSecondCall()
616
- .resolves(makeMaterialization({ id: "winner" }));
617
- ctx.repository.createMaterialization.rejects(
618
- new DuplicateActiveMaterializationError("env-1", "pkg"),
619
- );
620
- await expect(
621
- ctx.service.createMaterialization("my-env", "pkg"),
622
- ).rejects.toThrow(/winner/);
623
- });
624
- });
625
-
626
- describe("createMaterialization (orchestrated)", () => {
627
- it("creates a PENDING record with mode=orchestrated for valid instructions", async () => {
628
- setPackage(ctx.environmentStore, {
629
- getBuildPlan: () => makeBuildPlan(),
630
- });
631
- ctx.repository.getActiveMaterialization.resolves(null);
632
- ctx.repository.createMaterialization.resolves(
633
- makeMaterialization({ status: "PENDING" }),
634
- );
635
-
636
- const result = await ctx.service.createMaterialization(
637
- "my-env",
638
- "pkg",
639
- { buildInstructions: [makeInstruction()] },
640
- );
641
-
642
- expect(result.status).toBe("PENDING");
643
- expect(
644
- ctx.repository.createMaterialization.firstCall.args[3],
645
- ).toMatchObject({ mode: "orchestrated" });
646
- });
647
-
648
- it("rejects instructions referencing an unknown sourceEntityId before creating", async () => {
649
- setPackage(ctx.environmentStore, {
650
- getBuildPlan: () => makeBuildPlan(),
651
- });
652
- await expect(
653
- ctx.service.createMaterialization("my-env", "pkg", {
654
- buildInstructions: [
655
- makeInstruction({ sourceEntityId: "ghost" }),
656
- ],
657
- }),
658
- ).rejects.toThrow(BadRequestError);
659
- expect(ctx.repository.createMaterialization.called).toBe(false);
660
- });
661
-
662
- it("rejects SNAPSHOT realization", async () => {
663
- setPackage(ctx.environmentStore, {
664
- getBuildPlan: () => makeBuildPlan(),
665
- });
666
- await expect(
667
- ctx.service.createMaterialization("my-env", "pkg", {
668
- buildInstructions: [
669
- makeInstruction({ realization: "SNAPSHOT" }),
670
- ],
671
- }),
672
- ).rejects.toThrow(BadRequestError);
673
- });
674
-
675
- it("rejects buildInstructions when the package has no persist sources", async () => {
676
- setPackage(ctx.environmentStore, { getBuildPlan: () => null });
677
- await expect(
678
- ctx.service.createMaterialization("my-env", "pkg", {
679
- buildInstructions: [makeInstruction()],
680
- }),
681
- ).rejects.toThrow(BadRequestError);
682
- });
683
- });
684
-
685
- describe("stopMaterialization", () => {
686
- const cancellable = ["PENDING", "MANIFEST_ROWS_READY"] as const;
687
-
688
- for (const status of cancellable) {
689
- it(`cancels a ${status} materialization`, async () => {
690
- ctx.repository.getMaterializationById.resolves(
691
- makeMaterialization({ status }),
692
- );
693
- ctx.repository.updateMaterialization.resolves(
694
- makeMaterialization({ status: "CANCELLED" }),
695
- );
696
- const result = await ctx.service.stopMaterialization(
697
- "my-env",
698
- "pkg",
699
- "mat-1",
700
- );
701
- expect(result.status).toBe("CANCELLED");
702
- });
703
- }
704
-
705
- const terminal = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"] as const;
706
- for (const status of terminal) {
707
- it(`rejects stopping a ${status} materialization`, async () => {
708
- ctx.repository.getMaterializationById.resolves(
709
- makeMaterialization({ status }),
710
- );
711
- await expect(
712
- ctx.service.stopMaterialization("my-env", "pkg", "mat-1"),
713
- ).rejects.toThrow(InvalidStateTransitionError);
714
- });
715
- }
716
- });
717
-
718
- describe("deleteMaterialization", () => {
719
- const terminal = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"] as const;
720
- for (const status of terminal) {
721
- it(`deletes a ${status} materialization`, async () => {
722
- ctx.repository.getMaterializationById.resolves(
723
- makeMaterialization({ status }),
724
- );
725
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
726
- expect(
727
- ctx.repository.deleteMaterialization.calledOnceWith("mat-1"),
728
- ).toBe(true);
729
- });
730
- }
731
-
732
- const active = ["PENDING", "MANIFEST_ROWS_READY"] as const;
733
- for (const status of active) {
734
- it(`rejects deleting a ${status} materialization`, async () => {
735
- ctx.repository.getMaterializationById.resolves(
736
- makeMaterialization({ status }),
737
- );
738
- await expect(
739
- ctx.service.deleteMaterialization("my-env", "pkg", "mat-1"),
740
- ).rejects.toThrow(InvalidStateTransitionError);
741
- expect(ctx.repository.deleteMaterialization.called).toBe(false);
742
- });
743
- }
744
-
745
- it("does not drop tables by default (record-only delete)", async () => {
746
- const runSQL = sinon.stub().resolves();
747
- const getMalloyConnection = sinon.stub().resolves({ runSQL });
748
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
749
- getPackage: sinon.stub().resolves({ getMalloyConnection }),
750
- withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
751
- fn(),
752
- });
753
- ctx.repository.getMaterializationById.resolves(
754
- makeMaterialization({
755
- status: "MANIFEST_FILE_READY",
756
- manifest: {
757
- builtAt: new Date().toISOString(),
758
- strict: false,
759
- entries: {
760
- b1: {
761
- sourceEntityId: "b1",
762
- physicalTableName: "orders_mz",
763
- connectionName: "duckdb",
764
- },
765
- },
766
- },
767
- }),
768
- );
769
-
770
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
771
-
772
- expect(getMalloyConnection.called).toBe(false);
773
- expect(runSQL.called).toBe(false);
774
- expect(
775
- ctx.repository.deleteMaterialization.calledOnceWith("mat-1"),
776
- ).toBe(true);
777
- });
778
-
779
- it("drops manifest tables (and staging) when dropTables is set", async () => {
780
- const runSQL = sinon.stub().resolves();
781
- const getMalloyConnection = sinon
782
- .stub()
783
- .resolves({ runSQL, dialectName: "duckdb" });
784
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
785
- getPackage: sinon.stub().resolves({ getMalloyConnection }),
786
- withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
787
- fn(),
788
- });
789
- ctx.repository.getMaterializationById.resolves(
790
- makeMaterialization({
791
- status: "MANIFEST_FILE_READY",
792
- manifest: {
793
- builtAt: new Date().toISOString(),
794
- strict: false,
795
- entries: {
796
- b1: {
797
- sourceEntityId: "b1",
798
- physicalTableName: "orders_mz",
799
- connectionName: "duckdb",
800
- },
801
- },
802
- },
803
- }),
804
- );
805
-
806
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
807
- dropTables: true,
808
- });
809
-
810
- expect(getMalloyConnection.calledOnceWith("duckdb")).toBe(true);
811
- const dropped = runSQL.getCalls().map((c) => c.args[0] as string);
812
- expect(dropped).toContain('DROP TABLE IF EXISTS "orders_mz"');
813
- expect(dropped.some((s) => s.includes('"orders_mz_'))).toBe(true);
814
- expect(
815
- ctx.repository.deleteMaterialization.calledOnceWith("mat-1"),
816
- ).toBe(true);
817
- });
818
-
819
- it("dialect-quotes the drop DDL from the live connection (backtick + container path)", async () => {
820
- const runSQL = sinon.stub().resolves();
821
- const getMalloyConnection = sinon
822
- .stub()
823
- .resolves({ runSQL, dialectName: "standardsql" });
824
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
825
- getPackage: sinon.stub().resolves({ getMalloyConnection }),
826
- withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
827
- fn(),
828
- });
829
- ctx.repository.getMaterializationById.resolves(
830
- makeMaterialization({
831
- status: "MANIFEST_FILE_READY",
832
- manifest: {
833
- builtAt: new Date().toISOString(),
834
- strict: false,
835
- entries: {
836
- b1: {
837
- sourceEntityId: "b1",
838
- // Container path on a hyphenated BigQuery project: each
839
- // segment must be backtick-quoted independently.
840
- physicalTableName: "my-proj.ds.engaged",
841
- connectionName: "bq",
842
- },
843
- },
844
- },
845
- }),
846
- );
847
-
848
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
849
- dropTables: true,
850
- });
851
-
852
- const dropped = runSQL.getCalls().map((c) => c.args[0] as string);
853
- expect(dropped).toContain(
854
- "DROP TABLE IF EXISTS `my-proj`.`ds`.`engaged`",
855
- );
856
- });
857
-
858
- it("still deletes the record when a table drop fails", async () => {
859
- const runSQL = sinon.stub().rejects(new Error("boom"));
860
- const getMalloyConnection = sinon.stub().resolves({ runSQL });
861
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
862
- getPackage: sinon.stub().resolves({ getMalloyConnection }),
863
- withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
864
- fn(),
865
- });
866
- ctx.repository.getMaterializationById.resolves(
867
- makeMaterialization({
868
- status: "FAILED",
869
- manifest: {
870
- builtAt: new Date().toISOString(),
871
- strict: false,
872
- entries: {
873
- b1: {
874
- sourceEntityId: "b1",
875
- physicalTableName: "orders_mz",
876
- connectionName: "duckdb",
877
- },
878
- },
879
- },
880
- }),
881
- );
882
-
883
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
884
- dropTables: true,
885
- });
886
-
887
- expect(
888
- ctx.repository.deleteMaterialization.calledOnceWith("mat-1"),
889
- ).toBe(true);
890
- });
891
- });
892
- });
893
-
894
- describe("deleteMaterialization telemetry", () => {
895
- let ctx: ReturnType<typeof createMocks>;
896
- let harness: MetricsHarness;
897
- withQueryMetadataOn();
898
-
899
- beforeEach(async () => {
900
- ctx = createMocks();
901
- harness = await startMetricsHarness();
902
- // Drop cached instruments so recordDropTables re-binds to this provider.
903
- resetMaterializationTelemetryForTesting();
904
- });
905
-
906
- afterEach(async () => {
907
- resetMaterializationTelemetryForTesting();
908
- await harness.shutdown();
909
- });
910
-
911
- const DROP_COUNTER = "publisher_materialization_drop_tables_total";
912
-
913
- // A terminal materialization whose manifest names one physical table, so
914
- // dropMaterializedTables issues exactly one DROP — the emission under test.
915
- function dropTablesFixture(runSQL: sinon.SinonStub) {
916
- const getMalloyConnection = sinon
917
- .stub()
918
- .resolves({ runSQL, dialectName: "duckdb" });
919
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
920
- getPackage: sinon.stub().resolves({ getMalloyConnection }),
921
- withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
922
- fn(),
923
- });
924
- ctx.repository.getMaterializationById.resolves(
925
- makeMaterialization({
926
- status: "MANIFEST_FILE_READY",
927
- manifest: {
928
- builtAt: new Date().toISOString(),
929
- strict: false,
930
- entries: {
931
- b1: {
932
- sourceEntityId: "b1",
933
- physicalTableName: "orders_mz",
934
- connectionName: "duckdb",
935
- },
936
- },
937
- },
938
- }),
939
- );
940
- }
941
-
942
- it("meters a successful physical-table drop as outcome=success", async () => {
943
- dropTablesFixture(sinon.stub().resolves());
944
-
945
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
946
- dropTables: true,
947
- });
948
-
949
- expect(
950
- await harness.collectCounter(DROP_COUNTER, { outcome: "success" }),
951
- ).toBe(1);
952
- expect(
953
- await harness.collectCounter(DROP_COUNTER, { outcome: "failure" }),
954
- ).toBe(0);
955
- });
956
-
957
- it("tags the retiring drops as ops work", async () => {
958
- // A drop is warehouse work someone accounts for later, and it is the one
959
- // statement an operator goes looking for after a table disappears.
960
- const runSQL = sinon.stub().resolves();
961
- dropTablesFixture(runSQL);
962
-
963
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
964
- dropTables: true,
965
- });
966
-
967
- expect(runSQL.callCount).toBeGreaterThan(0);
968
- for (const call of runSQL.getCalls()) {
969
- expect(call.args[1]?.queryMetadata).toMatchObject({
970
- class: "ops",
971
- environment: "my-env",
972
- package: "pkg",
973
- run_id: "mat-1",
974
- });
975
- }
976
- });
977
-
978
- it("meters a failed drop as outcome=failure and still deletes the record", async () => {
979
- dropTablesFixture(sinon.stub().rejects(new Error("drop boom")));
980
-
981
- // The drop is best-effort: a failure is metered but never surfaces to the
982
- // caller, and the record is still removed.
983
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
984
- dropTables: true,
985
- });
986
-
987
- expect(
988
- await harness.collectCounter(DROP_COUNTER, { outcome: "failure" }),
989
- ).toBe(1);
990
- expect(ctx.repository.deleteMaterialization.calledOnceWith("mat-1")).toBe(
991
- true,
992
- );
993
- });
994
- });
995
-
996
- describe("stagingSuffix", () => {
997
- it("derives a short, stable suffix from the sourceEntityId", () => {
998
- expect(stagingSuffix("abcdef1234567890")).toBe("_abcdef123456");
999
- });
1000
- });
1001
-
1002
- describe("manifestExcludingStorage (chained-storage inline)", () => {
1003
- it("drops storage entries, keeps colocated entries with the manifest's quoting, preserves strict", () => {
1004
- // The source manifest is already dialect-quoted by the seed loop (#904);
1005
- // manifestExcludingStorage reuses that quoting for the kept entries.
1006
- const manifest = new Manifest();
1007
- manifest.strict = true;
1008
- manifest.update("wh_up", { tableName: '"wh_up_v1"' });
1009
- manifest.update("lake_up", { tableName: '"lake"."lake_up__mabc"' });
1010
- const built: Record<string, unknown> = {
1011
- wh_up: {
1012
- sourceEntityId: "wh_up",
1013
- physicalTableName: "wh_up_v1",
1014
- connectionName: "duckdb",
1015
- // no storageConnectionName -> colocated, keep it
1016
- },
1017
- lake_up: {
1018
- sourceEntityId: "lake_up",
1019
- physicalTableName: "lake_up__mabc",
1020
- connectionName: "wh",
1021
- storageConnectionName: "lake", // storage -> exclude it
1022
- },
1023
- };
1024
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
1025
- const reduced = manifestExcludingStorage(manifest, built as any) as any;
1026
- expect(Object.keys(reduced.entries)).toEqual(["wh_up"]);
1027
- // Kept with the manifest's dialect-quoted name, not the raw physical name.
1028
- expect(reduced.entries.wh_up.tableName).toBe('"wh_up_v1"');
1029
- expect(reduced.entries.lake_up).toBeUndefined();
1030
- expect(reduced.strict).toBe(true);
1031
- });
1032
- });
1033
-
1034
- describe("deriveSelfInstructions", () => {
1035
- let ctx: ReturnType<typeof createMocks>;
1036
- beforeEach(() => {
1037
- ctx = createMocks();
1038
- });
1039
-
1040
- it("carries forward unchanged sourceEntityIds and builds the rest (deduping repeats)", () => {
1041
- const compiled = compiledWith(
1042
- {
1043
- s1: fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" }),
1044
- s2: fakeSource({ name: "s2", sourceEntityId: "b2bbbbbbbbbbbbbb" }),
1045
- },
1046
- [["s1", "s2"], ["s2"]],
1047
- );
1048
- const priorEntries = {
1049
- b1aaaaaaaaaaaaaa: {
1050
- sourceEntityId: "b1aaaaaaaaaaaaaa",
1051
- physicalTableName: "s1_prev",
1052
- connectionName: "duckdb",
1053
- },
1054
- };
1055
-
1056
- const { instructions, carried } = (
1057
- ctx.service as unknown as {
1058
- deriveSelfInstructions: (
1059
- c: unknown,
1060
- n: string[] | undefined,
1061
- p: unknown,
1062
- ) => { instructions: BuildInstruction[]; carried: unknown };
1063
- }
1064
- ).deriveSelfInstructions(compiled, undefined, priorEntries);
1065
-
1066
- expect(instructions).toHaveLength(1);
1067
- expect(instructions[0].sourceEntityId).toBe("b2bbbbbbbbbbbbbb");
1068
- expect(instructions[0].physicalTableName).toBe("s2");
1069
- expect(instructions[0].realization).toBe("COPY");
1070
- expect(instructions[0].materializedTableId).toBe("local-b2bbbbbbbbbb");
1071
- expect(
1072
- (carried as Record<string, unknown>)["b1aaaaaaaaaaaaaa"],
1073
- ).toBeDefined();
1074
- });
1075
-
1076
- it("self-assigns the name= verbatim for a storage= source, exactly as a colocated build", () => {
1077
- process.env.PERSIST_STORAGE_MODE = "on";
1078
- try {
1079
- const compiled = compiledWith(
1080
- {
1081
- lake_src: fakeSource({
1082
- name: "lake_src",
1083
- sourceEntityId: "aa11aa11aa11aa11aa11",
1084
- annotationFields: { storage: "lake", name: "daily" },
1085
- }),
1086
- wh_src: fakeSource({
1087
- name: "wh_src",
1088
- sourceEntityId: "bb22bb22bb22bb22bb22",
1089
- }),
1090
- },
1091
- [["lake_src", "wh_src"]],
1092
- );
1093
- const { instructions } = (
1094
- ctx.service as unknown as {
1095
- deriveSelfInstructions: (
1096
- c: unknown,
1097
- n: string[] | undefined,
1098
- p: unknown,
1099
- ) => { instructions: BuildInstruction[] };
1100
- }
1101
- ).deriveSelfInstructions(compiled, undefined, {});
1102
- const byId = Object.fromEntries(
1103
- instructions.map((i) => [i.sourceEntityId, i]),
1104
- );
1105
- const lake = byId["aa11aa11aa11aa11aa11"];
1106
- const wh = byId["bb22bb22bb22bb22bb22"];
1107
- // storage= → the `name=` value verbatim in the destination (no
1108
- // content-hash decoration); only the destination differs from a colocated build.
1109
- expect(lake.destination).toBe("lake");
1110
- expect(lake.physicalTableName).toBe("daily");
1111
- // colocated → the plain logical (source) name, no destination.
1112
- expect(wh.destination).toBeUndefined();
1113
- expect(wh.physicalTableName).toBe("wh_src");
1114
- } finally {
1115
- delete process.env.PERSIST_STORAGE_MODE;
1116
- }
1117
- });
1118
-
1119
- it("honors the sourceNames filter (excluded sources are neither built nor carried)", () => {
1120
- const compiled = compiledWith(
1121
- {
1122
- s1: fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" }),
1123
- s2: fakeSource({ name: "s2", sourceEntityId: "b2bbbbbbbbbbbbbb" }),
1124
- },
1125
- [["s1", "s2"]],
1126
- );
1127
- const { instructions, carried } = (
1128
- ctx.service as unknown as {
1129
- deriveSelfInstructions: (
1130
- c: unknown,
1131
- n: string[] | undefined,
1132
- p: unknown,
1133
- ) => { instructions: BuildInstruction[]; carried: unknown };
1134
- }
1135
- ).deriveSelfInstructions(compiled, ["s2"], {});
1136
-
1137
- expect(instructions).toHaveLength(1);
1138
- expect(instructions[0].sourceEntityId).toBe("b2bbbbbbbbbbbbbb");
1139
- expect(Object.keys(carried as Record<string, unknown>)).toHaveLength(0);
1140
- });
1141
- });
1142
-
1143
- describe("getMostRecentManifestEntries (skip-if-unchanged)", () => {
1144
- let ctx: ReturnType<typeof createMocks>;
1145
- beforeEach(() => {
1146
- ctx = createMocks();
1147
- });
1148
-
1149
- it("returns entries from the most recent successful run, excluding the in-flight one", async () => {
1150
- const entries = {
1151
- b1: {
1152
- sourceEntityId: "b1",
1153
- physicalTableName: "orders_v1",
1154
- connectionName: "duckdb",
1155
- },
1156
- };
1157
- ctx.repository.listMaterializations.resolves([
1158
- makeMaterialization({
1159
- id: "in-flight",
1160
- status: "MANIFEST_ROWS_READY",
1161
- }),
1162
- makeMaterialization({
1163
- id: "old-success",
1164
- status: "MANIFEST_FILE_READY",
1165
- manifest: { builtAt: "t", strict: false, entries },
1166
- }),
1167
- ]);
1168
-
1169
- const result = await (
1170
- ctx.service as unknown as {
1171
- getMostRecentManifestEntries: (
1172
- e: string,
1173
- p: string,
1174
- x: string,
1175
- ) => Promise<unknown>;
1176
- }
1177
- ).getMostRecentManifestEntries("env-1", "pkg", "in-flight");
1178
-
1179
- expect(result).toEqual(entries);
1180
- });
1181
-
1182
- it("returns {} when the only successful run is the excluded one", async () => {
1183
- ctx.repository.listMaterializations.resolves([
1184
- makeMaterialization({
1185
- id: "self",
1186
- status: "MANIFEST_FILE_READY",
1187
- manifest: {
1188
- builtAt: "t",
1189
- strict: false,
1190
- entries: {
1191
- b1: { sourceEntityId: "b1", physicalTableName: "t1" },
1192
- },
1193
- },
1194
- }),
1195
- ]);
1196
-
1197
- const result = await (
1198
- ctx.service as unknown as {
1199
- getMostRecentManifestEntries: (
1200
- e: string,
1201
- p: string,
1202
- x: string,
1203
- ) => Promise<unknown>;
1204
- }
1205
- ).getMostRecentManifestEntries("env-1", "pkg", "self");
1206
-
1207
- expect(result).toEqual({});
1208
- });
1209
- });
1210
-
1211
- describe("stopMaterialization (in-flight)", () => {
1212
- let ctx: ReturnType<typeof createMocks>;
1213
- beforeEach(() => {
1214
- ctx = createMocks();
1215
- });
1216
-
1217
- it("aborts a running build cooperatively without forcing a transition", async () => {
1218
- ctx.repository.getMaterializationById.resolves(
1219
- makeMaterialization({ status: "MANIFEST_ROWS_READY" }),
1220
- );
1221
- const controller = new AbortController();
1222
- (
1223
- ctx.service as unknown as {
1224
- runningAbortControllers: Map<string, AbortController>;
1225
- }
1226
- ).runningAbortControllers.set("mat-1", controller);
1227
-
1228
- const result = await ctx.service.stopMaterialization(
1229
- "my-env",
1230
- "pkg",
1231
- "mat-1",
1232
- );
1233
-
1234
- expect(controller.signal.aborted).toBe(true);
1235
- // The background run records the terminal state; stop must not also write.
1236
- expect(ctx.repository.updateMaterialization.called).toBe(false);
1237
- expect(result.status).toBe("MANIFEST_ROWS_READY");
1238
- });
1239
- });
1240
-
1241
- describe("executeInstructedBuild", () => {
1242
- let ctx: ReturnType<typeof createMocks>;
1243
- beforeEach(() => {
1244
- ctx = createMocks();
1245
- });
1246
-
1247
- function callExecute(
1248
- compiled: unknown,
1249
- instructions: BuildInstruction[],
1250
- seed: Record<string, unknown>,
1251
- ): Promise<Record<string, { physicalTableName?: string }>> {
1252
- // A stub BuildEnvironment: only the `storage=` branch touches it, and
1253
- // these tests exercise the default in-warehouse path, so it is never read.
1254
- const environment = {
1255
- getApiConnection: () => {
1256
- throw new Error("no connection config in this test");
1257
- },
1258
- getEnvironmentPath: () => "/tmp/env",
1259
- };
1260
- return (
1261
- ctx.service as unknown as {
1262
- executeInstructedBuild: (
1263
- c: unknown,
1264
- env: unknown,
1265
- i: BuildInstruction[],
1266
- s: Record<string, unknown>,
1267
- sig: AbortSignal,
1268
- ) => Promise<Record<string, { physicalTableName?: string }>>;
1269
- }
1270
- ).executeInstructedBuild(
1271
- compiled,
1272
- environment,
1273
- instructions,
1274
- seed,
1275
- new AbortController().signal,
1276
- );
1277
- }
1278
-
1279
- it("builds instructed sources in dependency order and seeds carried entries", async () => {
1280
- const runSQL = sinon.stub().resolves();
1281
- const connection = { runSQL } as unknown as MalloyConnection;
1282
- const s1 = fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" });
1283
- const s2 = fakeSource({ name: "s2", sourceEntityId: "b2bbbbbbbbbbbbbb" });
1284
- const compiled = compiledWith(
1285
- { s1, s2 },
1286
- [["s1"], ["s2"]],
1287
- new Map([["duckdb", connection]]),
1288
- );
1289
-
1290
- const entries = await callExecute(
1291
- compiled,
1292
- [
1293
- {
1294
- sourceEntityId: "b1aaaaaaaaaaaaaa",
1295
- materializedTableId: "mt-1",
1296
- physicalTableName: "s1_v1",
1297
- realization: "COPY",
1298
- },
1299
- {
1300
- sourceEntityId: "b2bbbbbbbbbbbbbb",
1301
- materializedTableId: "mt-2",
1302
- physicalTableName: "s2_v1",
1303
- realization: "COPY",
1304
- },
1305
- ],
1306
- {
1307
- carried0: {
1308
- sourceEntityId: "carried0",
1309
- physicalTableName: "carried_tbl",
1310
- connectionName: "duckdb",
1311
- },
1312
- },
1313
- );
1314
-
1315
- // Both instructed sources built, plus the carried seed entry retained.
1316
- expect(entries["b1aaaaaaaaaaaaaa"].physicalTableName).toBe("s1_v1");
1317
- expect(entries["b2bbbbbbbbbbbbbb"].physicalTableName).toBe("s2_v1");
1318
- expect(entries["carried0"].physicalTableName).toBe("carried_tbl");
1319
- });
1320
-
1321
- it("builds an intermediate persist source nested under a root node", async () => {
1322
- // Mirrors malloy getBuildPlan(): only the terminal `root` is a graph
1323
- // node; its persist dependency `mid` is nested in dependsOn (not a node).
1324
- // Before the iterGraphSources fix, `mid` was silently never built even
1325
- // though the caller instructed it. Both must now build, `mid` first.
1326
- const runSQL = sinon.stub().resolves();
1327
- const connection = { runSQL } as unknown as MalloyConnection;
1328
- const root = fakeSource({
1329
- name: "root",
1330
- sourceEntityId: "brootaaaaaaaaaa",
1331
- });
1332
- const mid = fakeSource({
1333
- name: "mid",
1334
- sourceEntityId: "bmidbbbbbbbbbbb",
1335
- });
1336
- const compiled = {
1337
- graphs: [
1338
- {
1339
- connectionName: "duckdb",
1340
- nodes: [
1341
- [
1342
- {
1343
- sourceID: "root",
1344
- dependsOn: [{ sourceID: "mid", dependsOn: [] }],
1345
- },
1346
- ],
1347
- ],
1348
- },
1349
- ],
1350
- sources: { root, mid },
1351
- connectionDigests: { duckdb: "dig" },
1352
- connections: new Map([["duckdb", connection]]),
1353
- };
1354
-
1355
- const entries = await callExecute(
1356
- compiled,
1357
- [
1358
- {
1359
- sourceEntityId: "brootaaaaaaaaaa",
1360
- materializedTableId: "mt-r",
1361
- physicalTableName: "root_v1",
1362
- realization: "COPY",
1363
- },
1364
- {
1365
- sourceEntityId: "bmidbbbbbbbbbbb",
1366
- materializedTableId: "mt-m",
1367
- physicalTableName: "mid_v1",
1368
- realization: "COPY",
1369
- },
1370
- ],
1371
- {},
1372
- );
1373
-
1374
- expect(entries["bmidbbbbbbbbbbb"].physicalTableName).toBe("mid_v1");
1375
- expect(entries["brootaaaaaaaaaa"].physicalTableName).toBe("root_v1");
1376
- // The dependency's CREATE precedes its dependent's (dependency order).
1377
- const creates = runSQL
1378
- .getCalls()
1379
- .map((c) => c.args[0] as string)
1380
- .filter((s) => s.startsWith("CREATE TABLE"));
1381
- const midIdx = creates.findIndex((s) => s.includes("mid_v1"));
1382
- const rootIdx = creates.findIndex((s) => s.includes("root_v1"));
1383
- expect(midIdx).toBeGreaterThanOrEqual(0);
1384
- expect(rootIdx).toBeGreaterThan(midIdx);
1385
- });
1386
-
1387
- it("seeds a downstream build with the QUOTED upstream reference (case-folding dialect)", async () => {
1388
- // A carried/seeded upstream (built in a prior run or unit, e.g. via
1389
- // referenceManifest) must reach the downstream's SQL generation as the
1390
- // SAME quoted path the builder CREATEd it with; otherwise the downstream
1391
- // CREATE misses the case-preserved table on a case-folding engine
1392
- // (Snowflake). Capture the buildManifest Malloy is handed for the
1393
- // downstream to prove the seed was quoted for its connection's dialect
1394
- // before it becomes a FROM.
1395
- let seen:
1396
- | {
1397
- buildManifest?: {
1398
- entries?: Record<string, { tableName?: string }>;
1399
- };
1400
- }
1401
- | undefined;
1402
- const runSQL = sinon.stub().resolves();
1403
- const connection = {
1404
- runSQL,
1405
- dialectName: "snowflake",
1406
- } as unknown as MalloyConnection;
1407
- const down = fakeSource({
1408
- name: "down",
1409
- sourceEntityId: "bdownaaaaaaaaaa",
1410
- dialectName: "snowflake",
1411
- onGetSQL: (o) => {
1412
- seen = o as typeof seen;
1413
- },
1414
- });
1415
- const compiled = compiledWith(
1416
- { down },
1417
- [["down"]],
1418
- new Map([["duckdb", connection]]),
1419
- );
1420
-
1421
- await callExecute(
1422
- compiled,
1423
- [
1424
- {
1425
- sourceEntityId: "bdownaaaaaaaaaa",
1426
- materializedTableId: "mt-d",
1427
- physicalTableName: "down_v1",
1428
- realization: "COPY",
1429
- },
1430
- ],
1431
- {
1432
- up: {
1433
- sourceEntityId: "up",
1434
- physicalTableName: "schema.orders_base",
1435
- connectionName: "duckdb",
1436
- },
1437
- },
1438
- );
1439
-
1440
- expect(seen?.buildManifest?.entries?.up?.tableName).toBe(
1441
- '"schema"."orders_base"',
1442
- );
1443
- });
1444
-
1445
- it("seeds unquoted when the upstream's connection is absent from the build (older CP / cross-connection)", async () => {
1446
- // No connectionName on the seed (an older control plane) -> bind verbatim,
1447
- // exactly the pre-change behavior. Non-case-folding engines are unaffected;
1448
- // a case-folding engine self-signals by failing the downstream CREATE.
1449
- let seen:
1450
- | {
1451
- buildManifest?: {
1452
- entries?: Record<string, { tableName?: string }>;
1453
- };
1454
- }
1455
- | undefined;
1456
- const runSQL = sinon.stub().resolves();
1457
- const connection = {
1458
- runSQL,
1459
- dialectName: "snowflake",
1460
- } as unknown as MalloyConnection;
1461
- const down = fakeSource({
1462
- name: "down",
1463
- sourceEntityId: "bdownaaaaaaaaaa",
1464
- dialectName: "snowflake",
1465
- onGetSQL: (o) => {
1466
- seen = o as typeof seen;
1467
- },
1468
- });
1469
- const compiled = compiledWith(
1470
- { down },
1471
- [["down"]],
1472
- new Map([["duckdb", connection]]),
1473
- );
1474
-
1475
- await callExecute(
1476
- compiled,
1477
- [
1478
- {
1479
- sourceEntityId: "bdownaaaaaaaaaa",
1480
- materializedTableId: "mt-d",
1481
- physicalTableName: "down_v1",
1482
- realization: "COPY",
1483
- },
1484
- ],
1485
- { up: { sourceEntityId: "up", physicalTableName: "orders_base" } },
1486
- );
1487
-
1488
- expect(seen?.buildManifest?.entries?.up?.tableName).toBe("orders_base");
1489
- });
1490
-
1491
- it("throws when an instructed graph's connection is missing", async () => {
1492
- const s1 = fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" });
1493
- const compiled = compiledWith({ s1 }, [["s1"]], new Map());
1494
-
1495
- await expect(
1496
- callExecute(
1497
- compiled,
1498
- [
1499
- {
1500
- sourceEntityId: "b1aaaaaaaaaaaaaa",
1501
- materializedTableId: "mt-1",
1502
- physicalTableName: "s1_v1",
1503
- realization: "COPY",
1504
- },
1505
- ],
1506
- {},
1507
- ),
1508
- ).rejects.toThrow(BadRequestError);
1509
- });
1510
- });
1511
-
1512
- describe("buildOneSource", () => {
1513
- let ctx: ReturnType<typeof createMocks>;
1514
- withQueryMetadataOn();
1515
- beforeEach(() => {
1516
- ctx = createMocks();
1517
- });
1518
-
1519
- function callBuildOneSource(
1520
- connection: { runSQL: sinon.SinonStub },
1521
- physicalTableName: string,
1522
- dialectName?: string,
1523
- manifest: Manifest = new Manifest(),
1524
- metadata?: {
1525
- source?: Record<string, string>;
1526
- package?: Record<string, string>;
1527
- connection?: Record<string, string>;
1528
- context?: Record<string, string | undefined>;
1529
- },
1530
- ): Promise<{ sourceEntityId: string; physicalTableName: string }> {
1531
- const source = fakeSource({
1532
- name: "orders",
1533
- sourceEntityId: "abcdef1234567890",
1534
- sql: "SELECT * FROM t",
1535
- dialectName,
1536
- queryMetadata: metadata?.source,
1537
- });
1538
- const instruction: BuildInstruction = {
1539
- sourceEntityId: "abcdef1234567890",
1540
- materializedTableId: "mt-1",
1541
- physicalTableName,
1542
- realization: "COPY",
1543
- };
1544
- return (
1545
- ctx.service as unknown as {
1546
- buildOneSource: (
1547
- s: unknown,
1548
- i: BuildInstruction,
1549
- c: unknown,
1550
- d: Record<string, string>,
1551
- m: Manifest,
1552
- env: unknown,
1553
- built: Record<string, unknown>,
1554
- buildMetadata?: unknown,
1555
- ) => Promise<{ sourceEntityId: string; physicalTableName: string }>;
1556
- }
1557
- ).buildOneSource(
1558
- source,
1559
- instruction,
1560
- connection,
1561
- { duckdb: "dig" },
1562
- manifest,
1563
- {
1564
- getApiConnection: () => {
1565
- if (metadata?.connection) {
1566
- return { queryMetadata: metadata.connection };
1567
- }
1568
- throw new Error("no connection config in this colocated test");
1569
- },
1570
- getEnvironmentPath: () => "/tmp/env",
1571
- },
1572
- {}, // builtEntries — colocated build with no prior storage upstreams
1573
- metadata
1574
- ? {
1575
- packageMaterialization: metadata.package
1576
- ? {
1577
- schedule: null,
1578
- freshness: null,
1579
- queryMetadata: metadata.package,
1580
- }
1581
- : null,
1582
- context: metadata.context ?? { queryClass: "materialize" },
1583
- }
1584
- : undefined,
1585
- );
1586
- }
1587
-
1588
- it("tags every statement of the build with the resolved metadata", async () => {
1589
- // One unit of work: the warehouse's query history has to show the staging
1590
- // drop, the CTAS, the swap and the rename as the same build.
1591
- const runSQL = sinon.stub().resolves();
1592
- await callBuildOneSource({ runSQL }, "orders_v1", undefined, undefined, {
1593
- connection: { deployment: "eu" },
1594
- package: { team: "finance", tier: "bronze" },
1595
- source: { tier: "gold" },
1596
- context: { queryClass: "materialize", runId: "run-7" },
1597
- });
1598
-
1599
- const bags = runSQL.getCalls().map((c) => c.args[1]?.queryMetadata);
1600
- expect(bags).toHaveLength(4);
1601
- for (const bag of bags) {
1602
- expect(bag).toEqual({
1603
- deployment: "eu",
1604
- team: "finance",
1605
- // The source's own declaration wins over the package's.
1606
- tier: "gold",
1607
- class: "materialize",
1608
- run_id: "run-7",
1609
- source: "orders",
1610
- });
1611
- }
1612
- });
1613
-
1614
- it("issues untagged statements when nothing declares metadata", async () => {
1615
- // No metadata context (an internal caller) means the statements are exactly
1616
- // what they were before this existed.
1617
- const runSQL = sinon.stub().resolves();
1618
- await callBuildOneSource({ runSQL }, "orders_v1");
1619
- for (const call of runSQL.getCalls()) {
1620
- expect(call.args[1]?.queryMetadata).toBeUndefined();
1621
- }
1622
- });
1623
-
1624
- it("tags the cleanup drop after a failed build", async () => {
1625
- // The cleanup statement belongs to the same build, and it is the one an
1626
- // operator goes looking for after a failure.
1627
- const runSQL = sinon.stub();
1628
- runSQL.onCall(0).resolves();
1629
- runSQL.onCall(1).rejects(new Error("create boom"));
1630
- runSQL.onCall(2).resolves();
1631
- await expect(
1632
- callBuildOneSource({ runSQL }, "orders_v1", undefined, undefined, {
1633
- source: { team: "finance" },
1634
- context: { queryClass: "materialize" },
1635
- }),
1636
- ).rejects.toThrow("create boom");
1637
- expect(runSQL.lastCall.args[1]?.queryMetadata).toEqual({
1638
- team: "finance",
1639
- class: "materialize",
1640
- source: "orders",
1641
- });
1642
- });
1643
-
1644
- it("stages, swaps, and renames in a crash-safe order", async () => {
1645
- const runSQL = sinon.stub().resolves();
1646
- const entry = await callBuildOneSource({ runSQL }, "orders_v1");
1647
-
1648
- const sql = runSQL.getCalls().map((c) => c.args[0] as string);
1649
- expect(sql).toEqual([
1650
- 'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
1651
- 'CREATE TABLE "orders_v1_abcdef123456" AS (SELECT * FROM t)',
1652
- 'DROP TABLE IF EXISTS "orders_v1"',
1653
- 'ALTER TABLE "orders_v1_abcdef123456" RENAME TO "orders_v1"',
1654
- ]);
1655
- expect(entry.physicalTableName).toBe("orders_v1");
1656
- expect(entry.sourceEntityId).toBe("abcdef1234567890");
1657
- });
1658
-
1659
- it("drops the staging table and rethrows when the build SQL fails", async () => {
1660
- const runSQL = sinon.stub();
1661
- runSQL.onCall(0).resolves(); // initial staging drop
1662
- runSQL.onCall(1).rejects(new Error("create boom")); // CREATE TABLE AS
1663
- runSQL.onCall(2).resolves(); // cleanup staging drop
1664
- await expect(callBuildOneSource({ runSQL }, "orders_v1")).rejects.toThrow(
1665
- "create boom",
1666
- );
1667
- expect(runSQL.lastCall.args[0]).toBe(
1668
- 'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
1669
- );
1670
- });
1671
-
1672
- it("rethrows the original build error when staging cleanup also fails", async () => {
1673
- const runSQL = sinon.stub();
1674
- runSQL.onCall(0).resolves(); // initial staging drop
1675
- runSQL.onCall(1).rejects(new Error("create boom")); // CREATE TABLE AS
1676
- runSQL.onCall(2).rejects(new Error("cleanup boom")); // cleanup drop fails
1677
- // The cleanup failure is swallowed (logged as a physical leak); the
1678
- // original build error is what propagates, so the run is classified by its
1679
- // real cause rather than the cleanup noise.
1680
- await expect(callBuildOneSource({ runSQL }, "orders_v1")).rejects.toThrow(
1681
- "create boom",
1682
- );
1683
- expect(runSQL.callCount).toBe(3);
1684
- expect(runSQL.lastCall.args[0]).toBe(
1685
- 'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
1686
- );
1687
- });
1688
-
1689
- it("drops staging when the build is cancelled mid-run (no _staging orphan)", async () => {
1690
- // buildOneSource is cancellation-agnostic: a cancel surfaces as a thrown
1691
- // build SQL and takes the same staging-cleanup path as a failure, so a
1692
- // cancelled build leaves no orphaned _staging table behind.
1693
- const runSQL = sinon.stub();
1694
- runSQL.onCall(0).resolves(); // initial staging drop
1695
- runSQL.onCall(1).rejects(new Error("Build cancelled")); // aborted mid-CREATE
1696
- runSQL.onCall(2).resolves(); // cleanup staging drop
1697
- await expect(callBuildOneSource({ runSQL }, "orders_v1")).rejects.toThrow(
1698
- "Build cancelled",
1699
- );
1700
- expect(runSQL.lastCall.args[0]).toBe(
1701
- 'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
1702
- );
1703
- });
1704
-
1705
- it("quotes each segment of a container path for a backtick dialect", async () => {
1706
- const runSQL = sinon.stub().resolves();
1707
- // Container path (dataset.table) on a backtick dialect (BigQuery): each
1708
- // segment is quoted independently, and the rename targets the bare name.
1709
- const entry = await callBuildOneSource(
1710
- { runSQL },
1711
- "ds.orders_v1",
1712
- "standardsql",
1713
- );
1714
-
1715
- const sql = runSQL.getCalls().map((c) => c.args[0] as string);
1716
- expect(sql).toEqual([
1717
- "DROP TABLE IF EXISTS `ds`.`orders_v1_abcdef123456`",
1718
- "CREATE TABLE `ds`.`orders_v1_abcdef123456` AS (SELECT * FROM t)",
1719
- "DROP TABLE IF EXISTS `ds`.`orders_v1`",
1720
- "ALTER TABLE `ds`.`orders_v1_abcdef123456` RENAME TO `orders_v1`",
1721
- ]);
1722
- expect(entry.physicalTableName).toBe("ds.orders_v1");
1723
- });
1724
-
1725
- it("colocated build excludes a storage upstream from its warehouse manifest (mixed chain)", async () => {
1726
- // A colocated downstream that reads a storage-materialized
1727
- // upstream must NOT get the lake table name in its warehouse build SQL —
1728
- // the warehouse can't resolve it. The storage upstream is excluded (⇒
1729
- // inlined/recomputed); a colocated upstream is kept.
1730
- const runSQL = sinon.stub().resolves();
1731
- let seenManifest:
1732
- | { entries?: Record<string, { tableName?: string }> }
1733
- | undefined;
1734
- const down = fakeSource({
1735
- name: "down",
1736
- sourceEntityId: "downdowndowndown",
1737
- sql: "SELECT 1",
1738
- onGetSQL: (o) => {
1739
- seenManifest = (o as { buildManifest?: typeof seenManifest })
1740
- .buildManifest;
1741
- },
1742
- });
1743
- const manifest = new Manifest();
1744
- manifest.update("up_storage", { tableName: "daily__mabc" });
1745
- manifest.update("up_pathc", { tableName: '"orders_v1"' });
1746
- const builtEntries = {
1747
- up_storage: {
1748
- sourceEntityId: "up_storage",
1749
- physicalTableName: "daily__mabc",
1750
- storageConnectionName: "lake",
1751
- },
1752
- up_pathc: {
1753
- sourceEntityId: "up_pathc",
1754
- physicalTableName: "orders_v1",
1755
- connectionName: "duckdb",
1756
- },
1757
- };
1758
- await (
1759
- ctx.service as unknown as {
1760
- buildOneSource: (...a: unknown[]) => Promise<unknown>;
1761
- }
1762
- ).buildOneSource(
1763
- down,
1764
- {
1765
- sourceEntityId: "downdowndowndown",
1766
- materializedTableId: "mt",
1767
- physicalTableName: "down_v1",
1768
- realization: "COPY",
1769
- }, // no destination ⇒ colocated
1770
- { runSQL },
1771
- { duckdb: "dig" },
1772
- manifest,
1773
- {
1774
- getApiConnection: () => {
1775
- throw new Error("unused in the colocated build");
1776
- },
1777
- getEnvironmentPath: () => "/tmp/env",
1778
- },
1779
- builtEntries,
1780
- );
1781
- expect(seenManifest?.entries?.up_storage).toBeUndefined();
1782
- expect(seenManifest?.entries?.up_pathc?.tableName).toBe('"orders_v1"');
1783
- });
1784
-
1785
- it("records the QUOTED just-built name in the build manifest (matches the CREATE)", async () => {
1786
- // The build manifest feeds a downstream persist's FROM verbatim, so the
1787
- // recorded name must be the SAME quoted path the CREATE used — else a
1788
- // downstream built later in this run misses the case-preserved table on a
1789
- // case-folding engine. buildManifest also passes through Malloy's canonical
1790
- // path validation (requireCanonicalTablePathAnyDialect), so this proves the
1791
- // quoted form is accepted, not just produced.
1792
- const runSQL = sinon.stub().resolves();
1793
- const manifest = new Manifest();
1794
- await callBuildOneSource(
1795
- { runSQL },
1796
- "schema.orders_base",
1797
- "snowflake",
1798
- manifest,
1799
- );
1800
- expect(manifest.buildManifest.entries["abcdef1234567890"].tableName).toBe(
1801
- '"schema"."orders_base"',
1802
- );
1803
- // The logical (unquoted) name is what the CREATE was told to produce, and
1804
- // is exactly what quoteTablePath quoted for the DDL — read mirrors write.
1805
- const created = runSQL
1806
- .getCalls()
1807
- .map((c) => c.args[0] as string)
1808
- .find((s) => s.startsWith("CREATE TABLE"))!;
1809
- expect(created).toContain('"schema"."orders_base_abcdef123456"');
1810
- });
1811
- });
1812
-
1813
- describe("buildOneSourceIntoStorage (chained-build fallback ladder)", () => {
1814
- let ctx: ReturnType<typeof createMocks>;
1815
- beforeEach(() => {
1816
- ctx = createMocks();
1817
- });
1818
-
1819
- function callInto(opts: {
1820
- strict: boolean;
1821
- stackOnParent: "ok" | "throw" | "infra";
1822
- }): Promise<{
1823
- physicalTableName: string;
1824
- storageConnectionName?: string;
1825
- }> {
1826
- const source = fakeSource({
1827
- name: "monthly",
1828
- sourceEntityId: "abcdef1234567890",
1829
- annotationFields: { storage: "lake" },
1830
- });
1831
- const instruction: BuildInstruction = {
1832
- sourceEntityId: "abcdef1234567890",
1833
- materializedTableId: "mt",
1834
- physicalTableName: "monthly__mabc",
1835
- realization: "COPY",
1836
- destination: "lake",
1837
- };
1838
- const manifest = new Manifest();
1839
- manifest.strict = opts.strict;
1840
- const environment = {
1841
- getApiConnection: (n: string) => ({
1842
- name: n,
1843
- type: n === "lake" ? "duckdb" : "postgres",
1844
- }),
1845
- getEnvironmentPath: () => "/tmp/env",
1846
- };
1847
- const svc = ctx.service as unknown as {
1848
- buildDownstreamViaParents: unknown;
1849
- buildOneSourceIntoStorage: (
1850
- s: unknown,
1851
- i: BuildInstruction,
1852
- m: Manifest,
1853
- e: unknown,
1854
- sql: string,
1855
- built: Record<string, unknown>,
1856
- dep: boolean,
1857
- ) => Promise<{
1858
- physicalTableName: string;
1859
- storageConnectionName?: string;
1860
- }>;
1861
- };
1862
- // Stub the stack-on-parent seam so the test exercises the LADDER, not the build.
1863
- svc.buildDownstreamViaParents =
1864
- opts.stackOnParent === "ok"
1865
- ? sinon.stub().resolves({
1866
- storageConnectionName: "lake",
1867
- schema: [
1868
- { name: "order_month", type: "DATE" },
1869
- { name: "monthly_total", type: "DOUBLE" },
1870
- ],
1871
- })
1872
- : opts.stackOnParent === "infra"
1873
- ? // An outage, not a modelling limit: a plain Error, as a failed
1874
- // ATTACH or CTAS would throw.
1875
- sinon.stub().rejects(new Error("IO Error: destination is down"))
1876
- : // A shape limit: the downstream cannot be expressed over its
1877
- // rebound parents.
1878
- sinon.stub().rejects(
1879
- new MaterializationEligibilityError({
1880
- message: "uncarried parent",
1881
- }),
1882
- );
1883
- return svc.buildOneSourceIntoStorage(
1884
- source,
1885
- instruction,
1886
- manifest,
1887
- environment,
1888
- "SELECT should_not_run",
1889
- {
1890
- up: {
1891
- sourceEntityId: "up",
1892
- sourceName: "daily",
1893
- physicalTableName: "daily__mabc",
1894
- storageConnectionName: "lake",
1895
- schema: [{ name: "order_date", type: "DATE" }],
1896
- },
1897
- },
1898
- true, // dependsOnStorageUpstream
1899
- );
1900
- }
1901
-
1902
- it("stacks on the parent: builds by reading it and returns the storage entry", async () => {
1903
- const entry = await callInto({ strict: false, stackOnParent: "ok" });
1904
- expect(entry.storageConnectionName).toBe("lake");
1905
- expect(entry.physicalTableName).toBe("monthly__mabc");
1906
- });
1907
-
1908
- it("strict + cannot stack on the parent: refuses loudly instead of recomputing from raw", async () => {
1909
- await expect(
1910
- callInto({ strict: true, stackOnParent: "throw" }),
1911
- ).rejects.toThrow(/strict upstreams forbid/i);
1912
- });
1913
-
1914
- it("non-strict + an INFRA failure fails rather than recomputing from raw", async () => {
1915
- // Recompute-from-raw writes to the same destination, so retrying an outage
1916
- // there just fails again — and metering it as a fallback files the outage
1917
- // under the same label as a legitimate shape miss. Only a shape failure is
1918
- // a reason to try the other path.
1919
- await expect(
1920
- callInto({ strict: false, stackOnParent: "infra" }),
1921
- ).rejects.toThrow(/Failed to materialize chained source/i);
1922
- });
1923
-
1924
- it("non-strict + a SHAPE failure falls through to recompute-from-raw", async () => {
1925
- // Only the parent-reuse seam is stubbed, so the recompute runs for real and
1926
- // fails for its own reason (no destination file). That is the proof it was
1927
- // reached: the two paths report differently — the chained path says
1928
- // "chained source", the single-source recompute says "source". A shape
1929
- // failure must reach the second; an infra failure must not.
1930
- await expect(
1931
- callInto({ strict: false, stackOnParent: "throw" }),
1932
- ).rejects.toThrow(/Failed to materialize source 'monthly'/i);
1933
- });
1934
- });
1935
-
1936
- describe("runBuild (branch behavior)", () => {
1937
- let ctx: ReturnType<typeof createMocks>;
1938
- beforeEach(() => {
1939
- ctx = createMocks();
1940
- });
1941
-
1942
- // Exercise runBuild's orchestration in isolation: the build engine and
1943
- // manifest commit are stubbed so the test asserts which instructions flow
1944
- // through and whether the manifest is auto-loaded, not how a source is built.
1945
- type RunBuildInternals = {
1946
- executeInstructedBuild: sinon.SinonStub;
1947
- commitManifest: sinon.SinonStub;
1948
- autoLoadManifest: sinon.SinonStub;
1949
- runBuild: (
1950
- id: string,
1951
- env: string,
1952
- pkg: string,
1953
- opts: {
1954
- sourceNames: string[] | undefined;
1955
- forceRefresh: boolean;
1956
- buildInstructions: BuildInstruction[] | undefined;
1957
- referenceManifest?:
1958
- | {
1959
- sourceEntityId: string;
1960
- physicalTableName: string;
1961
- connectionName?: string;
1962
- }[]
1963
- | undefined;
1964
- strictUpstreams?: boolean | undefined;
1965
- },
1966
- signal: AbortSignal,
1967
- ) => Promise<void>;
1968
- };
1969
-
1970
- function stubEngine(): RunBuildInternals {
1971
- const entries = {
1972
- "build-orders": {
1973
- sourceEntityId: "build-orders",
1974
- physicalTableName: '"orders_v1"',
1975
- connectionName: "duckdb",
1976
- },
1977
- };
1978
- const svc = ctx.service as unknown as RunBuildInternals;
1979
- svc.executeInstructedBuild = sinon.stub().resolves(entries);
1980
- svc.commitManifest = sinon.stub().resolves();
1981
- svc.autoLoadManifest = sinon.stub().resolves();
1982
- return svc;
1983
- }
1984
-
1985
- it("orchestrated: builds caller instructions and does not auto-load", async () => {
1986
- const svc = stubEngine();
1987
- const instructions = [makeInstruction()];
1988
-
1989
- await svc.runBuild(
1990
- "mat-1",
1991
- "my-env",
1992
- "pkg",
1993
- {
1994
- sourceNames: undefined,
1995
- forceRefresh: false,
1996
- buildInstructions: instructions,
1997
- },
1998
- new AbortController().signal,
1999
- );
2000
-
2001
- expect(svc.executeInstructedBuild.calledOnce).toBe(true);
2002
- // Caller instructions pass straight through; nothing is carried/reused.
2003
- // (arg[1] is the environment; instructions are arg[2], seed entries arg[3].)
2004
- expect(svc.executeInstructedBuild.firstCall.args[2]).toEqual(
2005
- instructions,
2006
- );
2007
- expect(svc.executeInstructedBuild.firstCall.args[3]).toEqual({});
2008
- expect(svc.commitManifest.firstCall.args[2]).toMatchObject({
2009
- mode: "orchestrated",
2010
- sourcesBuilt: 1,
2011
- sourcesReused: 0,
2012
- });
2013
- // Orchestrated leaves distribution to the caller.
2014
- expect(svc.autoLoadManifest.called).toBe(false);
2015
- });
2016
-
2017
- it("orchestrated: seeds the build from referenceManifest and honors strictUpstreams", async () => {
2018
- const svc = stubEngine();
2019
- const instructions = [makeInstruction()];
2020
-
2021
- await svc.runBuild(
2022
- "mat-1",
2023
- "my-env",
2024
- "pkg",
2025
- {
2026
- sourceNames: undefined,
2027
- forceRefresh: false,
2028
- buildInstructions: instructions,
2029
- referenceManifest: [
2030
- {
2031
- sourceEntityId: "up-1",
2032
- physicalTableName: "upstream_tbl",
2033
- connectionName: "sf",
2034
- },
2035
- ],
2036
- strictUpstreams: true,
2037
- },
2038
- new AbortController().signal,
2039
- );
2040
-
2041
- // The reference manifest becomes the seed (carried) entries so a
2042
- // downstream build resolves its upstream to the existing physical table.
2043
- // arg[1] is environment (the storage build session), so the seed
2044
- // (carried) entries are arg[3]. connectionName is carried through so the
2045
- // seed can be quoted for the upstream's dialect (see quoteSeedTablePath).
2046
- expect(svc.executeInstructedBuild.firstCall.args[3]).toEqual({
2047
- "up-1": {
2048
- sourceEntityId: "up-1",
2049
- physicalTableName: "upstream_tbl",
2050
- connectionName: "sf",
2051
- },
2052
- });
2053
- // strictUpstreams flows through as the strict flag (6th arg).
2054
- expect(svc.executeInstructedBuild.firstCall.args[5]).toBe(true);
2055
- // Reused upstreams are counted as carried, not built.
2056
- expect(svc.commitManifest.firstCall.args[2]).toMatchObject({
2057
- mode: "orchestrated",
2058
- sourcesBuilt: 1,
2059
- sourcesReused: 1,
2060
- });
2061
- });
2062
-
2063
- it("auto-run: derives instructions and auto-loads the manifest", async () => {
2064
- const svc = stubEngine();
2065
-
2066
- await svc.runBuild(
2067
- "mat-1",
2068
- "my-env",
2069
- "pkg",
2070
- {
2071
- sourceNames: undefined,
2072
- forceRefresh: true,
2073
- buildInstructions: undefined,
2074
- },
2075
- new AbortController().signal,
2076
- );
2077
-
2078
- expect(svc.commitManifest.firstCall.args[2]).toMatchObject({
2079
- mode: "auto",
2080
- });
2081
- // Auto-run owns distribution: it loads the fresh manifest into the models.
2082
- expect(svc.autoLoadManifest.calledOnce).toBe(true);
2083
- });
2084
- });
2085
-
2086
- describe("autoLoadManifest", () => {
2087
- let ctx: ReturnType<typeof createMocks>;
2088
- beforeEach(() => {
2089
- ctx = createMocks();
2090
- });
2091
-
2092
- it("loads only entries with a physical table into the package models", async () => {
2093
- const reloadAllModelsForPackage = sinon.stub().resolves();
2094
- const entries = {
2095
- b1: {
2096
- sourceEntityId: "b1",
2097
- physicalTableName: "orders_v1",
2098
- connectionName: "duckdb",
2099
- },
2100
- b2: { sourceEntityId: "b2", physicalTableName: undefined },
2101
- };
2102
-
2103
- await (
2104
- ctx.service as unknown as {
2105
- autoLoadManifest: (
2106
- env: { reloadAllModelsForPackage: sinon.SinonStub },
2107
- pkg: string,
2108
- entries: Record<string, unknown>,
2109
- ) => Promise<void>;
2110
- }
2111
- ).autoLoadManifest({ reloadAllModelsForPackage }, "pkg", entries);
2112
-
2113
- expect(reloadAllModelsForPackage.calledOnce).toBe(true);
2114
- // connectionName is carried alongside tableName so the bind step
2115
- // (Package.quoteBoundTableNames) can quote the path for its dialect.
2116
- expect(reloadAllModelsForPackage.firstCall.args).toEqual([
2117
- "pkg",
2118
- { b1: { tableName: "orders_v1", connectionName: "duckdb" } },
2119
- ]);
2120
- });
2121
-
2122
- it("swallows a reload failure (best-effort; run already succeeded)", async () => {
2123
- const reloadAllModelsForPackage = sinon
2124
- .stub()
2125
- .rejects(new Error("reload boom"));
2126
- await expect(
2127
- (
2128
- ctx.service as unknown as {
2129
- autoLoadManifest: (
2130
- env: { reloadAllModelsForPackage: sinon.SinonStub },
2131
- pkg: string,
2132
- entries: Record<string, unknown>,
2133
- ) => Promise<void>;
2134
- }
2135
- ).autoLoadManifest({ reloadAllModelsForPackage }, "pkg", {
2136
- b1: { sourceEntityId: "b1", physicalTableName: "t" },
2137
- }),
2138
- ).resolves.toBeUndefined();
2139
- });
2140
- });
2141
-
2142
- describe("runInBackground (terminal recording)", () => {
2143
- let ctx: ReturnType<typeof createMocks>;
2144
- beforeEach(() => {
2145
- ctx = createMocks();
2146
- });
2147
-
2148
- const flush = () => new Promise((r) => setTimeout(r, 20));
2149
-
2150
- function background(): {
2151
- runInBackground: (
2152
- id: string,
2153
- run: (signal: AbortSignal) => Promise<void>,
2154
- ) => void;
2155
- runningAbortControllers: Map<string, AbortController>;
2156
- } {
2157
- return ctx.service as unknown as {
2158
- runInBackground: (
2159
- id: string,
2160
- run: (signal: AbortSignal) => Promise<void>,
2161
- ) => void;
2162
- runningAbortControllers: Map<string, AbortController>;
2163
- };
2164
- }
2165
-
2166
- it("records FAILED with the error message when the run rejects", async () => {
2167
- background().runInBackground("bg-1", async () => {
2168
- throw new Error("boom");
2169
- });
2170
- await flush();
2171
-
2172
- expect(ctx.repository.updateMaterialization.calledOnce).toBe(true);
2173
- expect(ctx.repository.updateMaterialization.firstCall.args[0]).toBe(
2174
- "bg-1",
2175
- );
2176
- expect(
2177
- ctx.repository.updateMaterialization.firstCall.args[1],
2178
- ).toMatchObject({ status: "FAILED", error: "boom" });
2179
- });
2180
-
2181
- it("records CANCELLED when the run aborts cooperatively", async () => {
2182
- const bg = background();
2183
- bg.runInBackground("bg-2", async () => {
2184
- bg.runningAbortControllers.get("bg-2")!.abort();
2185
- throw new Error("boom");
2186
- });
2187
- await flush();
2188
-
2189
- expect(
2190
- ctx.repository.updateMaterialization.firstCall.args[1],
2191
- ).toMatchObject({ status: "CANCELLED", error: "Cancelled" });
2192
- });
2193
-
2194
- it("clears the abort controller once the run settles", async () => {
2195
- const bg = background();
2196
- bg.runInBackground("bg-3", async () => {});
2197
- await flush();
2198
- expect(bg.runningAbortControllers.has("bg-3")).toBe(false);
2199
- });
2200
- });
2201
-
2202
- describe("transition (state machine)", () => {
2203
- let ctx: ReturnType<typeof createMocks>;
2204
- beforeEach(() => {
2205
- ctx = createMocks();
2206
- });
2207
-
2208
- function transition(): (
2209
- id: string,
2210
- next: MaterializationStatus,
2211
- ) => Promise<unknown> {
2212
- return (
2213
- ctx.service as unknown as {
2214
- transition: (
2215
- id: string,
2216
- next: MaterializationStatus,
2217
- ) => Promise<unknown>;
2218
- }
2219
- ).transition.bind(ctx.service);
2220
- }
2221
-
2222
- it("throws MaterializationNotFoundError when the record is gone", async () => {
2223
- ctx.repository.getMaterializationById.resolves(undefined);
2224
- await expect(
2225
- transition()("mat-1", "MANIFEST_ROWS_READY"),
2226
- ).rejects.toThrow(MaterializationNotFoundError);
2227
- });
2228
-
2229
- it("rejects a transition the state machine disallows", async () => {
2230
- ctx.repository.getMaterializationById.resolves(
2231
- makeMaterialization({ status: "MANIFEST_FILE_READY" }),
2232
- );
2233
- // MANIFEST_FILE_READY is terminal; nothing follows it.
2234
- await expect(transition()("mat-1", "PENDING")).rejects.toThrow(
2235
- InvalidStateTransitionError,
2236
- );
2237
- expect(ctx.repository.updateMaterialization.called).toBe(false);
2238
- });
2239
-
2240
- it("persists a valid transition", async () => {
2241
- ctx.repository.getMaterializationById.resolves(
2242
- makeMaterialization({ status: "PENDING" }),
2243
- );
2244
- ctx.repository.updateMaterialization.resolves(
2245
- makeMaterialization({ status: "MANIFEST_ROWS_READY" }),
2246
- );
2247
- await transition()("mat-1", "MANIFEST_ROWS_READY");
2248
- expect(
2249
- ctx.repository.updateMaterialization.calledOnceWith("mat-1", {
2250
- status: "MANIFEST_ROWS_READY",
2251
- }),
2252
- ).toBe(true);
2253
- });
2254
- });
2255
-
2256
- // The only path here that issues a DROP against a customer's warehouse, so its
2257
- // guards get assertions rather than a trace. Scenario 64 covers it end to end,
2258
- // but hammer does not run in CI — and the failure mode is silent data loss, not
2259
- // a broken build.
2260
- describe("reclaimStorageTablesFromFailedRun", () => {
2261
- let ctx: ReturnType<typeof createMocks>;
2262
- let infoLog: sinon.SinonStub;
2263
- let warnLog: sinon.SinonStub;
2264
-
2265
- beforeEach(() => {
2266
- ctx = createMocks();
2267
- infoLog = ctx.sandbox.stub(logger, "info");
2268
- warnLog = ctx.sandbox.stub(logger, "warn");
2269
- });
2270
- afterEach(() => ctx.sandbox.restore());
2271
-
2272
- const entry = (table: string) =>
2273
- ({
2274
- sourceEntityId: `eid-${table}`,
2275
- sourceName: "daily",
2276
- physicalTableName: table,
2277
- storageConnectionName: "lake",
2278
- }) as unknown as Parameters<
2279
- MaterializationService["reclaimStorageTablesFromFailedRun"]
2280
- >[0][number];
2281
-
2282
- /** A destination whose drop would fail loudly if one were ever attempted. */
2283
- const environment = {
2284
- getApiConnection: () => ({ name: "lake", type: "duckdb" }),
2285
- getEnvironmentPath: () => "/test",
2286
- } as unknown as Parameters<
2287
- MaterializationService["reclaimStorageTablesFromFailedRun"]
2288
- >[1];
2289
-
2290
- const reclaim = (
2291
- built: ReturnType<typeof entry>[],
2292
- others: unknown[],
2293
- ): Promise<void> => {
2294
- (ctx.repository.listMaterializations as sinon.SinonStub).resolves(others);
2295
- return (
2296
- ctx.service as unknown as {
2297
- reclaimStorageTablesFromFailedRun: (
2298
- b: unknown,
2299
- e: unknown,
2300
- o: unknown,
2301
- ) => Promise<void>;
2302
- }
2303
- ).reclaimStorageTablesFromFailedRun(built, environment, {
2304
- environmentId: "env-1",
2305
- packageName: "pkg",
2306
- });
2307
- };
2308
-
2309
- const said = (stub: sinon.SinonStub, needle: RegExp) =>
2310
- stub.getCalls().some((c) => needle.test(String(c.args[0])));
2311
-
2312
- it("never drops a table a live manifest still serves", async () => {
2313
- // The carried-forward case: a previous successful run's manifest names the
2314
- // same destination + table. Dropping it would delete a table that is being
2315
- // served right now.
2316
- await reclaim(
2317
- [entry("daily_v1")],
2318
- [
2319
- {
2320
- status: "MANIFEST_FILE_READY",
2321
- manifest: {
2322
- entries: {
2323
- other: {
2324
- storageConnectionName: "lake",
2325
- physicalTableName: "daily_v1",
2326
- },
2327
- },
2328
- },
2329
- },
2330
- ],
2331
- );
2332
-
2333
- expect(said(infoLog, /Keeping a table from a failed run/)).toBe(true);
2334
- expect(said(infoLog, /Reclaimed a table/)).toBe(false);
2335
- expect(said(warnLog, /Failed to reclaim/)).toBe(false);
2336
- });
2337
-
2338
- it("does attempt the drop when nothing references the table", async () => {
2339
- // The inverse, so the test above is known to be the still-referenced check
2340
- // doing the work rather than the reclaim being inert. No real destination
2341
- // exists here, so the attempt surfaces as the drop-failure warning.
2342
- await reclaim([entry("daily_v2")], []);
2343
-
2344
- expect(said(infoLog, /Keeping a table from a failed run/)).toBe(false);
2345
- expect(
2346
- said(warnLog, /Failed to reclaim/) ||
2347
- said(infoLog, /Reclaimed a table/),
2348
- ).toBe(true);
2349
- });
2350
-
2351
- it("ignores a superseded run's entries when deciding", async () => {
2352
- // Only MANIFEST_FILE_READY runs count as live. A superseded run naming the
2353
- // same table must not pin it forever.
2354
- await reclaim(
2355
- [entry("daily_v3")],
2356
- [
2357
- {
2358
- status: "SUPERSEDED",
2359
- manifest: {
2360
- entries: {
2361
- old: {
2362
- storageConnectionName: "lake",
2363
- physicalTableName: "daily_v3",
2364
- },
2365
- },
2366
- },
2367
- },
2368
- ],
2369
- );
2370
-
2371
- expect(said(infoLog, /Keeping a table from a failed run/)).toBe(false);
2372
- });
2373
- });