@malloy-publisher/server 0.0.234 → 0.0.236

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (318) hide show
  1. package/dist/app/api-doc.yaml +47 -17
  2. package/dist/app/assets/{EnvironmentPage-DTZQ4Gxc.js → EnvironmentPage-BsAnavYN.js} +1 -1
  3. package/dist/app/assets/{HomePage-C5mlDPXK.js → HomePage-CADE138j.js} +1 -1
  4. package/dist/app/assets/{LightMode-DGNmhG0u.js → LightMode-Cfh7KzN8.js} +1 -1
  5. package/dist/app/assets/{MainPage-CVL_wmP4.js → MainPage-CO3pRlnV.js} +1 -1
  6. package/dist/app/assets/{MaterializationsPage-DmzMBCpy.js → MaterializationsPage-p9YjkRXZ.js} +1 -1
  7. package/dist/app/assets/{ModelPage-Dbvf4QbB.js → ModelPage-C1OSTv-x.js} +1 -1
  8. package/dist/app/assets/{PackagePage-DxdHc2Qs.js → PackagePage-e4kN75YR.js} +1 -1
  9. package/dist/app/assets/{RouteError-OJdT4tCd.js → RouteError-CzbfOkng.js} +1 -1
  10. package/dist/app/assets/{ThemeEditorPage-Bk7s0KXY.js → ThemeEditorPage-CciagFTq.js} +1 -1
  11. package/dist/app/assets/{WorkbookPage-j_vCWdN3.js → WorkbookPage-DNWmkCXa.js} +1 -1
  12. package/dist/app/assets/{core-Rj_4rRnA.es-DoIfLxDJ.js → core-Rj_4rRnA.es-BZyvITuO.js} +1 -1
  13. package/dist/app/assets/{index-B_jKMR35.js → index-CH2AcDzc.js} +4 -4
  14. package/dist/app/assets/{index-hw-xn0X7.js → index-DQa463gC.js} +2 -2
  15. package/dist/app/assets/{index-DWIe_hK0.js → index-DQpV7MyA.js} +1 -1
  16. package/dist/app/assets/{index-BabP-V-S.js → index-DzaYbhnD.js} +1 -1
  17. package/dist/app/assets/{index-D-rDyK11.js → index-VBbcc8s6.js} +1 -1
  18. package/dist/app/index.html +1 -1
  19. package/dist/package_load_worker.mjs +8 -1
  20. package/dist/server.mjs +471 -54
  21. package/package.json +16 -13
  22. package/.eslintrc.json +0 -44
  23. package/.prettierignore +0 -1
  24. package/.prettierrc +0 -4
  25. package/build.ts +0 -97
  26. package/eslint.config.mjs +0 -76
  27. package/k6-tests/README.md +0 -144
  28. package/k6-tests/bun.lock +0 -797
  29. package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
  30. package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
  31. package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
  32. package/k6-tests/load-test/load-test-crud.ts +0 -142
  33. package/k6-tests/load-test/load-test.ts +0 -784
  34. package/k6-tests/package.json +0 -27
  35. package/k6-tests/scripts/run-with-otel.sh +0 -50
  36. package/k6-tests/smoke-test/smoke-test.ts +0 -158
  37. package/k6-tests/tsconfig.json +0 -29
  38. package/k6-tests/utils/client_factory.ts +0 -128
  39. package/k6-tests/utils/common.ts +0 -489
  40. package/k6-tests/utils/logger.ts +0 -62
  41. package/k6-tests/utils/types.d.ts +0 -35
  42. package/malloy_mcp.dxt +0 -0
  43. package/publisher.config.example.bigquery.json +0 -33
  44. package/publisher.config.example.duckdb.json +0 -23
  45. package/publisher.config.json +0 -23
  46. package/scripts/bake-duckdb-extensions.js +0 -110
  47. package/src/config.spec.ts +0 -1535
  48. package/src/config.theme.spec.ts +0 -178
  49. package/src/config.ts +0 -1142
  50. package/src/constants.ts +0 -105
  51. package/src/controller/compile.controller.ts +0 -40
  52. package/src/controller/connection.controller.spec.ts +0 -849
  53. package/src/controller/connection.controller.ts +0 -901
  54. package/src/controller/database.controller.ts +0 -24
  55. package/src/controller/materialization.controller.spec.ts +0 -293
  56. package/src/controller/materialization.controller.ts +0 -319
  57. package/src/controller/model.controller.ts +0 -164
  58. package/src/controller/package.controller.spec.ts +0 -273
  59. package/src/controller/package.controller.ts +0 -306
  60. package/src/controller/query.controller.ts +0 -152
  61. package/src/controller/theme.controller.ts +0 -83
  62. package/src/controller/watch-mode.controller.ts +0 -243
  63. package/src/data_styles.ts +0 -71
  64. package/src/default-publisher.config.json +0 -23
  65. package/src/ducklake_version.spec.ts +0 -163
  66. package/src/ducklake_version.ts +0 -153
  67. package/src/errors.spec.ts +0 -119
  68. package/src/errors.ts +0 -237
  69. package/src/filter_deprecation.spec.ts +0 -64
  70. package/src/filter_deprecation.ts +0 -42
  71. package/src/health.spec.ts +0 -123
  72. package/src/health.ts +0 -233
  73. package/src/heap_check.spec.ts +0 -144
  74. package/src/heap_check.ts +0 -144
  75. package/src/instrumentation.ts +0 -214
  76. package/src/json_utils.spec.ts +0 -51
  77. package/src/json_utils.ts +0 -33
  78. package/src/logger.spec.ts +0 -210
  79. package/src/logger.ts +0 -251
  80. package/src/malloy_pin_prereqs.spec.ts +0 -25
  81. package/src/materialization_metrics.spec.ts +0 -169
  82. package/src/materialization_metrics.ts +0 -346
  83. package/src/mcp/error_messages.spec.ts +0 -35
  84. package/src/mcp/error_messages.ts +0 -183
  85. package/src/mcp/handler_utils.spec.ts +0 -108
  86. package/src/mcp/handler_utils.ts +0 -251
  87. package/src/mcp/mcp_constants.ts +0 -6
  88. package/src/mcp/query_envelope.spec.ts +0 -229
  89. package/src/mcp/query_envelope.ts +0 -240
  90. package/src/mcp/server.protocol.spec.ts +0 -250
  91. package/src/mcp/server.ts +0 -87
  92. package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
  93. package/src/mcp/skills/build_skills_bundle.ts +0 -176
  94. package/src/mcp/skills/skills_bundle.json +0 -1
  95. package/src/mcp/skills/skills_bundle.spec.ts +0 -171
  96. package/src/mcp/tool_response.spec.ts +0 -108
  97. package/src/mcp/tool_response.ts +0 -138
  98. package/src/mcp/tools/compile_tool.spec.ts +0 -315
  99. package/src/mcp/tools/compile_tool.ts +0 -208
  100. package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
  101. package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
  102. package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
  103. package/src/mcp/tools/docs_search_tool.ts +0 -128
  104. package/src/mcp/tools/embedding_index.spec.ts +0 -1236
  105. package/src/mcp/tools/embedding_index.ts +0 -808
  106. package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
  107. package/src/mcp/tools/execute_query_tool.ts +0 -316
  108. package/src/mcp/tools/get_context_eval.ts +0 -275
  109. package/src/mcp/tools/get_context_tool.spec.ts +0 -592
  110. package/src/mcp/tools/get_context_tool.ts +0 -585
  111. package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
  112. package/src/mcp/tools/reload_package_tool.ts +0 -132
  113. package/src/mcp_config.spec.ts +0 -919
  114. package/src/mcp_config.ts +0 -425
  115. package/src/oom_guards.integration.spec.ts +0 -269
  116. package/src/package_load/package_load_pool.spec.ts +0 -252
  117. package/src/package_load/package_load_pool.ts +0 -931
  118. package/src/package_load/package_load_worker.ts +0 -1034
  119. package/src/package_load/protocol.ts +0 -364
  120. package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
  121. package/src/package_load/rpc_wait_accountant.ts +0 -76
  122. package/src/package_load_metrics.spec.ts +0 -114
  123. package/src/package_load_metrics.ts +0 -127
  124. package/src/path_safety.spec.ts +0 -158
  125. package/src/path_safety.ts +0 -146
  126. package/src/pg_helpers.spec.ts +0 -223
  127. package/src/pg_helpers.ts +0 -52
  128. package/src/query_cap_metrics.spec.ts +0 -89
  129. package/src/query_cap_metrics.ts +0 -116
  130. package/src/query_concurrency.spec.ts +0 -247
  131. package/src/query_concurrency.ts +0 -237
  132. package/src/query_metadata_metrics.ts +0 -49
  133. package/src/query_param_utils.ts +0 -29
  134. package/src/query_timeout.spec.ts +0 -224
  135. package/src/query_timeout.ts +0 -179
  136. package/src/runtime/publisher.js +0 -328
  137. package/src/server-old.ts +0 -988
  138. package/src/server.ts +0 -2059
  139. package/src/service/annotations.spec.ts +0 -118
  140. package/src/service/annotations.ts +0 -91
  141. package/src/service/authorize.spec.ts +0 -154
  142. package/src/service/authorize.ts +0 -498
  143. package/src/service/authorize_integration.spec.ts +0 -2041
  144. package/src/service/build_plan.spec.ts +0 -685
  145. package/src/service/build_plan.ts +0 -797
  146. package/src/service/compile_authorize.spec.ts +0 -85
  147. package/src/service/compile_fragment_techniques.spec.ts +0 -156
  148. package/src/service/connection.spec.ts +0 -2360
  149. package/src/service/connection.ts +0 -2187
  150. package/src/service/connection_config.spec.ts +0 -734
  151. package/src/service/connection_config.ts +0 -899
  152. package/src/service/connection_federation.spec.ts +0 -184
  153. package/src/service/connection_fingerprint.spec.ts +0 -102
  154. package/src/service/connection_service.spec.ts +0 -711
  155. package/src/service/connection_service.ts +0 -291
  156. package/src/service/cron_evaluator.spec.ts +0 -79
  157. package/src/service/cron_evaluator.ts +0 -105
  158. package/src/service/db_utils.spec.ts +0 -1085
  159. package/src/service/db_utils.ts +0 -1502
  160. package/src/service/duckdb_instance_isolation.spec.ts +0 -137
  161. package/src/service/ducklake_lazy_attach.spec.ts +0 -110
  162. package/src/service/embedding_provider.spec.ts +0 -329
  163. package/src/service/embedding_provider.ts +0 -236
  164. package/src/service/environment.ts +0 -2191
  165. package/src/service/environment_admission.spec.ts +0 -344
  166. package/src/service/environment_compile.spec.ts +0 -197
  167. package/src/service/environment_store.spec.ts +0 -2274
  168. package/src/service/environment_store.ts +0 -2434
  169. package/src/service/environment_store_anchoring.spec.ts +0 -107
  170. package/src/service/environment_store_clone.spec.ts +0 -350
  171. package/src/service/explore_visibility.spec.ts +0 -434
  172. package/src/service/exports_probe.spec.ts +0 -107
  173. package/src/service/extension_fetch_policy.spec.ts +0 -256
  174. package/src/service/filter.spec.ts +0 -458
  175. package/src/service/filter.ts +0 -341
  176. package/src/service/filter_bypass.spec.ts +0 -418
  177. package/src/service/filter_integration.spec.ts +0 -1004
  178. package/src/service/freshness.spec.ts +0 -183
  179. package/src/service/freshness.ts +0 -112
  180. package/src/service/gcs_s3_utils.ts +0 -427
  181. package/src/service/given.ts +0 -105
  182. package/src/service/givens_integration.spec.ts +0 -219
  183. package/src/service/manifest_loader.spec.ts +0 -189
  184. package/src/service/manifest_loader.ts +0 -156
  185. package/src/service/materialization_build_session.spec.ts +0 -435
  186. package/src/service/materialization_build_session.ts +0 -681
  187. package/src/service/materialization_config_validation.spec.ts +0 -99
  188. package/src/service/materialization_config_validation.ts +0 -120
  189. package/src/service/materialization_eligibility.spec.ts +0 -158
  190. package/src/service/materialization_eligibility.ts +0 -305
  191. package/src/service/materialization_schedule_surface.spec.ts +0 -256
  192. package/src/service/materialization_scheduler.spec.ts +0 -289
  193. package/src/service/materialization_scheduler.ts +0 -304
  194. package/src/service/materialization_serve_transform.spec.ts +0 -1003
  195. package/src/service/materialization_serve_transform.ts +0 -779
  196. package/src/service/materialization_service.spec.ts +0 -2373
  197. package/src/service/materialization_service.ts +0 -2342
  198. package/src/service/materialization_test_fixtures.ts +0 -249
  199. package/src/service/model.spec.ts +0 -1157
  200. package/src/service/model.ts +0 -3370
  201. package/src/service/model_limits.spec.ts +0 -209
  202. package/src/service/model_limits.ts +0 -131
  203. package/src/service/model_storage_serve.spec.ts +0 -193
  204. package/src/service/model_storage_serve_joins.spec.ts +0 -193
  205. package/src/service/package.spec.ts +0 -612
  206. package/src/service/package.ts +0 -1799
  207. package/src/service/package_manifest.spec.ts +0 -230
  208. package/src/service/package_manifest.ts +0 -241
  209. package/src/service/package_memory_governor.spec.ts +0 -173
  210. package/src/service/package_memory_governor.ts +0 -233
  211. package/src/service/package_quote_bound_tables.spec.ts +0 -171
  212. package/src/service/package_race.spec.ts +0 -208
  213. package/src/service/package_reload_safety.spec.ts +0 -193
  214. package/src/service/package_rollback.spec.ts +0 -190
  215. package/src/service/package_worker_path.spec.ts +0 -645
  216. package/src/service/path_injection.spec.ts +0 -39
  217. package/src/service/persist_annotation_validation.spec.ts +0 -89
  218. package/src/service/persist_annotation_validation.ts +0 -59
  219. package/src/service/persistence_policy.spec.ts +0 -374
  220. package/src/service/proxy.spec.ts +0 -414
  221. package/src/service/proxy.ts +0 -248
  222. package/src/service/query_boundary.spec.ts +0 -470
  223. package/src/service/query_metadata.spec.ts +0 -408
  224. package/src/service/query_metadata.ts +0 -492
  225. package/src/service/query_metadata_identity.spec.ts +0 -149
  226. package/src/service/query_text.spec.ts +0 -79
  227. package/src/service/query_text.ts +0 -43
  228. package/src/service/quoting.spec.ts +0 -135
  229. package/src/service/quoting.ts +0 -78
  230. package/src/service/resolve_environment.ts +0 -15
  231. package/src/service/restricted_mode.spec.ts +0 -299
  232. package/src/service/source_extraction.ts +0 -225
  233. package/src/service/theme_store.ts +0 -199
  234. package/src/storage/DatabaseInterface.ts +0 -198
  235. package/src/storage/StorageManager.ts +0 -113
  236. package/src/storage/duckdb/ConnectionRepository.ts +0 -143
  237. package/src/storage/duckdb/DuckDBConnection.ts +0 -123
  238. package/src/storage/duckdb/DuckDBRepository.ts +0 -208
  239. package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
  240. package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
  241. package/src/storage/duckdb/MaterializationRepository.ts +0 -336
  242. package/src/storage/duckdb/PackageRepository.ts +0 -135
  243. package/src/storage/duckdb/schema.ts +0 -229
  244. package/src/stream_helpers.spec.ts +0 -280
  245. package/src/stream_helpers.ts +0 -162
  246. package/src/telemetry.ts +0 -20
  247. package/src/test_helpers/metrics_harness.ts +0 -166
  248. package/src/theme_key_parity.spec.ts +0 -57
  249. package/src/utils.ts +0 -29
  250. package/tests/fixtures/authorize-compile/model.malloy +0 -9
  251. package/tests/fixtures/authorize-compile/publisher.json +0 -4
  252. package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
  253. package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
  254. package/tests/fixtures/html-pages-test/data.csv +0 -3
  255. package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
  256. package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
  257. package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
  258. package/tests/fixtures/html-pages-test/public/data.json +0 -1
  259. package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
  260. package/tests/fixtures/html-pages-test/public/index.html +0 -9
  261. package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
  262. package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
  263. package/tests/fixtures/html-pages-test/public/slides.html +0 -12
  264. package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
  265. package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
  266. package/tests/fixtures/html-pages-test/publisher.json +0 -5
  267. package/tests/fixtures/html-pages-test/report.malloy +0 -1
  268. package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
  269. package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
  270. package/tests/fixtures/persist-multi-level/publisher.json +0 -5
  271. package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
  272. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
  273. package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
  274. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
  275. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
  276. package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
  277. package/tests/fixtures/persist-test/data/orders.csv +0 -5
  278. package/tests/fixtures/persist-test/persist_test.malloy +0 -11
  279. package/tests/fixtures/persist-test/publisher.json +0 -5
  280. package/tests/fixtures/publisher.config.json +0 -15
  281. package/tests/fixtures/query-givens/data/orders.csv +0 -7
  282. package/tests/fixtures/query-givens/model.malloy +0 -34
  283. package/tests/fixtures/query-givens/publisher.json +0 -5
  284. package/tests/fixtures/xlsx/database.xlsx +0 -0
  285. package/tests/harness/mcp_test_setup.ts +0 -332
  286. package/tests/harness/rest_e2e.ts +0 -68
  287. package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
  288. package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
  289. package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
  290. package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
  291. package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
  292. package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
  293. package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
  294. package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
  295. package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
  296. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
  297. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
  298. package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
  299. package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
  300. package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
  301. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
  302. package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
  303. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
  304. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
  305. package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
  306. package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
  307. package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
  308. package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
  309. package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
  310. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
  311. package/tests/unit/controller/theme.controller.spec.ts +0 -141
  312. package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
  313. package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
  314. package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
  315. package/tests/unit/duckdb/repositories.test.ts +0 -208
  316. package/tests/unit/ducklake/ducklake.test.ts +0 -952
  317. package/tests/unit/service/theme_store.spec.ts +0 -139
  318. package/tsconfig.json +0 -32
@@ -1,2373 +0,0 @@
1
- import type { Connection as MalloyConnection } from "@malloydata/malloy";
2
- import { Manifest } from "@malloydata/malloy";
3
- import { afterEach, beforeEach, describe, expect, it } from "bun:test";
4
- import * as sinon from "sinon";
5
- import {
6
- MaterializationEligibilityError,
7
- BadRequestError,
8
- EnvironmentNotFoundError,
9
- InvalidStateTransitionError,
10
- MaterializationConflictError,
11
- MaterializationNotFoundError,
12
- } from "../errors";
13
- import {
14
- BuildInstruction,
15
- MaterializationStatus,
16
- ResourceRepository,
17
- } from "../storage/DatabaseInterface";
18
- import { DuplicateActiveMaterializationError } from "../storage/duckdb/MaterializationRepository";
19
- import { EnvironmentStore } from "./environment_store";
20
- import {
21
- compiledWith,
22
- fakeSource,
23
- makeBuildPlan,
24
- makeInstruction,
25
- makeMaterialization,
26
- } from "./materialization_test_fixtures";
27
- import {
28
- manifestExcludingStorage,
29
- MaterializationService,
30
- redactConnectionSecrets,
31
- stagingSuffix,
32
- } from "./materialization_service";
33
- import { logger } from "../logger";
34
- import { resetMaterializationTelemetryForTesting } from "../materialization_metrics";
35
- import {
36
- startMetricsHarness,
37
- type MetricsHarness,
38
- } from "../test_helpers/metrics_harness";
39
-
40
- type MockRepo = sinon.SinonStubbedInstance<ResourceRepository>;
41
-
42
- /**
43
- * Turn per-query metadata on for one test and put the environment back after.
44
- * The feature ships dark, so a test that asserts what a statement carries has
45
- * to enable it — and saying so per test keeps the default honest here: a test
46
- * that forgets is testing the shipped default, which is "nothing attached".
47
- */
48
- function withQueryMetadataOn(): void {
49
- const original = process.env.PUBLISHER_QUERY_METADATA;
50
- beforeEach(() => {
51
- process.env.PUBLISHER_QUERY_METADATA = "on";
52
- });
53
- afterEach(() => {
54
- if (original === undefined) delete process.env.PUBLISHER_QUERY_METADATA;
55
- else process.env.PUBLISHER_QUERY_METADATA = original;
56
- });
57
- }
58
-
59
- describe("redactConnectionSecrets", () => {
60
- it("strips credential values but keeps the message legible", () => {
61
- const source = {
62
- name: "orders_pg",
63
- type: "postgres",
64
- postgresConnection: {
65
- host: "db.internal",
66
- userName: "svc",
67
- password: "sup3r-secret-pw",
68
- },
69
- };
70
- const msg =
71
- "IO Error: could not connect password=sup3r-secret-pw to db.internal";
72
- const out = redactConnectionSecrets(msg, source);
73
- expect(out).not.toContain("sup3r-secret-pw");
74
- expect(out).toContain("could not connect");
75
- expect(out).toContain("db.internal"); // host isn't a secret — stays
76
- });
77
-
78
- it("passes a credential-free error (e.g. schema not found) through verbatim", () => {
79
- const source = {
80
- name: "lake",
81
- type: "ducklake",
82
- ducklakeConnection: {
83
- catalog: { postgresConnection: { password: "catpw123" } },
84
- },
85
- };
86
- const msg =
87
- "Catalog Error: Schema 'analytics' not found in DuckLake 'lake'";
88
- expect(redactConnectionSecrets(msg, source)).toBe(msg);
89
- });
90
-
91
- it("redacts a secret in its SQL-escaped form (single quotes doubled)", () => {
92
- // A DuckDB error echoes the offending statement, in which a secret with a
93
- // single quote appears escaped (`'` -> `''`), so the raw value won't match.
94
- const source = {
95
- name: "pg",
96
- type: "postgres",
97
- postgresConnection: { password: "pa'ss'word" },
98
- };
99
- const msg =
100
- "Parser Error: syntax error near 'password=pa''ss''word host=h'";
101
- const out = redactConnectionSecrets(msg, source);
102
- expect(out).not.toContain("pa''ss''word");
103
- expect(out).toContain("***");
104
- });
105
-
106
- // Stubbed at the seam because the leak needs an ATTACH (not CTAS) failure:
107
- // bad catalog creds fast-fail at connection validation, so no black-box test
108
- // can reach this branch.
109
- it("redacts an INFRA chained-build failure, and does not call it a refusal", async () => {
110
- const sandbox = sinon.createSandbox();
111
- try {
112
- const destinationConnection = {
113
- name: "lake",
114
- type: "ducklake",
115
- ducklakeConnection: {
116
- catalog: {
117
- postgresConnection: {
118
- host: "catalog.internal",
119
- userName: "lake_rw",
120
- password: "c4talog-pw",
121
- },
122
- },
123
- },
124
- };
125
- const sourceConnection = {
126
- name: "orders_pg",
127
- type: "postgres",
128
- postgresConnection: { host: "db.internal", password: "src-pw-99" },
129
- };
130
-
131
- // A failed RW ATTACH echoes the statement, connstring included.
132
- sandbox
133
- .stub(
134
- MaterializationService.prototype as unknown as Record<
135
- string,
136
- unknown
137
- >,
138
- "buildDownstreamViaParents" as never,
139
- )
140
- .rejects(
141
- new Error(
142
- "IO Error: failed to ATTACH 'ducklake:postgres:host=catalog.internal " +
143
- "user=lake_rw password=c4talog-pw' (db lake)",
144
- ),
145
- );
146
-
147
- const { service } = createMocks();
148
- const environment = {
149
- getApiConnection: (name: string) =>
150
- name === "lake" ? destinationConnection : sourceConnection,
151
- getEnvironmentPath: () => "/tmp/env",
152
- };
153
-
154
- const call = (
155
- service as unknown as {
156
- buildOneSourceIntoStorage: (...a: unknown[]) => Promise<unknown>;
157
- }
158
- ).buildOneSourceIntoStorage(
159
- { name: "orders_by_month", connectionName: "orders_pg" },
160
- {
161
- sourceEntityId: "sid-1",
162
- physicalTableName: "mz_orders_by_month",
163
- destination: "lake",
164
- },
165
- // strict: throw instead of falling through to recompute-from-raw.
166
- { strict: true, update: () => {} },
167
- environment,
168
- "SELECT 1",
169
- {},
170
- true, // dependsOnStorageUpstream — take the chained path
171
- );
172
-
173
- // A failed ATTACH is infrastructure, not a shape limit: it must NOT
174
- // present as the strict-upstreams refusal (that message means "we could
175
- // have recomputed from raw but you forbade it", which is not what
176
- // happened here).
177
- await expect(call).rejects.toThrow(
178
- /Failed to materialize chained source/i,
179
- );
180
- await expect(call).rejects.not.toThrow(/strict upstreams forbid/i);
181
- const message = await call.then(
182
- () => "",
183
- (e: unknown) => (e instanceof Error ? e.message : String(e)),
184
- );
185
- expect(message).not.toContain("c4talog-pw");
186
- expect(message).toContain("***");
187
- // Redacted but still legible.
188
- expect(message).toContain("failed to ATTACH");
189
- expect(message).toContain("catalog.internal");
190
- expect(message).toContain("orders_by_month");
191
- } finally {
192
- sandbox.restore();
193
- }
194
- });
195
- });
196
-
197
- function createMocks() {
198
- const sandbox = sinon.createSandbox();
199
-
200
- const repository = {
201
- getEnvironmentByName: sandbox.stub(),
202
- listMaterializations: sandbox.stub(),
203
- getMaterializationById: sandbox.stub(),
204
- getActiveMaterialization: sandbox.stub(),
205
- createMaterialization: sandbox.stub(),
206
- updateMaterialization: sandbox.stub(),
207
- deleteMaterialization: sandbox.stub(),
208
- } as unknown as MockRepo;
209
-
210
- const storageManager = { getRepository: () => repository };
211
-
212
- const environmentStore = {
213
- storageManager,
214
- getEnvironment: sandbox.stub(),
215
- } as unknown as EnvironmentStore;
216
-
217
- // Default: environment resolves and yields a package whose build plan is
218
- // empty (no persist sources). Individual tests override getBuildPlan to
219
- // exercise the orchestrated path. getModelPaths()=>[] keeps the background
220
- // build from touching the Malloy runtime.
221
- repository.getEnvironmentByName.resolves({
222
- id: "env-1",
223
- name: "my-env",
224
- path: "/test",
225
- createdAt: new Date(),
226
- updatedAt: new Date(),
227
- });
228
- setPackage(environmentStore, { getBuildPlan: () => null });
229
-
230
- const service = new MaterializationService(environmentStore);
231
-
232
- return { sandbox, repository, environmentStore, service };
233
- }
234
-
235
- /** Point environmentStore.getEnvironment at a package with the given overrides. */
236
- function setPackage(
237
- environmentStore: EnvironmentStore,
238
- pkgOverrides: Record<string, unknown>,
239
- ): void {
240
- const pkg = {
241
- getBuildPlan: () => null,
242
- getModelPaths: () => [],
243
- getPackagePath: () => "/test",
244
- getMalloyConfig: () => ({}),
245
- getMalloyConnection: async () => ({}),
246
- // No bound manifestLocation by default, so seedFromBoundManifest no-ops.
247
- getPackageMetadata: () => ({}),
248
- ...pkgOverrides,
249
- };
250
- (environmentStore.getEnvironment as sinon.SinonStub).resolves({
251
- getPackage: sinon.stub().resolves(pkg),
252
- withPackageLock: async (_name: string, fn: () => Promise<unknown>) =>
253
- fn(),
254
- reloadAllModelsForPackage: sinon.stub().resolves(),
255
- });
256
- }
257
-
258
- describe("MaterializationService", () => {
259
- let ctx: ReturnType<typeof createMocks>;
260
-
261
- beforeEach(() => {
262
- ctx = createMocks();
263
- });
264
-
265
- describe("resolveEnvironmentId", () => {
266
- it("throws EnvironmentNotFoundError when the environment is missing", async () => {
267
- ctx.repository.getEnvironmentByName.resolves(null);
268
- await expect(
269
- ctx.service.listMaterializations("unknown", "pkg"),
270
- ).rejects.toThrow(EnvironmentNotFoundError);
271
- });
272
- });
273
-
274
- describe("deleteMaterialization drop of a storage= table", () => {
275
- it("routes a storage entry to a destination-aware RW drop (best-effort, delete still completes)", async () => {
276
- // getApiConnection returns a non-storage type so the RW drop refuses
277
- // at its destination gate (pre-session), exercising the routing + the
278
- // best-effort failure handling without needing a live DuckDB attach.
279
- const getApiConnection = sinon
280
- .stub()
281
- .returns({ name: "lake", type: "postgres" });
282
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
283
- getPackage: sinon
284
- .stub()
285
- .resolves({ getMalloyConnection: async () => ({}) }),
286
- getApiConnection,
287
- getEnvironmentPath: () => "/test",
288
- });
289
- ctx.repository.getMaterializationById.resolves(
290
- makeMaterialization({
291
- status: "MANIFEST_FILE_READY",
292
- manifest: {
293
- entries: {
294
- se1: {
295
- sourceEntityId: "se1",
296
- sourceName: "daily",
297
- physicalTableName: "daily__mabc123",
298
- connectionName: "wh",
299
- storageConnectionName: "lake",
300
- },
301
- },
302
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
303
- } as any,
304
- }),
305
- );
306
-
307
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
308
- dropTables: true,
309
- });
310
-
311
- // Routed to the storage path (resolved the destination connection for
312
- // an RW drop, not the old skip), and the delete completed despite the
313
- // drop's best-effort failure.
314
- expect(getApiConnection.calledWith("lake")).toBe(true);
315
- expect(
316
- (
317
- ctx.repository.deleteMaterialization as sinon.SinonStub
318
- ).calledWith("mat-1"),
319
- ).toBe(true);
320
- });
321
-
322
- it("skips the drop when another MANIFEST_FILE_READY run still references the table (MED-3)", async () => {
323
- // name=-verbatim: generations of a source share one physical name, so
324
- // dropping a superseded record's table must NOT take out the table a
325
- // remaining run still serves.
326
- const getApiConnection = sinon
327
- .stub()
328
- .returns({ name: "lake", type: "duckdb" });
329
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
330
- getPackage: sinon
331
- .stub()
332
- .resolves({ getMalloyConnection: async () => ({}) }),
333
- getApiConnection,
334
- getEnvironmentPath: () => "/test",
335
- });
336
- const entries = {
337
- se1: {
338
- sourceEntityId: "se1",
339
- sourceName: "daily",
340
- physicalTableName: "daily", // shared logical name
341
- connectionName: "wh",
342
- storageConnectionName: "lake",
343
- },
344
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
345
- } as any;
346
- ctx.repository.getMaterializationById.resolves(
347
- makeMaterialization({
348
- id: "mat-1",
349
- status: "MANIFEST_FILE_READY",
350
- manifest: { entries },
351
- }),
352
- );
353
- // Another remaining successful run references the SAME lake.daily table.
354
- ctx.repository.listMaterializations.resolves([
355
- makeMaterialization({ id: "mat-1", manifest: { entries } }),
356
- makeMaterialization({
357
- id: "mat-2",
358
- status: "MANIFEST_FILE_READY",
359
- manifest: { entries },
360
- }),
361
- ]);
362
-
363
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
364
- dropTables: true,
365
- });
366
-
367
- // The drop was skipped (never resolved the destination connection), and
368
- // the record was still deleted.
369
- expect(getApiConnection.called).toBe(false);
370
- expect(
371
- (
372
- ctx.repository.deleteMaterialization as sinon.SinonStub
373
- ).calledWith("mat-1"),
374
- ).toBe(true);
375
- });
376
- });
377
-
378
- describe("deleteMaterialization rebinds serve bindings (both tiers)", () => {
379
- const storageEntry = {
380
- se1: {
381
- sourceEntityId: "se1",
382
- sourceName: "daily",
383
- physicalTableName: "daily__mabc",
384
- connectionName: "wh",
385
- storageConnectionName: "lake",
386
- schema: [{ name: "a", type: "BIGINT" }],
387
- },
388
- };
389
- // A colocated entry carries NO storageConnectionName.
390
- const colocatedEntry = {
391
- ce1: {
392
- sourceEntityId: "ce1",
393
- sourceName: "daily",
394
- physicalTableName: "daily__cabc",
395
- connectionName: "wh",
396
- },
397
- };
398
-
399
- function setupEnv(): {
400
- storageSpy: sinon.SinonStub;
401
- colocatedSpy: sinon.SinonStub;
402
- } {
403
- const storageSpy = sinon.stub().resolves();
404
- const colocatedSpy = sinon.stub().resolves();
405
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
406
- getPackage: sinon
407
- .stub()
408
- .resolves({ getMalloyConnection: async () => ({}) }),
409
- getApiConnection: () => ({ name: "lake", type: "duckdb" }),
410
- getEnvironmentPath: () => "/test",
411
- bindPackageStorageServeBindings: storageSpy,
412
- bindPackageColocatedServeManifest: colocatedSpy,
413
- });
414
- ctx.repository.getMaterializationById.resolves(
415
- makeMaterialization({
416
- status: "MANIFEST_FILE_READY",
417
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
418
- manifest: { entries: {} } as any,
419
- }),
420
- );
421
- return { storageSpy, colocatedSpy };
422
- }
423
-
424
- it("re-derives storage bindings from the next-latest materialization (on)", async () => {
425
- process.env.PERSIST_STORAGE_MODE = "on";
426
- try {
427
- const { storageSpy, colocatedSpy } = setupEnv();
428
- // The latest REMAINING successful run carries a storage entry.
429
- ctx.repository.listMaterializations.resolves([
430
- makeMaterialization({
431
- id: "m-prev",
432
- status: "MANIFEST_FILE_READY",
433
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
434
- manifest: { entries: storageEntry } as any,
435
- }),
436
- ]);
437
-
438
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
439
-
440
- expect(storageSpy.calledOnce).toBe(true);
441
- expect(Object.keys(storageSpy.firstCall.args[1])).toContain("se1");
442
- // Colocated is re-derived too (with no colocated entries ⇒ cleared).
443
- expect(colocatedSpy.calledOnce).toBe(true);
444
- expect(colocatedSpy.firstCall.args[1]).toEqual({});
445
- } finally {
446
- delete process.env.PERSIST_STORAGE_MODE;
447
- }
448
- });
449
-
450
- it("re-derives colocated bindings regardless of tier mode (off)", async () => {
451
- // PERSIST_STORAGE_MODE unset ⇒ off. Colocated is the v0 path, not gated
452
- // by the storage kill switch, so it is still re-derived; storage is not.
453
- const { storageSpy, colocatedSpy } = setupEnv();
454
- ctx.repository.listMaterializations.resolves([
455
- makeMaterialization({
456
- id: "m-prev",
457
- status: "MANIFEST_FILE_READY",
458
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
459
- manifest: { entries: colocatedEntry } as any,
460
- }),
461
- ]);
462
-
463
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
464
-
465
- expect(colocatedSpy.calledOnce).toBe(true);
466
- expect(Object.keys(colocatedSpy.firstCall.args[1])).toContain("ce1");
467
- // Storage rebind is skipped when the tier is off.
468
- expect(storageSpy.called).toBe(false);
469
- });
470
-
471
- it("clears both tiers' bindings when no successful materialization remains (on)", async () => {
472
- process.env.PERSIST_STORAGE_MODE = "on";
473
- try {
474
- const { storageSpy, colocatedSpy } = setupEnv();
475
- ctx.repository.listMaterializations.resolves([]); // none remain
476
-
477
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
478
-
479
- expect(storageSpy.calledOnce).toBe(true);
480
- expect(storageSpy.firstCall.args[1]).toEqual({}); // empty ⇒ serve live
481
- expect(colocatedSpy.calledOnce).toBe(true);
482
- expect(colocatedSpy.firstCall.args[1]).toEqual({});
483
- } finally {
484
- delete process.env.PERSIST_STORAGE_MODE;
485
- }
486
- });
487
-
488
- it("skips only the storage rebind when the tier is off (dark-ship)", async () => {
489
- const { storageSpy, colocatedSpy } = setupEnv(); // off
490
- ctx.repository.listMaterializations.resolves([]); // none remain
491
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
492
- // Storage is skipped when off; colocated still re-derives (to clear).
493
- expect(storageSpy.called).toBe(false);
494
- expect(colocatedSpy.calledOnce).toBe(true);
495
- expect(colocatedSpy.firstCall.args[1]).toEqual({});
496
- });
497
- });
498
-
499
- describe("queries", () => {
500
- it("lists materializations for a package", async () => {
501
- const rows = [
502
- makeMaterialization(),
503
- makeMaterialization({ id: "m2" }),
504
- ];
505
- ctx.repository.listMaterializations.resolves(rows);
506
- expect(
507
- await ctx.service.listMaterializations("my-env", "pkg"),
508
- ).toEqual(rows);
509
- });
510
-
511
- it("returns a specific materialization", async () => {
512
- const m = makeMaterialization();
513
- ctx.repository.getMaterializationById.resolves(m);
514
- expect(
515
- await ctx.service.getMaterialization("my-env", "pkg", "mat-1"),
516
- ).toEqual(m);
517
- });
518
-
519
- it("throws when the materialization is missing", async () => {
520
- ctx.repository.getMaterializationById.resolves(null);
521
- await expect(
522
- ctx.service.getMaterialization("my-env", "pkg", "nope"),
523
- ).rejects.toThrow(MaterializationNotFoundError);
524
- });
525
-
526
- it("throws when the materialization belongs to another package", async () => {
527
- ctx.repository.getMaterializationById.resolves(
528
- makeMaterialization({ packageName: "other" }),
529
- );
530
- await expect(
531
- ctx.service.getMaterialization("my-env", "pkg", "mat-1"),
532
- ).rejects.toThrow(MaterializationNotFoundError);
533
- });
534
- });
535
-
536
- describe("createMaterialization (auto-run)", () => {
537
- it("creates a PENDING record and persists options with mode=auto", async () => {
538
- ctx.repository.getActiveMaterialization.resolves(null);
539
- const pending = makeMaterialization({ status: "PENDING" });
540
- ctx.repository.createMaterialization.resolves(pending);
541
-
542
- const result = await ctx.service.createMaterialization(
543
- "my-env",
544
- "pkg",
545
- {
546
- forceRefresh: true,
547
- sourceNames: ["orders"],
548
- },
549
- );
550
-
551
- expect(result.status).toBe("PENDING");
552
- expect(ctx.repository.createMaterialization.firstCall.args).toEqual([
553
- "env-1",
554
- "pkg",
555
- "PENDING",
556
- {
557
- forceRefresh: true,
558
- sourceNames: ["orders"],
559
- mode: "auto",
560
- trigger: "ON_DEMAND",
561
- },
562
- ]);
563
- });
564
-
565
- it("defaults to auto mode in metadata", async () => {
566
- ctx.repository.getActiveMaterialization.resolves(null);
567
- ctx.repository.createMaterialization.resolves(
568
- makeMaterialization({ status: "PENDING" }),
569
- );
570
-
571
- await ctx.service.createMaterialization("my-env", "pkg", {});
572
-
573
- expect(ctx.repository.createMaterialization.firstCall.args[3]).toEqual(
574
- {
575
- forceRefresh: false,
576
- sourceNames: null,
577
- mode: "auto",
578
- trigger: "ON_DEMAND",
579
- },
580
- );
581
- });
582
-
583
- it("records trigger=SCHEDULER when the scheduler fires the run", async () => {
584
- ctx.repository.getActiveMaterialization.resolves(null);
585
- ctx.repository.createMaterialization.resolves(
586
- makeMaterialization({ status: "PENDING" }),
587
- );
588
-
589
- await ctx.service.createMaterialization("my-env", "pkg", {
590
- forceRefresh: true,
591
- trigger: "SCHEDULER",
592
- });
593
-
594
- expect(
595
- ctx.repository.createMaterialization.firstCall.args[3],
596
- ).toMatchObject({ mode: "auto", trigger: "SCHEDULER" });
597
- });
598
-
599
- it("rejects when an active materialization already exists", async () => {
600
- ctx.repository.getActiveMaterialization.resolves(
601
- makeMaterialization({
602
- id: "existing",
603
- status: "MANIFEST_ROWS_READY",
604
- }),
605
- );
606
- await expect(
607
- ctx.service.createMaterialization("my-env", "pkg"),
608
- ).rejects.toThrow(MaterializationConflictError);
609
- });
610
-
611
- it("translates a lost create race into a conflict", async () => {
612
- ctx.repository.getActiveMaterialization
613
- .onFirstCall()
614
- .resolves(null)
615
- .onSecondCall()
616
- .resolves(makeMaterialization({ id: "winner" }));
617
- ctx.repository.createMaterialization.rejects(
618
- new DuplicateActiveMaterializationError("env-1", "pkg"),
619
- );
620
- await expect(
621
- ctx.service.createMaterialization("my-env", "pkg"),
622
- ).rejects.toThrow(/winner/);
623
- });
624
- });
625
-
626
- describe("createMaterialization (orchestrated)", () => {
627
- it("creates a PENDING record with mode=orchestrated for valid instructions", async () => {
628
- setPackage(ctx.environmentStore, {
629
- getBuildPlan: () => makeBuildPlan(),
630
- });
631
- ctx.repository.getActiveMaterialization.resolves(null);
632
- ctx.repository.createMaterialization.resolves(
633
- makeMaterialization({ status: "PENDING" }),
634
- );
635
-
636
- const result = await ctx.service.createMaterialization(
637
- "my-env",
638
- "pkg",
639
- { buildInstructions: [makeInstruction()] },
640
- );
641
-
642
- expect(result.status).toBe("PENDING");
643
- expect(
644
- ctx.repository.createMaterialization.firstCall.args[3],
645
- ).toMatchObject({ mode: "orchestrated" });
646
- });
647
-
648
- it("rejects instructions referencing an unknown sourceEntityId before creating", async () => {
649
- setPackage(ctx.environmentStore, {
650
- getBuildPlan: () => makeBuildPlan(),
651
- });
652
- await expect(
653
- ctx.service.createMaterialization("my-env", "pkg", {
654
- buildInstructions: [
655
- makeInstruction({ sourceEntityId: "ghost" }),
656
- ],
657
- }),
658
- ).rejects.toThrow(BadRequestError);
659
- expect(ctx.repository.createMaterialization.called).toBe(false);
660
- });
661
-
662
- it("rejects SNAPSHOT realization", async () => {
663
- setPackage(ctx.environmentStore, {
664
- getBuildPlan: () => makeBuildPlan(),
665
- });
666
- await expect(
667
- ctx.service.createMaterialization("my-env", "pkg", {
668
- buildInstructions: [
669
- makeInstruction({ realization: "SNAPSHOT" }),
670
- ],
671
- }),
672
- ).rejects.toThrow(BadRequestError);
673
- });
674
-
675
- it("rejects buildInstructions when the package has no persist sources", async () => {
676
- setPackage(ctx.environmentStore, { getBuildPlan: () => null });
677
- await expect(
678
- ctx.service.createMaterialization("my-env", "pkg", {
679
- buildInstructions: [makeInstruction()],
680
- }),
681
- ).rejects.toThrow(BadRequestError);
682
- });
683
- });
684
-
685
- describe("stopMaterialization", () => {
686
- const cancellable = ["PENDING", "MANIFEST_ROWS_READY"] as const;
687
-
688
- for (const status of cancellable) {
689
- it(`cancels a ${status} materialization`, async () => {
690
- ctx.repository.getMaterializationById.resolves(
691
- makeMaterialization({ status }),
692
- );
693
- ctx.repository.updateMaterialization.resolves(
694
- makeMaterialization({ status: "CANCELLED" }),
695
- );
696
- const result = await ctx.service.stopMaterialization(
697
- "my-env",
698
- "pkg",
699
- "mat-1",
700
- );
701
- expect(result.status).toBe("CANCELLED");
702
- });
703
- }
704
-
705
- const terminal = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"] as const;
706
- for (const status of terminal) {
707
- it(`rejects stopping a ${status} materialization`, async () => {
708
- ctx.repository.getMaterializationById.resolves(
709
- makeMaterialization({ status }),
710
- );
711
- await expect(
712
- ctx.service.stopMaterialization("my-env", "pkg", "mat-1"),
713
- ).rejects.toThrow(InvalidStateTransitionError);
714
- });
715
- }
716
- });
717
-
718
- describe("deleteMaterialization", () => {
719
- const terminal = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"] as const;
720
- for (const status of terminal) {
721
- it(`deletes a ${status} materialization`, async () => {
722
- ctx.repository.getMaterializationById.resolves(
723
- makeMaterialization({ status }),
724
- );
725
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
726
- expect(
727
- ctx.repository.deleteMaterialization.calledOnceWith("mat-1"),
728
- ).toBe(true);
729
- });
730
- }
731
-
732
- const active = ["PENDING", "MANIFEST_ROWS_READY"] as const;
733
- for (const status of active) {
734
- it(`rejects deleting a ${status} materialization`, async () => {
735
- ctx.repository.getMaterializationById.resolves(
736
- makeMaterialization({ status }),
737
- );
738
- await expect(
739
- ctx.service.deleteMaterialization("my-env", "pkg", "mat-1"),
740
- ).rejects.toThrow(InvalidStateTransitionError);
741
- expect(ctx.repository.deleteMaterialization.called).toBe(false);
742
- });
743
- }
744
-
745
- it("does not drop tables by default (record-only delete)", async () => {
746
- const runSQL = sinon.stub().resolves();
747
- const getMalloyConnection = sinon.stub().resolves({ runSQL });
748
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
749
- getPackage: sinon.stub().resolves({ getMalloyConnection }),
750
- withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
751
- fn(),
752
- });
753
- ctx.repository.getMaterializationById.resolves(
754
- makeMaterialization({
755
- status: "MANIFEST_FILE_READY",
756
- manifest: {
757
- builtAt: new Date().toISOString(),
758
- strict: false,
759
- entries: {
760
- b1: {
761
- sourceEntityId: "b1",
762
- physicalTableName: "orders_mz",
763
- connectionName: "duckdb",
764
- },
765
- },
766
- },
767
- }),
768
- );
769
-
770
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
771
-
772
- expect(getMalloyConnection.called).toBe(false);
773
- expect(runSQL.called).toBe(false);
774
- expect(
775
- ctx.repository.deleteMaterialization.calledOnceWith("mat-1"),
776
- ).toBe(true);
777
- });
778
-
779
- it("drops manifest tables (and staging) when dropTables is set", async () => {
780
- const runSQL = sinon.stub().resolves();
781
- const getMalloyConnection = sinon
782
- .stub()
783
- .resolves({ runSQL, dialectName: "duckdb" });
784
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
785
- getPackage: sinon.stub().resolves({ getMalloyConnection }),
786
- withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
787
- fn(),
788
- });
789
- ctx.repository.getMaterializationById.resolves(
790
- makeMaterialization({
791
- status: "MANIFEST_FILE_READY",
792
- manifest: {
793
- builtAt: new Date().toISOString(),
794
- strict: false,
795
- entries: {
796
- b1: {
797
- sourceEntityId: "b1",
798
- physicalTableName: "orders_mz",
799
- connectionName: "duckdb",
800
- },
801
- },
802
- },
803
- }),
804
- );
805
-
806
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
807
- dropTables: true,
808
- });
809
-
810
- expect(getMalloyConnection.calledOnceWith("duckdb")).toBe(true);
811
- const dropped = runSQL.getCalls().map((c) => c.args[0] as string);
812
- expect(dropped).toContain('DROP TABLE IF EXISTS "orders_mz"');
813
- expect(dropped.some((s) => s.includes('"orders_mz_'))).toBe(true);
814
- expect(
815
- ctx.repository.deleteMaterialization.calledOnceWith("mat-1"),
816
- ).toBe(true);
817
- });
818
-
819
- it("dialect-quotes the drop DDL from the live connection (backtick + container path)", async () => {
820
- const runSQL = sinon.stub().resolves();
821
- const getMalloyConnection = sinon
822
- .stub()
823
- .resolves({ runSQL, dialectName: "standardsql" });
824
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
825
- getPackage: sinon.stub().resolves({ getMalloyConnection }),
826
- withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
827
- fn(),
828
- });
829
- ctx.repository.getMaterializationById.resolves(
830
- makeMaterialization({
831
- status: "MANIFEST_FILE_READY",
832
- manifest: {
833
- builtAt: new Date().toISOString(),
834
- strict: false,
835
- entries: {
836
- b1: {
837
- sourceEntityId: "b1",
838
- // Container path on a hyphenated BigQuery project: each
839
- // segment must be backtick-quoted independently.
840
- physicalTableName: "my-proj.ds.engaged",
841
- connectionName: "bq",
842
- },
843
- },
844
- },
845
- }),
846
- );
847
-
848
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
849
- dropTables: true,
850
- });
851
-
852
- const dropped = runSQL.getCalls().map((c) => c.args[0] as string);
853
- expect(dropped).toContain(
854
- "DROP TABLE IF EXISTS `my-proj`.`ds`.`engaged`",
855
- );
856
- });
857
-
858
- it("still deletes the record when a table drop fails", async () => {
859
- const runSQL = sinon.stub().rejects(new Error("boom"));
860
- const getMalloyConnection = sinon.stub().resolves({ runSQL });
861
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
862
- getPackage: sinon.stub().resolves({ getMalloyConnection }),
863
- withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
864
- fn(),
865
- });
866
- ctx.repository.getMaterializationById.resolves(
867
- makeMaterialization({
868
- status: "FAILED",
869
- manifest: {
870
- builtAt: new Date().toISOString(),
871
- strict: false,
872
- entries: {
873
- b1: {
874
- sourceEntityId: "b1",
875
- physicalTableName: "orders_mz",
876
- connectionName: "duckdb",
877
- },
878
- },
879
- },
880
- }),
881
- );
882
-
883
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
884
- dropTables: true,
885
- });
886
-
887
- expect(
888
- ctx.repository.deleteMaterialization.calledOnceWith("mat-1"),
889
- ).toBe(true);
890
- });
891
- });
892
- });
893
-
894
- describe("deleteMaterialization telemetry", () => {
895
- let ctx: ReturnType<typeof createMocks>;
896
- let harness: MetricsHarness;
897
- withQueryMetadataOn();
898
-
899
- beforeEach(async () => {
900
- ctx = createMocks();
901
- harness = await startMetricsHarness();
902
- // Drop cached instruments so recordDropTables re-binds to this provider.
903
- resetMaterializationTelemetryForTesting();
904
- });
905
-
906
- afterEach(async () => {
907
- resetMaterializationTelemetryForTesting();
908
- await harness.shutdown();
909
- });
910
-
911
- const DROP_COUNTER = "publisher_materialization_drop_tables_total";
912
-
913
- // A terminal materialization whose manifest names one physical table, so
914
- // dropMaterializedTables issues exactly one DROP — the emission under test.
915
- function dropTablesFixture(runSQL: sinon.SinonStub) {
916
- const getMalloyConnection = sinon
917
- .stub()
918
- .resolves({ runSQL, dialectName: "duckdb" });
919
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
920
- getPackage: sinon.stub().resolves({ getMalloyConnection }),
921
- withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
922
- fn(),
923
- });
924
- ctx.repository.getMaterializationById.resolves(
925
- makeMaterialization({
926
- status: "MANIFEST_FILE_READY",
927
- manifest: {
928
- builtAt: new Date().toISOString(),
929
- strict: false,
930
- entries: {
931
- b1: {
932
- sourceEntityId: "b1",
933
- physicalTableName: "orders_mz",
934
- connectionName: "duckdb",
935
- },
936
- },
937
- },
938
- }),
939
- );
940
- }
941
-
942
- it("meters a successful physical-table drop as outcome=success", async () => {
943
- dropTablesFixture(sinon.stub().resolves());
944
-
945
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
946
- dropTables: true,
947
- });
948
-
949
- expect(
950
- await harness.collectCounter(DROP_COUNTER, { outcome: "success" }),
951
- ).toBe(1);
952
- expect(
953
- await harness.collectCounter(DROP_COUNTER, { outcome: "failure" }),
954
- ).toBe(0);
955
- });
956
-
957
- it("tags the retiring drops as ops work", async () => {
958
- // A drop is warehouse work someone accounts for later, and it is the one
959
- // statement an operator goes looking for after a table disappears.
960
- const runSQL = sinon.stub().resolves();
961
- dropTablesFixture(runSQL);
962
-
963
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
964
- dropTables: true,
965
- });
966
-
967
- expect(runSQL.callCount).toBeGreaterThan(0);
968
- for (const call of runSQL.getCalls()) {
969
- expect(call.args[1]?.queryMetadata).toMatchObject({
970
- class: "ops",
971
- environment: "my-env",
972
- package: "pkg",
973
- run_id: "mat-1",
974
- });
975
- }
976
- });
977
-
978
- it("meters a failed drop as outcome=failure and still deletes the record", async () => {
979
- dropTablesFixture(sinon.stub().rejects(new Error("drop boom")));
980
-
981
- // The drop is best-effort: a failure is metered but never surfaces to the
982
- // caller, and the record is still removed.
983
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
984
- dropTables: true,
985
- });
986
-
987
- expect(
988
- await harness.collectCounter(DROP_COUNTER, { outcome: "failure" }),
989
- ).toBe(1);
990
- expect(ctx.repository.deleteMaterialization.calledOnceWith("mat-1")).toBe(
991
- true,
992
- );
993
- });
994
- });
995
-
996
- describe("stagingSuffix", () => {
997
- it("derives a short, stable suffix from the sourceEntityId", () => {
998
- expect(stagingSuffix("abcdef1234567890")).toBe("_abcdef123456");
999
- });
1000
- });
1001
-
1002
- describe("manifestExcludingStorage (chained-storage inline)", () => {
1003
- it("drops storage entries, keeps colocated entries with the manifest's quoting, preserves strict", () => {
1004
- // The source manifest is already dialect-quoted by the seed loop (#904);
1005
- // manifestExcludingStorage reuses that quoting for the kept entries.
1006
- const manifest = new Manifest();
1007
- manifest.strict = true;
1008
- manifest.update("wh_up", { tableName: '"wh_up_v1"' });
1009
- manifest.update("lake_up", { tableName: '"lake"."lake_up__mabc"' });
1010
- const built: Record<string, unknown> = {
1011
- wh_up: {
1012
- sourceEntityId: "wh_up",
1013
- physicalTableName: "wh_up_v1",
1014
- connectionName: "duckdb",
1015
- // no storageConnectionName -> colocated, keep it
1016
- },
1017
- lake_up: {
1018
- sourceEntityId: "lake_up",
1019
- physicalTableName: "lake_up__mabc",
1020
- connectionName: "wh",
1021
- storageConnectionName: "lake", // storage -> exclude it
1022
- },
1023
- };
1024
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
1025
- const reduced = manifestExcludingStorage(manifest, built as any) as any;
1026
- expect(Object.keys(reduced.entries)).toEqual(["wh_up"]);
1027
- // Kept with the manifest's dialect-quoted name, not the raw physical name.
1028
- expect(reduced.entries.wh_up.tableName).toBe('"wh_up_v1"');
1029
- expect(reduced.entries.lake_up).toBeUndefined();
1030
- expect(reduced.strict).toBe(true);
1031
- });
1032
- });
1033
-
1034
- describe("deriveSelfInstructions", () => {
1035
- let ctx: ReturnType<typeof createMocks>;
1036
- beforeEach(() => {
1037
- ctx = createMocks();
1038
- });
1039
-
1040
- it("carries forward unchanged sourceEntityIds and builds the rest (deduping repeats)", () => {
1041
- const compiled = compiledWith(
1042
- {
1043
- s1: fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" }),
1044
- s2: fakeSource({ name: "s2", sourceEntityId: "b2bbbbbbbbbbbbbb" }),
1045
- },
1046
- [["s1", "s2"], ["s2"]],
1047
- );
1048
- const priorEntries = {
1049
- b1aaaaaaaaaaaaaa: {
1050
- sourceEntityId: "b1aaaaaaaaaaaaaa",
1051
- physicalTableName: "s1_prev",
1052
- connectionName: "duckdb",
1053
- },
1054
- };
1055
-
1056
- const { instructions, carried } = (
1057
- ctx.service as unknown as {
1058
- deriveSelfInstructions: (
1059
- c: unknown,
1060
- n: string[] | undefined,
1061
- p: unknown,
1062
- ) => { instructions: BuildInstruction[]; carried: unknown };
1063
- }
1064
- ).deriveSelfInstructions(compiled, undefined, priorEntries);
1065
-
1066
- expect(instructions).toHaveLength(1);
1067
- expect(instructions[0].sourceEntityId).toBe("b2bbbbbbbbbbbbbb");
1068
- expect(instructions[0].physicalTableName).toBe("s2");
1069
- expect(instructions[0].realization).toBe("COPY");
1070
- expect(instructions[0].materializedTableId).toBe("local-b2bbbbbbbbbb");
1071
- expect(
1072
- (carried as Record<string, unknown>)["b1aaaaaaaaaaaaaa"],
1073
- ).toBeDefined();
1074
- });
1075
-
1076
- it("self-assigns the name= verbatim for a storage= source, exactly as a colocated build", () => {
1077
- process.env.PERSIST_STORAGE_MODE = "on";
1078
- try {
1079
- const compiled = compiledWith(
1080
- {
1081
- lake_src: fakeSource({
1082
- name: "lake_src",
1083
- sourceEntityId: "aa11aa11aa11aa11aa11",
1084
- annotationFields: { storage: "lake", name: "daily" },
1085
- }),
1086
- wh_src: fakeSource({
1087
- name: "wh_src",
1088
- sourceEntityId: "bb22bb22bb22bb22bb22",
1089
- }),
1090
- },
1091
- [["lake_src", "wh_src"]],
1092
- );
1093
- const { instructions } = (
1094
- ctx.service as unknown as {
1095
- deriveSelfInstructions: (
1096
- c: unknown,
1097
- n: string[] | undefined,
1098
- p: unknown,
1099
- ) => { instructions: BuildInstruction[] };
1100
- }
1101
- ).deriveSelfInstructions(compiled, undefined, {});
1102
- const byId = Object.fromEntries(
1103
- instructions.map((i) => [i.sourceEntityId, i]),
1104
- );
1105
- const lake = byId["aa11aa11aa11aa11aa11"];
1106
- const wh = byId["bb22bb22bb22bb22bb22"];
1107
- // storage= → the `name=` value verbatim in the destination (no
1108
- // content-hash decoration); only the destination differs from a colocated build.
1109
- expect(lake.destination).toBe("lake");
1110
- expect(lake.physicalTableName).toBe("daily");
1111
- // colocated → the plain logical (source) name, no destination.
1112
- expect(wh.destination).toBeUndefined();
1113
- expect(wh.physicalTableName).toBe("wh_src");
1114
- } finally {
1115
- delete process.env.PERSIST_STORAGE_MODE;
1116
- }
1117
- });
1118
-
1119
- it("honors the sourceNames filter (excluded sources are neither built nor carried)", () => {
1120
- const compiled = compiledWith(
1121
- {
1122
- s1: fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" }),
1123
- s2: fakeSource({ name: "s2", sourceEntityId: "b2bbbbbbbbbbbbbb" }),
1124
- },
1125
- [["s1", "s2"]],
1126
- );
1127
- const { instructions, carried } = (
1128
- ctx.service as unknown as {
1129
- deriveSelfInstructions: (
1130
- c: unknown,
1131
- n: string[] | undefined,
1132
- p: unknown,
1133
- ) => { instructions: BuildInstruction[]; carried: unknown };
1134
- }
1135
- ).deriveSelfInstructions(compiled, ["s2"], {});
1136
-
1137
- expect(instructions).toHaveLength(1);
1138
- expect(instructions[0].sourceEntityId).toBe("b2bbbbbbbbbbbbbb");
1139
- expect(Object.keys(carried as Record<string, unknown>)).toHaveLength(0);
1140
- });
1141
- });
1142
-
1143
- describe("getMostRecentManifestEntries (skip-if-unchanged)", () => {
1144
- let ctx: ReturnType<typeof createMocks>;
1145
- beforeEach(() => {
1146
- ctx = createMocks();
1147
- });
1148
-
1149
- it("returns entries from the most recent successful run, excluding the in-flight one", async () => {
1150
- const entries = {
1151
- b1: {
1152
- sourceEntityId: "b1",
1153
- physicalTableName: "orders_v1",
1154
- connectionName: "duckdb",
1155
- },
1156
- };
1157
- ctx.repository.listMaterializations.resolves([
1158
- makeMaterialization({
1159
- id: "in-flight",
1160
- status: "MANIFEST_ROWS_READY",
1161
- }),
1162
- makeMaterialization({
1163
- id: "old-success",
1164
- status: "MANIFEST_FILE_READY",
1165
- manifest: { builtAt: "t", strict: false, entries },
1166
- }),
1167
- ]);
1168
-
1169
- const result = await (
1170
- ctx.service as unknown as {
1171
- getMostRecentManifestEntries: (
1172
- e: string,
1173
- p: string,
1174
- x: string,
1175
- ) => Promise<unknown>;
1176
- }
1177
- ).getMostRecentManifestEntries("env-1", "pkg", "in-flight");
1178
-
1179
- expect(result).toEqual(entries);
1180
- });
1181
-
1182
- it("returns {} when the only successful run is the excluded one", async () => {
1183
- ctx.repository.listMaterializations.resolves([
1184
- makeMaterialization({
1185
- id: "self",
1186
- status: "MANIFEST_FILE_READY",
1187
- manifest: {
1188
- builtAt: "t",
1189
- strict: false,
1190
- entries: {
1191
- b1: { sourceEntityId: "b1", physicalTableName: "t1" },
1192
- },
1193
- },
1194
- }),
1195
- ]);
1196
-
1197
- const result = await (
1198
- ctx.service as unknown as {
1199
- getMostRecentManifestEntries: (
1200
- e: string,
1201
- p: string,
1202
- x: string,
1203
- ) => Promise<unknown>;
1204
- }
1205
- ).getMostRecentManifestEntries("env-1", "pkg", "self");
1206
-
1207
- expect(result).toEqual({});
1208
- });
1209
- });
1210
-
1211
- describe("stopMaterialization (in-flight)", () => {
1212
- let ctx: ReturnType<typeof createMocks>;
1213
- beforeEach(() => {
1214
- ctx = createMocks();
1215
- });
1216
-
1217
- it("aborts a running build cooperatively without forcing a transition", async () => {
1218
- ctx.repository.getMaterializationById.resolves(
1219
- makeMaterialization({ status: "MANIFEST_ROWS_READY" }),
1220
- );
1221
- const controller = new AbortController();
1222
- (
1223
- ctx.service as unknown as {
1224
- runningAbortControllers: Map<string, AbortController>;
1225
- }
1226
- ).runningAbortControllers.set("mat-1", controller);
1227
-
1228
- const result = await ctx.service.stopMaterialization(
1229
- "my-env",
1230
- "pkg",
1231
- "mat-1",
1232
- );
1233
-
1234
- expect(controller.signal.aborted).toBe(true);
1235
- // The background run records the terminal state; stop must not also write.
1236
- expect(ctx.repository.updateMaterialization.called).toBe(false);
1237
- expect(result.status).toBe("MANIFEST_ROWS_READY");
1238
- });
1239
- });
1240
-
1241
- describe("executeInstructedBuild", () => {
1242
- let ctx: ReturnType<typeof createMocks>;
1243
- beforeEach(() => {
1244
- ctx = createMocks();
1245
- });
1246
-
1247
- function callExecute(
1248
- compiled: unknown,
1249
- instructions: BuildInstruction[],
1250
- seed: Record<string, unknown>,
1251
- ): Promise<Record<string, { physicalTableName?: string }>> {
1252
- // A stub BuildEnvironment: only the `storage=` branch touches it, and
1253
- // these tests exercise the default in-warehouse path, so it is never read.
1254
- const environment = {
1255
- getApiConnection: () => {
1256
- throw new Error("no connection config in this test");
1257
- },
1258
- getEnvironmentPath: () => "/tmp/env",
1259
- };
1260
- return (
1261
- ctx.service as unknown as {
1262
- executeInstructedBuild: (
1263
- c: unknown,
1264
- env: unknown,
1265
- i: BuildInstruction[],
1266
- s: Record<string, unknown>,
1267
- sig: AbortSignal,
1268
- ) => Promise<Record<string, { physicalTableName?: string }>>;
1269
- }
1270
- ).executeInstructedBuild(
1271
- compiled,
1272
- environment,
1273
- instructions,
1274
- seed,
1275
- new AbortController().signal,
1276
- );
1277
- }
1278
-
1279
- it("builds instructed sources in dependency order and seeds carried entries", async () => {
1280
- const runSQL = sinon.stub().resolves();
1281
- const connection = { runSQL } as unknown as MalloyConnection;
1282
- const s1 = fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" });
1283
- const s2 = fakeSource({ name: "s2", sourceEntityId: "b2bbbbbbbbbbbbbb" });
1284
- const compiled = compiledWith(
1285
- { s1, s2 },
1286
- [["s1"], ["s2"]],
1287
- new Map([["duckdb", connection]]),
1288
- );
1289
-
1290
- const entries = await callExecute(
1291
- compiled,
1292
- [
1293
- {
1294
- sourceEntityId: "b1aaaaaaaaaaaaaa",
1295
- materializedTableId: "mt-1",
1296
- physicalTableName: "s1_v1",
1297
- realization: "COPY",
1298
- },
1299
- {
1300
- sourceEntityId: "b2bbbbbbbbbbbbbb",
1301
- materializedTableId: "mt-2",
1302
- physicalTableName: "s2_v1",
1303
- realization: "COPY",
1304
- },
1305
- ],
1306
- {
1307
- carried0: {
1308
- sourceEntityId: "carried0",
1309
- physicalTableName: "carried_tbl",
1310
- connectionName: "duckdb",
1311
- },
1312
- },
1313
- );
1314
-
1315
- // Both instructed sources built, plus the carried seed entry retained.
1316
- expect(entries["b1aaaaaaaaaaaaaa"].physicalTableName).toBe("s1_v1");
1317
- expect(entries["b2bbbbbbbbbbbbbb"].physicalTableName).toBe("s2_v1");
1318
- expect(entries["carried0"].physicalTableName).toBe("carried_tbl");
1319
- });
1320
-
1321
- it("builds an intermediate persist source nested under a root node", async () => {
1322
- // Mirrors malloy getBuildPlan(): only the terminal `root` is a graph
1323
- // node; its persist dependency `mid` is nested in dependsOn (not a node).
1324
- // Before the iterGraphSources fix, `mid` was silently never built even
1325
- // though the caller instructed it. Both must now build, `mid` first.
1326
- const runSQL = sinon.stub().resolves();
1327
- const connection = { runSQL } as unknown as MalloyConnection;
1328
- const root = fakeSource({
1329
- name: "root",
1330
- sourceEntityId: "brootaaaaaaaaaa",
1331
- });
1332
- const mid = fakeSource({
1333
- name: "mid",
1334
- sourceEntityId: "bmidbbbbbbbbbbb",
1335
- });
1336
- const compiled = {
1337
- graphs: [
1338
- {
1339
- connectionName: "duckdb",
1340
- nodes: [
1341
- [
1342
- {
1343
- sourceID: "root",
1344
- dependsOn: [{ sourceID: "mid", dependsOn: [] }],
1345
- },
1346
- ],
1347
- ],
1348
- },
1349
- ],
1350
- sources: { root, mid },
1351
- connectionDigests: { duckdb: "dig" },
1352
- connections: new Map([["duckdb", connection]]),
1353
- };
1354
-
1355
- const entries = await callExecute(
1356
- compiled,
1357
- [
1358
- {
1359
- sourceEntityId: "brootaaaaaaaaaa",
1360
- materializedTableId: "mt-r",
1361
- physicalTableName: "root_v1",
1362
- realization: "COPY",
1363
- },
1364
- {
1365
- sourceEntityId: "bmidbbbbbbbbbbb",
1366
- materializedTableId: "mt-m",
1367
- physicalTableName: "mid_v1",
1368
- realization: "COPY",
1369
- },
1370
- ],
1371
- {},
1372
- );
1373
-
1374
- expect(entries["bmidbbbbbbbbbbb"].physicalTableName).toBe("mid_v1");
1375
- expect(entries["brootaaaaaaaaaa"].physicalTableName).toBe("root_v1");
1376
- // The dependency's CREATE precedes its dependent's (dependency order).
1377
- const creates = runSQL
1378
- .getCalls()
1379
- .map((c) => c.args[0] as string)
1380
- .filter((s) => s.startsWith("CREATE TABLE"));
1381
- const midIdx = creates.findIndex((s) => s.includes("mid_v1"));
1382
- const rootIdx = creates.findIndex((s) => s.includes("root_v1"));
1383
- expect(midIdx).toBeGreaterThanOrEqual(0);
1384
- expect(rootIdx).toBeGreaterThan(midIdx);
1385
- });
1386
-
1387
- it("seeds a downstream build with the QUOTED upstream reference (case-folding dialect)", async () => {
1388
- // A carried/seeded upstream (built in a prior run or unit, e.g. via
1389
- // referenceManifest) must reach the downstream's SQL generation as the
1390
- // SAME quoted path the builder CREATEd it with; otherwise the downstream
1391
- // CREATE misses the case-preserved table on a case-folding engine
1392
- // (Snowflake). Capture the buildManifest Malloy is handed for the
1393
- // downstream to prove the seed was quoted for its connection's dialect
1394
- // before it becomes a FROM.
1395
- let seen:
1396
- | {
1397
- buildManifest?: {
1398
- entries?: Record<string, { tableName?: string }>;
1399
- };
1400
- }
1401
- | undefined;
1402
- const runSQL = sinon.stub().resolves();
1403
- const connection = {
1404
- runSQL,
1405
- dialectName: "snowflake",
1406
- } as unknown as MalloyConnection;
1407
- const down = fakeSource({
1408
- name: "down",
1409
- sourceEntityId: "bdownaaaaaaaaaa",
1410
- dialectName: "snowflake",
1411
- onGetSQL: (o) => {
1412
- seen = o as typeof seen;
1413
- },
1414
- });
1415
- const compiled = compiledWith(
1416
- { down },
1417
- [["down"]],
1418
- new Map([["duckdb", connection]]),
1419
- );
1420
-
1421
- await callExecute(
1422
- compiled,
1423
- [
1424
- {
1425
- sourceEntityId: "bdownaaaaaaaaaa",
1426
- materializedTableId: "mt-d",
1427
- physicalTableName: "down_v1",
1428
- realization: "COPY",
1429
- },
1430
- ],
1431
- {
1432
- up: {
1433
- sourceEntityId: "up",
1434
- physicalTableName: "schema.orders_base",
1435
- connectionName: "duckdb",
1436
- },
1437
- },
1438
- );
1439
-
1440
- expect(seen?.buildManifest?.entries?.up?.tableName).toBe(
1441
- '"schema"."orders_base"',
1442
- );
1443
- });
1444
-
1445
- it("seeds unquoted when the upstream's connection is absent from the build (older CP / cross-connection)", async () => {
1446
- // No connectionName on the seed (an older control plane) -> bind verbatim,
1447
- // exactly the pre-change behavior. Non-case-folding engines are unaffected;
1448
- // a case-folding engine self-signals by failing the downstream CREATE.
1449
- let seen:
1450
- | {
1451
- buildManifest?: {
1452
- entries?: Record<string, { tableName?: string }>;
1453
- };
1454
- }
1455
- | undefined;
1456
- const runSQL = sinon.stub().resolves();
1457
- const connection = {
1458
- runSQL,
1459
- dialectName: "snowflake",
1460
- } as unknown as MalloyConnection;
1461
- const down = fakeSource({
1462
- name: "down",
1463
- sourceEntityId: "bdownaaaaaaaaaa",
1464
- dialectName: "snowflake",
1465
- onGetSQL: (o) => {
1466
- seen = o as typeof seen;
1467
- },
1468
- });
1469
- const compiled = compiledWith(
1470
- { down },
1471
- [["down"]],
1472
- new Map([["duckdb", connection]]),
1473
- );
1474
-
1475
- await callExecute(
1476
- compiled,
1477
- [
1478
- {
1479
- sourceEntityId: "bdownaaaaaaaaaa",
1480
- materializedTableId: "mt-d",
1481
- physicalTableName: "down_v1",
1482
- realization: "COPY",
1483
- },
1484
- ],
1485
- { up: { sourceEntityId: "up", physicalTableName: "orders_base" } },
1486
- );
1487
-
1488
- expect(seen?.buildManifest?.entries?.up?.tableName).toBe("orders_base");
1489
- });
1490
-
1491
- it("throws when an instructed graph's connection is missing", async () => {
1492
- const s1 = fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" });
1493
- const compiled = compiledWith({ s1 }, [["s1"]], new Map());
1494
-
1495
- await expect(
1496
- callExecute(
1497
- compiled,
1498
- [
1499
- {
1500
- sourceEntityId: "b1aaaaaaaaaaaaaa",
1501
- materializedTableId: "mt-1",
1502
- physicalTableName: "s1_v1",
1503
- realization: "COPY",
1504
- },
1505
- ],
1506
- {},
1507
- ),
1508
- ).rejects.toThrow(BadRequestError);
1509
- });
1510
- });
1511
-
1512
- describe("buildOneSource", () => {
1513
- let ctx: ReturnType<typeof createMocks>;
1514
- withQueryMetadataOn();
1515
- beforeEach(() => {
1516
- ctx = createMocks();
1517
- });
1518
-
1519
- function callBuildOneSource(
1520
- connection: { runSQL: sinon.SinonStub },
1521
- physicalTableName: string,
1522
- dialectName?: string,
1523
- manifest: Manifest = new Manifest(),
1524
- metadata?: {
1525
- source?: Record<string, string>;
1526
- package?: Record<string, string>;
1527
- connection?: Record<string, string>;
1528
- context?: Record<string, string | undefined>;
1529
- },
1530
- ): Promise<{ sourceEntityId: string; physicalTableName: string }> {
1531
- const source = fakeSource({
1532
- name: "orders",
1533
- sourceEntityId: "abcdef1234567890",
1534
- sql: "SELECT * FROM t",
1535
- dialectName,
1536
- queryMetadata: metadata?.source,
1537
- });
1538
- const instruction: BuildInstruction = {
1539
- sourceEntityId: "abcdef1234567890",
1540
- materializedTableId: "mt-1",
1541
- physicalTableName,
1542
- realization: "COPY",
1543
- };
1544
- return (
1545
- ctx.service as unknown as {
1546
- buildOneSource: (
1547
- s: unknown,
1548
- i: BuildInstruction,
1549
- c: unknown,
1550
- d: Record<string, string>,
1551
- m: Manifest,
1552
- env: unknown,
1553
- built: Record<string, unknown>,
1554
- buildMetadata?: unknown,
1555
- ) => Promise<{ sourceEntityId: string; physicalTableName: string }>;
1556
- }
1557
- ).buildOneSource(
1558
- source,
1559
- instruction,
1560
- connection,
1561
- { duckdb: "dig" },
1562
- manifest,
1563
- {
1564
- getApiConnection: () => {
1565
- if (metadata?.connection) {
1566
- return { queryMetadata: metadata.connection };
1567
- }
1568
- throw new Error("no connection config in this colocated test");
1569
- },
1570
- getEnvironmentPath: () => "/tmp/env",
1571
- },
1572
- {}, // builtEntries — colocated build with no prior storage upstreams
1573
- metadata
1574
- ? {
1575
- packageMaterialization: metadata.package
1576
- ? {
1577
- schedule: null,
1578
- freshness: null,
1579
- queryMetadata: metadata.package,
1580
- }
1581
- : null,
1582
- context: metadata.context ?? { queryClass: "materialize" },
1583
- }
1584
- : undefined,
1585
- );
1586
- }
1587
-
1588
- it("tags every statement of the build with the resolved metadata", async () => {
1589
- // One unit of work: the warehouse's query history has to show the staging
1590
- // drop, the CTAS, the swap and the rename as the same build.
1591
- const runSQL = sinon.stub().resolves();
1592
- await callBuildOneSource({ runSQL }, "orders_v1", undefined, undefined, {
1593
- connection: { deployment: "eu" },
1594
- package: { team: "finance", tier: "bronze" },
1595
- source: { tier: "gold" },
1596
- context: { queryClass: "materialize", runId: "run-7" },
1597
- });
1598
-
1599
- const bags = runSQL.getCalls().map((c) => c.args[1]?.queryMetadata);
1600
- expect(bags).toHaveLength(4);
1601
- for (const bag of bags) {
1602
- expect(bag).toEqual({
1603
- deployment: "eu",
1604
- team: "finance",
1605
- // The source's own declaration wins over the package's.
1606
- tier: "gold",
1607
- class: "materialize",
1608
- run_id: "run-7",
1609
- source: "orders",
1610
- });
1611
- }
1612
- });
1613
-
1614
- it("issues untagged statements when nothing declares metadata", async () => {
1615
- // No metadata context (an internal caller) means the statements are exactly
1616
- // what they were before this existed.
1617
- const runSQL = sinon.stub().resolves();
1618
- await callBuildOneSource({ runSQL }, "orders_v1");
1619
- for (const call of runSQL.getCalls()) {
1620
- expect(call.args[1]?.queryMetadata).toBeUndefined();
1621
- }
1622
- });
1623
-
1624
- it("tags the cleanup drop after a failed build", async () => {
1625
- // The cleanup statement belongs to the same build, and it is the one an
1626
- // operator goes looking for after a failure.
1627
- const runSQL = sinon.stub();
1628
- runSQL.onCall(0).resolves();
1629
- runSQL.onCall(1).rejects(new Error("create boom"));
1630
- runSQL.onCall(2).resolves();
1631
- await expect(
1632
- callBuildOneSource({ runSQL }, "orders_v1", undefined, undefined, {
1633
- source: { team: "finance" },
1634
- context: { queryClass: "materialize" },
1635
- }),
1636
- ).rejects.toThrow("create boom");
1637
- expect(runSQL.lastCall.args[1]?.queryMetadata).toEqual({
1638
- team: "finance",
1639
- class: "materialize",
1640
- source: "orders",
1641
- });
1642
- });
1643
-
1644
- it("stages, swaps, and renames in a crash-safe order", async () => {
1645
- const runSQL = sinon.stub().resolves();
1646
- const entry = await callBuildOneSource({ runSQL }, "orders_v1");
1647
-
1648
- const sql = runSQL.getCalls().map((c) => c.args[0] as string);
1649
- expect(sql).toEqual([
1650
- 'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
1651
- 'CREATE TABLE "orders_v1_abcdef123456" AS (SELECT * FROM t)',
1652
- 'DROP TABLE IF EXISTS "orders_v1"',
1653
- 'ALTER TABLE "orders_v1_abcdef123456" RENAME TO "orders_v1"',
1654
- ]);
1655
- expect(entry.physicalTableName).toBe("orders_v1");
1656
- expect(entry.sourceEntityId).toBe("abcdef1234567890");
1657
- });
1658
-
1659
- it("drops the staging table and rethrows when the build SQL fails", async () => {
1660
- const runSQL = sinon.stub();
1661
- runSQL.onCall(0).resolves(); // initial staging drop
1662
- runSQL.onCall(1).rejects(new Error("create boom")); // CREATE TABLE AS
1663
- runSQL.onCall(2).resolves(); // cleanup staging drop
1664
- await expect(callBuildOneSource({ runSQL }, "orders_v1")).rejects.toThrow(
1665
- "create boom",
1666
- );
1667
- expect(runSQL.lastCall.args[0]).toBe(
1668
- 'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
1669
- );
1670
- });
1671
-
1672
- it("rethrows the original build error when staging cleanup also fails", async () => {
1673
- const runSQL = sinon.stub();
1674
- runSQL.onCall(0).resolves(); // initial staging drop
1675
- runSQL.onCall(1).rejects(new Error("create boom")); // CREATE TABLE AS
1676
- runSQL.onCall(2).rejects(new Error("cleanup boom")); // cleanup drop fails
1677
- // The cleanup failure is swallowed (logged as a physical leak); the
1678
- // original build error is what propagates, so the run is classified by its
1679
- // real cause rather than the cleanup noise.
1680
- await expect(callBuildOneSource({ runSQL }, "orders_v1")).rejects.toThrow(
1681
- "create boom",
1682
- );
1683
- expect(runSQL.callCount).toBe(3);
1684
- expect(runSQL.lastCall.args[0]).toBe(
1685
- 'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
1686
- );
1687
- });
1688
-
1689
- it("drops staging when the build is cancelled mid-run (no _staging orphan)", async () => {
1690
- // buildOneSource is cancellation-agnostic: a cancel surfaces as a thrown
1691
- // build SQL and takes the same staging-cleanup path as a failure, so a
1692
- // cancelled build leaves no orphaned _staging table behind.
1693
- const runSQL = sinon.stub();
1694
- runSQL.onCall(0).resolves(); // initial staging drop
1695
- runSQL.onCall(1).rejects(new Error("Build cancelled")); // aborted mid-CREATE
1696
- runSQL.onCall(2).resolves(); // cleanup staging drop
1697
- await expect(callBuildOneSource({ runSQL }, "orders_v1")).rejects.toThrow(
1698
- "Build cancelled",
1699
- );
1700
- expect(runSQL.lastCall.args[0]).toBe(
1701
- 'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
1702
- );
1703
- });
1704
-
1705
- it("quotes each segment of a container path for a backtick dialect", async () => {
1706
- const runSQL = sinon.stub().resolves();
1707
- // Container path (dataset.table) on a backtick dialect (BigQuery): each
1708
- // segment is quoted independently, and the rename targets the bare name.
1709
- const entry = await callBuildOneSource(
1710
- { runSQL },
1711
- "ds.orders_v1",
1712
- "standardsql",
1713
- );
1714
-
1715
- const sql = runSQL.getCalls().map((c) => c.args[0] as string);
1716
- expect(sql).toEqual([
1717
- "DROP TABLE IF EXISTS `ds`.`orders_v1_abcdef123456`",
1718
- "CREATE TABLE `ds`.`orders_v1_abcdef123456` AS (SELECT * FROM t)",
1719
- "DROP TABLE IF EXISTS `ds`.`orders_v1`",
1720
- "ALTER TABLE `ds`.`orders_v1_abcdef123456` RENAME TO `orders_v1`",
1721
- ]);
1722
- expect(entry.physicalTableName).toBe("ds.orders_v1");
1723
- });
1724
-
1725
- it("colocated build excludes a storage upstream from its warehouse manifest (mixed chain)", async () => {
1726
- // A colocated downstream that reads a storage-materialized
1727
- // upstream must NOT get the lake table name in its warehouse build SQL —
1728
- // the warehouse can't resolve it. The storage upstream is excluded (⇒
1729
- // inlined/recomputed); a colocated upstream is kept.
1730
- const runSQL = sinon.stub().resolves();
1731
- let seenManifest:
1732
- | { entries?: Record<string, { tableName?: string }> }
1733
- | undefined;
1734
- const down = fakeSource({
1735
- name: "down",
1736
- sourceEntityId: "downdowndowndown",
1737
- sql: "SELECT 1",
1738
- onGetSQL: (o) => {
1739
- seenManifest = (o as { buildManifest?: typeof seenManifest })
1740
- .buildManifest;
1741
- },
1742
- });
1743
- const manifest = new Manifest();
1744
- manifest.update("up_storage", { tableName: "daily__mabc" });
1745
- manifest.update("up_pathc", { tableName: '"orders_v1"' });
1746
- const builtEntries = {
1747
- up_storage: {
1748
- sourceEntityId: "up_storage",
1749
- physicalTableName: "daily__mabc",
1750
- storageConnectionName: "lake",
1751
- },
1752
- up_pathc: {
1753
- sourceEntityId: "up_pathc",
1754
- physicalTableName: "orders_v1",
1755
- connectionName: "duckdb",
1756
- },
1757
- };
1758
- await (
1759
- ctx.service as unknown as {
1760
- buildOneSource: (...a: unknown[]) => Promise<unknown>;
1761
- }
1762
- ).buildOneSource(
1763
- down,
1764
- {
1765
- sourceEntityId: "downdowndowndown",
1766
- materializedTableId: "mt",
1767
- physicalTableName: "down_v1",
1768
- realization: "COPY",
1769
- }, // no destination ⇒ colocated
1770
- { runSQL },
1771
- { duckdb: "dig" },
1772
- manifest,
1773
- {
1774
- getApiConnection: () => {
1775
- throw new Error("unused in the colocated build");
1776
- },
1777
- getEnvironmentPath: () => "/tmp/env",
1778
- },
1779
- builtEntries,
1780
- );
1781
- expect(seenManifest?.entries?.up_storage).toBeUndefined();
1782
- expect(seenManifest?.entries?.up_pathc?.tableName).toBe('"orders_v1"');
1783
- });
1784
-
1785
- it("records the QUOTED just-built name in the build manifest (matches the CREATE)", async () => {
1786
- // The build manifest feeds a downstream persist's FROM verbatim, so the
1787
- // recorded name must be the SAME quoted path the CREATE used — else a
1788
- // downstream built later in this run misses the case-preserved table on a
1789
- // case-folding engine. buildManifest also passes through Malloy's canonical
1790
- // path validation (requireCanonicalTablePathAnyDialect), so this proves the
1791
- // quoted form is accepted, not just produced.
1792
- const runSQL = sinon.stub().resolves();
1793
- const manifest = new Manifest();
1794
- await callBuildOneSource(
1795
- { runSQL },
1796
- "schema.orders_base",
1797
- "snowflake",
1798
- manifest,
1799
- );
1800
- expect(manifest.buildManifest.entries["abcdef1234567890"].tableName).toBe(
1801
- '"schema"."orders_base"',
1802
- );
1803
- // The logical (unquoted) name is what the CREATE was told to produce, and
1804
- // is exactly what quoteTablePath quoted for the DDL — read mirrors write.
1805
- const created = runSQL
1806
- .getCalls()
1807
- .map((c) => c.args[0] as string)
1808
- .find((s) => s.startsWith("CREATE TABLE"))!;
1809
- expect(created).toContain('"schema"."orders_base_abcdef123456"');
1810
- });
1811
- });
1812
-
1813
- describe("buildOneSourceIntoStorage (chained-build fallback ladder)", () => {
1814
- let ctx: ReturnType<typeof createMocks>;
1815
- beforeEach(() => {
1816
- ctx = createMocks();
1817
- });
1818
-
1819
- function callInto(opts: {
1820
- strict: boolean;
1821
- stackOnParent: "ok" | "throw" | "infra";
1822
- }): Promise<{
1823
- physicalTableName: string;
1824
- storageConnectionName?: string;
1825
- }> {
1826
- const source = fakeSource({
1827
- name: "monthly",
1828
- sourceEntityId: "abcdef1234567890",
1829
- annotationFields: { storage: "lake" },
1830
- });
1831
- const instruction: BuildInstruction = {
1832
- sourceEntityId: "abcdef1234567890",
1833
- materializedTableId: "mt",
1834
- physicalTableName: "monthly__mabc",
1835
- realization: "COPY",
1836
- destination: "lake",
1837
- };
1838
- const manifest = new Manifest();
1839
- manifest.strict = opts.strict;
1840
- const environment = {
1841
- getApiConnection: (n: string) => ({
1842
- name: n,
1843
- type: n === "lake" ? "duckdb" : "postgres",
1844
- }),
1845
- getEnvironmentPath: () => "/tmp/env",
1846
- };
1847
- const svc = ctx.service as unknown as {
1848
- buildDownstreamViaParents: unknown;
1849
- buildOneSourceIntoStorage: (
1850
- s: unknown,
1851
- i: BuildInstruction,
1852
- m: Manifest,
1853
- e: unknown,
1854
- sql: string,
1855
- built: Record<string, unknown>,
1856
- dep: boolean,
1857
- ) => Promise<{
1858
- physicalTableName: string;
1859
- storageConnectionName?: string;
1860
- }>;
1861
- };
1862
- // Stub the stack-on-parent seam so the test exercises the LADDER, not the build.
1863
- svc.buildDownstreamViaParents =
1864
- opts.stackOnParent === "ok"
1865
- ? sinon.stub().resolves({
1866
- storageConnectionName: "lake",
1867
- schema: [
1868
- { name: "order_month", type: "DATE" },
1869
- { name: "monthly_total", type: "DOUBLE" },
1870
- ],
1871
- })
1872
- : opts.stackOnParent === "infra"
1873
- ? // An outage, not a modelling limit: a plain Error, as a failed
1874
- // ATTACH or CTAS would throw.
1875
- sinon.stub().rejects(new Error("IO Error: destination is down"))
1876
- : // A shape limit: the downstream cannot be expressed over its
1877
- // rebound parents.
1878
- sinon.stub().rejects(
1879
- new MaterializationEligibilityError({
1880
- message: "uncarried parent",
1881
- }),
1882
- );
1883
- return svc.buildOneSourceIntoStorage(
1884
- source,
1885
- instruction,
1886
- manifest,
1887
- environment,
1888
- "SELECT should_not_run",
1889
- {
1890
- up: {
1891
- sourceEntityId: "up",
1892
- sourceName: "daily",
1893
- physicalTableName: "daily__mabc",
1894
- storageConnectionName: "lake",
1895
- schema: [{ name: "order_date", type: "DATE" }],
1896
- },
1897
- },
1898
- true, // dependsOnStorageUpstream
1899
- );
1900
- }
1901
-
1902
- it("stacks on the parent: builds by reading it and returns the storage entry", async () => {
1903
- const entry = await callInto({ strict: false, stackOnParent: "ok" });
1904
- expect(entry.storageConnectionName).toBe("lake");
1905
- expect(entry.physicalTableName).toBe("monthly__mabc");
1906
- });
1907
-
1908
- it("strict + cannot stack on the parent: refuses loudly instead of recomputing from raw", async () => {
1909
- await expect(
1910
- callInto({ strict: true, stackOnParent: "throw" }),
1911
- ).rejects.toThrow(/strict upstreams forbid/i);
1912
- });
1913
-
1914
- it("non-strict + an INFRA failure fails rather than recomputing from raw", async () => {
1915
- // Recompute-from-raw writes to the same destination, so retrying an outage
1916
- // there just fails again — and metering it as a fallback files the outage
1917
- // under the same label as a legitimate shape miss. Only a shape failure is
1918
- // a reason to try the other path.
1919
- await expect(
1920
- callInto({ strict: false, stackOnParent: "infra" }),
1921
- ).rejects.toThrow(/Failed to materialize chained source/i);
1922
- });
1923
-
1924
- it("non-strict + a SHAPE failure falls through to recompute-from-raw", async () => {
1925
- // Only the parent-reuse seam is stubbed, so the recompute runs for real and
1926
- // fails for its own reason (no destination file). That is the proof it was
1927
- // reached: the two paths report differently — the chained path says
1928
- // "chained source", the single-source recompute says "source". A shape
1929
- // failure must reach the second; an infra failure must not.
1930
- await expect(
1931
- callInto({ strict: false, stackOnParent: "throw" }),
1932
- ).rejects.toThrow(/Failed to materialize source 'monthly'/i);
1933
- });
1934
- });
1935
-
1936
- describe("runBuild (branch behavior)", () => {
1937
- let ctx: ReturnType<typeof createMocks>;
1938
- beforeEach(() => {
1939
- ctx = createMocks();
1940
- });
1941
-
1942
- // Exercise runBuild's orchestration in isolation: the build engine and
1943
- // manifest commit are stubbed so the test asserts which instructions flow
1944
- // through and whether the manifest is auto-loaded, not how a source is built.
1945
- type RunBuildInternals = {
1946
- executeInstructedBuild: sinon.SinonStub;
1947
- commitManifest: sinon.SinonStub;
1948
- autoLoadManifest: sinon.SinonStub;
1949
- runBuild: (
1950
- id: string,
1951
- env: string,
1952
- pkg: string,
1953
- opts: {
1954
- sourceNames: string[] | undefined;
1955
- forceRefresh: boolean;
1956
- buildInstructions: BuildInstruction[] | undefined;
1957
- referenceManifest?:
1958
- | {
1959
- sourceEntityId: string;
1960
- physicalTableName: string;
1961
- connectionName?: string;
1962
- }[]
1963
- | undefined;
1964
- strictUpstreams?: boolean | undefined;
1965
- },
1966
- signal: AbortSignal,
1967
- ) => Promise<void>;
1968
- };
1969
-
1970
- function stubEngine(): RunBuildInternals {
1971
- const entries = {
1972
- "build-orders": {
1973
- sourceEntityId: "build-orders",
1974
- physicalTableName: '"orders_v1"',
1975
- connectionName: "duckdb",
1976
- },
1977
- };
1978
- const svc = ctx.service as unknown as RunBuildInternals;
1979
- svc.executeInstructedBuild = sinon.stub().resolves(entries);
1980
- svc.commitManifest = sinon.stub().resolves();
1981
- svc.autoLoadManifest = sinon.stub().resolves();
1982
- return svc;
1983
- }
1984
-
1985
- it("orchestrated: builds caller instructions and does not auto-load", async () => {
1986
- const svc = stubEngine();
1987
- const instructions = [makeInstruction()];
1988
-
1989
- await svc.runBuild(
1990
- "mat-1",
1991
- "my-env",
1992
- "pkg",
1993
- {
1994
- sourceNames: undefined,
1995
- forceRefresh: false,
1996
- buildInstructions: instructions,
1997
- },
1998
- new AbortController().signal,
1999
- );
2000
-
2001
- expect(svc.executeInstructedBuild.calledOnce).toBe(true);
2002
- // Caller instructions pass straight through; nothing is carried/reused.
2003
- // (arg[1] is the environment; instructions are arg[2], seed entries arg[3].)
2004
- expect(svc.executeInstructedBuild.firstCall.args[2]).toEqual(
2005
- instructions,
2006
- );
2007
- expect(svc.executeInstructedBuild.firstCall.args[3]).toEqual({});
2008
- expect(svc.commitManifest.firstCall.args[2]).toMatchObject({
2009
- mode: "orchestrated",
2010
- sourcesBuilt: 1,
2011
- sourcesReused: 0,
2012
- });
2013
- // Orchestrated leaves distribution to the caller.
2014
- expect(svc.autoLoadManifest.called).toBe(false);
2015
- });
2016
-
2017
- it("orchestrated: seeds the build from referenceManifest and honors strictUpstreams", async () => {
2018
- const svc = stubEngine();
2019
- const instructions = [makeInstruction()];
2020
-
2021
- await svc.runBuild(
2022
- "mat-1",
2023
- "my-env",
2024
- "pkg",
2025
- {
2026
- sourceNames: undefined,
2027
- forceRefresh: false,
2028
- buildInstructions: instructions,
2029
- referenceManifest: [
2030
- {
2031
- sourceEntityId: "up-1",
2032
- physicalTableName: "upstream_tbl",
2033
- connectionName: "sf",
2034
- },
2035
- ],
2036
- strictUpstreams: true,
2037
- },
2038
- new AbortController().signal,
2039
- );
2040
-
2041
- // The reference manifest becomes the seed (carried) entries so a
2042
- // downstream build resolves its upstream to the existing physical table.
2043
- // arg[1] is environment (the storage build session), so the seed
2044
- // (carried) entries are arg[3]. connectionName is carried through so the
2045
- // seed can be quoted for the upstream's dialect (see quoteSeedTablePath).
2046
- expect(svc.executeInstructedBuild.firstCall.args[3]).toEqual({
2047
- "up-1": {
2048
- sourceEntityId: "up-1",
2049
- physicalTableName: "upstream_tbl",
2050
- connectionName: "sf",
2051
- },
2052
- });
2053
- // strictUpstreams flows through as the strict flag (6th arg).
2054
- expect(svc.executeInstructedBuild.firstCall.args[5]).toBe(true);
2055
- // Reused upstreams are counted as carried, not built.
2056
- expect(svc.commitManifest.firstCall.args[2]).toMatchObject({
2057
- mode: "orchestrated",
2058
- sourcesBuilt: 1,
2059
- sourcesReused: 1,
2060
- });
2061
- });
2062
-
2063
- it("auto-run: derives instructions and auto-loads the manifest", async () => {
2064
- const svc = stubEngine();
2065
-
2066
- await svc.runBuild(
2067
- "mat-1",
2068
- "my-env",
2069
- "pkg",
2070
- {
2071
- sourceNames: undefined,
2072
- forceRefresh: true,
2073
- buildInstructions: undefined,
2074
- },
2075
- new AbortController().signal,
2076
- );
2077
-
2078
- expect(svc.commitManifest.firstCall.args[2]).toMatchObject({
2079
- mode: "auto",
2080
- });
2081
- // Auto-run owns distribution: it loads the fresh manifest into the models.
2082
- expect(svc.autoLoadManifest.calledOnce).toBe(true);
2083
- });
2084
- });
2085
-
2086
- describe("autoLoadManifest", () => {
2087
- let ctx: ReturnType<typeof createMocks>;
2088
- beforeEach(() => {
2089
- ctx = createMocks();
2090
- });
2091
-
2092
- it("loads only entries with a physical table into the package models", async () => {
2093
- const reloadAllModelsForPackage = sinon.stub().resolves();
2094
- const entries = {
2095
- b1: {
2096
- sourceEntityId: "b1",
2097
- physicalTableName: "orders_v1",
2098
- connectionName: "duckdb",
2099
- },
2100
- b2: { sourceEntityId: "b2", physicalTableName: undefined },
2101
- };
2102
-
2103
- await (
2104
- ctx.service as unknown as {
2105
- autoLoadManifest: (
2106
- env: { reloadAllModelsForPackage: sinon.SinonStub },
2107
- pkg: string,
2108
- entries: Record<string, unknown>,
2109
- ) => Promise<void>;
2110
- }
2111
- ).autoLoadManifest({ reloadAllModelsForPackage }, "pkg", entries);
2112
-
2113
- expect(reloadAllModelsForPackage.calledOnce).toBe(true);
2114
- // connectionName is carried alongside tableName so the bind step
2115
- // (Package.quoteBoundTableNames) can quote the path for its dialect.
2116
- expect(reloadAllModelsForPackage.firstCall.args).toEqual([
2117
- "pkg",
2118
- { b1: { tableName: "orders_v1", connectionName: "duckdb" } },
2119
- ]);
2120
- });
2121
-
2122
- it("swallows a reload failure (best-effort; run already succeeded)", async () => {
2123
- const reloadAllModelsForPackage = sinon
2124
- .stub()
2125
- .rejects(new Error("reload boom"));
2126
- await expect(
2127
- (
2128
- ctx.service as unknown as {
2129
- autoLoadManifest: (
2130
- env: { reloadAllModelsForPackage: sinon.SinonStub },
2131
- pkg: string,
2132
- entries: Record<string, unknown>,
2133
- ) => Promise<void>;
2134
- }
2135
- ).autoLoadManifest({ reloadAllModelsForPackage }, "pkg", {
2136
- b1: { sourceEntityId: "b1", physicalTableName: "t" },
2137
- }),
2138
- ).resolves.toBeUndefined();
2139
- });
2140
- });
2141
-
2142
- describe("runInBackground (terminal recording)", () => {
2143
- let ctx: ReturnType<typeof createMocks>;
2144
- beforeEach(() => {
2145
- ctx = createMocks();
2146
- });
2147
-
2148
- const flush = () => new Promise((r) => setTimeout(r, 20));
2149
-
2150
- function background(): {
2151
- runInBackground: (
2152
- id: string,
2153
- run: (signal: AbortSignal) => Promise<void>,
2154
- ) => void;
2155
- runningAbortControllers: Map<string, AbortController>;
2156
- } {
2157
- return ctx.service as unknown as {
2158
- runInBackground: (
2159
- id: string,
2160
- run: (signal: AbortSignal) => Promise<void>,
2161
- ) => void;
2162
- runningAbortControllers: Map<string, AbortController>;
2163
- };
2164
- }
2165
-
2166
- it("records FAILED with the error message when the run rejects", async () => {
2167
- background().runInBackground("bg-1", async () => {
2168
- throw new Error("boom");
2169
- });
2170
- await flush();
2171
-
2172
- expect(ctx.repository.updateMaterialization.calledOnce).toBe(true);
2173
- expect(ctx.repository.updateMaterialization.firstCall.args[0]).toBe(
2174
- "bg-1",
2175
- );
2176
- expect(
2177
- ctx.repository.updateMaterialization.firstCall.args[1],
2178
- ).toMatchObject({ status: "FAILED", error: "boom" });
2179
- });
2180
-
2181
- it("records CANCELLED when the run aborts cooperatively", async () => {
2182
- const bg = background();
2183
- bg.runInBackground("bg-2", async () => {
2184
- bg.runningAbortControllers.get("bg-2")!.abort();
2185
- throw new Error("boom");
2186
- });
2187
- await flush();
2188
-
2189
- expect(
2190
- ctx.repository.updateMaterialization.firstCall.args[1],
2191
- ).toMatchObject({ status: "CANCELLED", error: "Cancelled" });
2192
- });
2193
-
2194
- it("clears the abort controller once the run settles", async () => {
2195
- const bg = background();
2196
- bg.runInBackground("bg-3", async () => {});
2197
- await flush();
2198
- expect(bg.runningAbortControllers.has("bg-3")).toBe(false);
2199
- });
2200
- });
2201
-
2202
- describe("transition (state machine)", () => {
2203
- let ctx: ReturnType<typeof createMocks>;
2204
- beforeEach(() => {
2205
- ctx = createMocks();
2206
- });
2207
-
2208
- function transition(): (
2209
- id: string,
2210
- next: MaterializationStatus,
2211
- ) => Promise<unknown> {
2212
- return (
2213
- ctx.service as unknown as {
2214
- transition: (
2215
- id: string,
2216
- next: MaterializationStatus,
2217
- ) => Promise<unknown>;
2218
- }
2219
- ).transition.bind(ctx.service);
2220
- }
2221
-
2222
- it("throws MaterializationNotFoundError when the record is gone", async () => {
2223
- ctx.repository.getMaterializationById.resolves(undefined);
2224
- await expect(
2225
- transition()("mat-1", "MANIFEST_ROWS_READY"),
2226
- ).rejects.toThrow(MaterializationNotFoundError);
2227
- });
2228
-
2229
- it("rejects a transition the state machine disallows", async () => {
2230
- ctx.repository.getMaterializationById.resolves(
2231
- makeMaterialization({ status: "MANIFEST_FILE_READY" }),
2232
- );
2233
- // MANIFEST_FILE_READY is terminal; nothing follows it.
2234
- await expect(transition()("mat-1", "PENDING")).rejects.toThrow(
2235
- InvalidStateTransitionError,
2236
- );
2237
- expect(ctx.repository.updateMaterialization.called).toBe(false);
2238
- });
2239
-
2240
- it("persists a valid transition", async () => {
2241
- ctx.repository.getMaterializationById.resolves(
2242
- makeMaterialization({ status: "PENDING" }),
2243
- );
2244
- ctx.repository.updateMaterialization.resolves(
2245
- makeMaterialization({ status: "MANIFEST_ROWS_READY" }),
2246
- );
2247
- await transition()("mat-1", "MANIFEST_ROWS_READY");
2248
- expect(
2249
- ctx.repository.updateMaterialization.calledOnceWith("mat-1", {
2250
- status: "MANIFEST_ROWS_READY",
2251
- }),
2252
- ).toBe(true);
2253
- });
2254
- });
2255
-
2256
- // The only path here that issues a DROP against a customer's warehouse, so its
2257
- // guards get assertions rather than a trace. Scenario 64 covers it end to end,
2258
- // but hammer does not run in CI — and the failure mode is silent data loss, not
2259
- // a broken build.
2260
- describe("reclaimStorageTablesFromFailedRun", () => {
2261
- let ctx: ReturnType<typeof createMocks>;
2262
- let infoLog: sinon.SinonStub;
2263
- let warnLog: sinon.SinonStub;
2264
-
2265
- beforeEach(() => {
2266
- ctx = createMocks();
2267
- infoLog = ctx.sandbox.stub(logger, "info");
2268
- warnLog = ctx.sandbox.stub(logger, "warn");
2269
- });
2270
- afterEach(() => ctx.sandbox.restore());
2271
-
2272
- const entry = (table: string) =>
2273
- ({
2274
- sourceEntityId: `eid-${table}`,
2275
- sourceName: "daily",
2276
- physicalTableName: table,
2277
- storageConnectionName: "lake",
2278
- }) as unknown as Parameters<
2279
- MaterializationService["reclaimStorageTablesFromFailedRun"]
2280
- >[0][number];
2281
-
2282
- /** A destination whose drop would fail loudly if one were ever attempted. */
2283
- const environment = {
2284
- getApiConnection: () => ({ name: "lake", type: "duckdb" }),
2285
- getEnvironmentPath: () => "/test",
2286
- } as unknown as Parameters<
2287
- MaterializationService["reclaimStorageTablesFromFailedRun"]
2288
- >[1];
2289
-
2290
- const reclaim = (
2291
- built: ReturnType<typeof entry>[],
2292
- others: unknown[],
2293
- ): Promise<void> => {
2294
- (ctx.repository.listMaterializations as sinon.SinonStub).resolves(others);
2295
- return (
2296
- ctx.service as unknown as {
2297
- reclaimStorageTablesFromFailedRun: (
2298
- b: unknown,
2299
- e: unknown,
2300
- o: unknown,
2301
- ) => Promise<void>;
2302
- }
2303
- ).reclaimStorageTablesFromFailedRun(built, environment, {
2304
- environmentId: "env-1",
2305
- packageName: "pkg",
2306
- });
2307
- };
2308
-
2309
- const said = (stub: sinon.SinonStub, needle: RegExp) =>
2310
- stub.getCalls().some((c) => needle.test(String(c.args[0])));
2311
-
2312
- it("never drops a table a live manifest still serves", async () => {
2313
- // The carried-forward case: a previous successful run's manifest names the
2314
- // same destination + table. Dropping it would delete a table that is being
2315
- // served right now.
2316
- await reclaim(
2317
- [entry("daily_v1")],
2318
- [
2319
- {
2320
- status: "MANIFEST_FILE_READY",
2321
- manifest: {
2322
- entries: {
2323
- other: {
2324
- storageConnectionName: "lake",
2325
- physicalTableName: "daily_v1",
2326
- },
2327
- },
2328
- },
2329
- },
2330
- ],
2331
- );
2332
-
2333
- expect(said(infoLog, /Keeping a table from a failed run/)).toBe(true);
2334
- expect(said(infoLog, /Reclaimed a table/)).toBe(false);
2335
- expect(said(warnLog, /Failed to reclaim/)).toBe(false);
2336
- });
2337
-
2338
- it("does attempt the drop when nothing references the table", async () => {
2339
- // The inverse, so the test above is known to be the still-referenced check
2340
- // doing the work rather than the reclaim being inert. No real destination
2341
- // exists here, so the attempt surfaces as the drop-failure warning.
2342
- await reclaim([entry("daily_v2")], []);
2343
-
2344
- expect(said(infoLog, /Keeping a table from a failed run/)).toBe(false);
2345
- expect(
2346
- said(warnLog, /Failed to reclaim/) ||
2347
- said(infoLog, /Reclaimed a table/),
2348
- ).toBe(true);
2349
- });
2350
-
2351
- it("ignores a superseded run's entries when deciding", async () => {
2352
- // Only MANIFEST_FILE_READY runs count as live. A superseded run naming the
2353
- // same table must not pin it forever.
2354
- await reclaim(
2355
- [entry("daily_v3")],
2356
- [
2357
- {
2358
- status: "SUPERSEDED",
2359
- manifest: {
2360
- entries: {
2361
- old: {
2362
- storageConnectionName: "lake",
2363
- physicalTableName: "daily_v3",
2364
- },
2365
- },
2366
- },
2367
- },
2368
- ],
2369
- );
2370
-
2371
- expect(said(infoLog, /Keeping a table from a failed run/)).toBe(false);
2372
- });
2373
- });