@malloy-publisher/server 0.0.235 → 0.0.237

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (319) hide show
  1. package/dist/app/api-doc.yaml +83 -27
  2. package/dist/app/assets/{EnvironmentPage-BsAnavYN.js → EnvironmentPage-DaP6CpgK.js} +1 -1
  3. package/dist/app/assets/{HomePage-CADE138j.js → HomePage-C9yNw3iu.js} +1 -1
  4. package/dist/app/assets/{LightMode-Cfh7KzN8.js → LightMode-Be4-G-dX.js} +1 -1
  5. package/dist/app/assets/{MainPage-CO3pRlnV.js → MainPage-J_rU2SgI.js} +2 -2
  6. package/dist/app/assets/{MaterializationsPage-p9YjkRXZ.js → MaterializationsPage-DHgPI4MO.js} +1 -1
  7. package/dist/app/assets/ModelPage-BzXfjTu5.js +1 -0
  8. package/dist/app/assets/{PackagePage-e4kN75YR.js → PackagePage-ByoSTLJ4.js} +1 -1
  9. package/dist/app/assets/RouteError-CrSeKN2r.js +1 -0
  10. package/dist/app/assets/{ThemeEditorPage-CciagFTq.js → ThemeEditorPage-Btxnmhtk.js} +1 -1
  11. package/dist/app/assets/{WorkbookPage-DNWmkCXa.js → WorkbookPage-D6JLPnWO.js} +1 -1
  12. package/dist/app/assets/{core-Rj_4rRnA.es-BZyvITuO.js → core-BSRgt1mG.es-Bc_D2Lq4.js} +1 -1
  13. package/dist/app/assets/{index-CH2AcDzc.js → index-BRijWj6K.js} +15 -15
  14. package/dist/app/assets/{index-DQa463gC.js → index-BX4D94xw.js} +1 -1
  15. package/dist/app/assets/{index-VBbcc8s6.js → index-Cvy0xtb7.js} +1 -1
  16. package/dist/app/assets/{index-DQpV7MyA.js → index-DMAhIJNC.js} +1 -1
  17. package/dist/app/index.html +1 -1
  18. package/dist/package_load_worker.mjs +59 -20
  19. package/dist/server.mjs +1342 -616
  20. package/package.json +8 -2
  21. package/.eslintrc.json +0 -44
  22. package/.prettierignore +0 -1
  23. package/.prettierrc +0 -4
  24. package/build.ts +0 -97
  25. package/dist/app/assets/ModelPage-C1OSTv-x.js +0 -1
  26. package/dist/app/assets/RouteError-CzbfOkng.js +0 -1
  27. package/eslint.config.mjs +0 -76
  28. package/k6-tests/README.md +0 -144
  29. package/k6-tests/bun.lock +0 -797
  30. package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
  31. package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
  32. package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
  33. package/k6-tests/load-test/load-test-crud.ts +0 -142
  34. package/k6-tests/load-test/load-test.ts +0 -784
  35. package/k6-tests/package.json +0 -27
  36. package/k6-tests/scripts/run-with-otel.sh +0 -50
  37. package/k6-tests/smoke-test/smoke-test.ts +0 -158
  38. package/k6-tests/tsconfig.json +0 -29
  39. package/k6-tests/utils/client_factory.ts +0 -128
  40. package/k6-tests/utils/common.ts +0 -489
  41. package/k6-tests/utils/logger.ts +0 -62
  42. package/k6-tests/utils/types.d.ts +0 -35
  43. package/malloy_mcp.dxt +0 -0
  44. package/publisher.config.example.bigquery.json +0 -33
  45. package/publisher.config.example.duckdb.json +0 -23
  46. package/publisher.config.json +0 -23
  47. package/scripts/bake-duckdb-extensions.js +0 -110
  48. package/src/config.spec.ts +0 -1535
  49. package/src/config.theme.spec.ts +0 -178
  50. package/src/config.ts +0 -1142
  51. package/src/constants.ts +0 -105
  52. package/src/controller/compile.controller.ts +0 -40
  53. package/src/controller/connection.controller.spec.ts +0 -849
  54. package/src/controller/connection.controller.ts +0 -901
  55. package/src/controller/database.controller.ts +0 -24
  56. package/src/controller/materialization.controller.spec.ts +0 -293
  57. package/src/controller/materialization.controller.ts +0 -319
  58. package/src/controller/model.controller.ts +0 -164
  59. package/src/controller/package.controller.spec.ts +0 -273
  60. package/src/controller/package.controller.ts +0 -306
  61. package/src/controller/query.controller.ts +0 -152
  62. package/src/controller/theme.controller.ts +0 -83
  63. package/src/controller/watch-mode.controller.ts +0 -243
  64. package/src/data_styles.ts +0 -71
  65. package/src/default-publisher.config.json +0 -23
  66. package/src/ducklake_version.spec.ts +0 -163
  67. package/src/ducklake_version.ts +0 -153
  68. package/src/errors.spec.ts +0 -119
  69. package/src/errors.ts +0 -237
  70. package/src/filter_deprecation.spec.ts +0 -64
  71. package/src/filter_deprecation.ts +0 -42
  72. package/src/health.spec.ts +0 -123
  73. package/src/health.ts +0 -233
  74. package/src/heap_check.spec.ts +0 -144
  75. package/src/heap_check.ts +0 -144
  76. package/src/instrumentation.ts +0 -214
  77. package/src/json_utils.spec.ts +0 -51
  78. package/src/json_utils.ts +0 -33
  79. package/src/logger.spec.ts +0 -210
  80. package/src/logger.ts +0 -251
  81. package/src/malloy_pin_prereqs.spec.ts +0 -25
  82. package/src/materialization_metrics.spec.ts +0 -169
  83. package/src/materialization_metrics.ts +0 -346
  84. package/src/mcp/error_messages.spec.ts +0 -35
  85. package/src/mcp/error_messages.ts +0 -183
  86. package/src/mcp/handler_utils.spec.ts +0 -108
  87. package/src/mcp/handler_utils.ts +0 -251
  88. package/src/mcp/mcp_constants.ts +0 -6
  89. package/src/mcp/query_envelope.spec.ts +0 -229
  90. package/src/mcp/query_envelope.ts +0 -240
  91. package/src/mcp/server.protocol.spec.ts +0 -250
  92. package/src/mcp/server.ts +0 -87
  93. package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
  94. package/src/mcp/skills/build_skills_bundle.ts +0 -176
  95. package/src/mcp/skills/skills_bundle.json +0 -1
  96. package/src/mcp/skills/skills_bundle.spec.ts +0 -171
  97. package/src/mcp/tool_response.spec.ts +0 -108
  98. package/src/mcp/tool_response.ts +0 -138
  99. package/src/mcp/tools/compile_tool.spec.ts +0 -315
  100. package/src/mcp/tools/compile_tool.ts +0 -208
  101. package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
  102. package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
  103. package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
  104. package/src/mcp/tools/docs_search_tool.ts +0 -128
  105. package/src/mcp/tools/embedding_index.spec.ts +0 -1236
  106. package/src/mcp/tools/embedding_index.ts +0 -808
  107. package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
  108. package/src/mcp/tools/execute_query_tool.ts +0 -316
  109. package/src/mcp/tools/get_context_eval.ts +0 -275
  110. package/src/mcp/tools/get_context_tool.spec.ts +0 -592
  111. package/src/mcp/tools/get_context_tool.ts +0 -585
  112. package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
  113. package/src/mcp/tools/reload_package_tool.ts +0 -132
  114. package/src/mcp_config.spec.ts +0 -919
  115. package/src/mcp_config.ts +0 -425
  116. package/src/oom_guards.integration.spec.ts +0 -269
  117. package/src/package_load/package_load_pool.spec.ts +0 -252
  118. package/src/package_load/package_load_pool.ts +0 -931
  119. package/src/package_load/package_load_worker.ts +0 -1034
  120. package/src/package_load/protocol.ts +0 -364
  121. package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
  122. package/src/package_load/rpc_wait_accountant.ts +0 -76
  123. package/src/package_load_metrics.spec.ts +0 -114
  124. package/src/package_load_metrics.ts +0 -127
  125. package/src/path_safety.spec.ts +0 -158
  126. package/src/path_safety.ts +0 -146
  127. package/src/pg_helpers.spec.ts +0 -223
  128. package/src/pg_helpers.ts +0 -52
  129. package/src/query_cap_metrics.spec.ts +0 -89
  130. package/src/query_cap_metrics.ts +0 -116
  131. package/src/query_concurrency.spec.ts +0 -247
  132. package/src/query_concurrency.ts +0 -237
  133. package/src/query_metadata_metrics.ts +0 -49
  134. package/src/query_param_utils.ts +0 -29
  135. package/src/query_timeout.spec.ts +0 -224
  136. package/src/query_timeout.ts +0 -179
  137. package/src/runtime/publisher.js +0 -328
  138. package/src/server-old.ts +0 -988
  139. package/src/server.ts +0 -2059
  140. package/src/service/annotations.spec.ts +0 -118
  141. package/src/service/annotations.ts +0 -91
  142. package/src/service/authorize.spec.ts +0 -154
  143. package/src/service/authorize.ts +0 -498
  144. package/src/service/authorize_integration.spec.ts +0 -2041
  145. package/src/service/build_plan.spec.ts +0 -685
  146. package/src/service/build_plan.ts +0 -797
  147. package/src/service/compile_authorize.spec.ts +0 -85
  148. package/src/service/compile_fragment_techniques.spec.ts +0 -156
  149. package/src/service/connection.spec.ts +0 -2360
  150. package/src/service/connection.ts +0 -2187
  151. package/src/service/connection_config.spec.ts +0 -734
  152. package/src/service/connection_config.ts +0 -899
  153. package/src/service/connection_federation.spec.ts +0 -184
  154. package/src/service/connection_fingerprint.spec.ts +0 -102
  155. package/src/service/connection_service.spec.ts +0 -711
  156. package/src/service/connection_service.ts +0 -291
  157. package/src/service/cron_evaluator.spec.ts +0 -79
  158. package/src/service/cron_evaluator.ts +0 -105
  159. package/src/service/db_utils.spec.ts +0 -1085
  160. package/src/service/db_utils.ts +0 -1502
  161. package/src/service/duckdb_instance_isolation.spec.ts +0 -137
  162. package/src/service/ducklake_lazy_attach.spec.ts +0 -110
  163. package/src/service/embedding_provider.spec.ts +0 -329
  164. package/src/service/embedding_provider.ts +0 -236
  165. package/src/service/environment.ts +0 -2191
  166. package/src/service/environment_admission.spec.ts +0 -344
  167. package/src/service/environment_compile.spec.ts +0 -197
  168. package/src/service/environment_store.spec.ts +0 -2274
  169. package/src/service/environment_store.ts +0 -2434
  170. package/src/service/environment_store_anchoring.spec.ts +0 -107
  171. package/src/service/environment_store_clone.spec.ts +0 -350
  172. package/src/service/explore_visibility.spec.ts +0 -434
  173. package/src/service/exports_probe.spec.ts +0 -107
  174. package/src/service/extension_fetch_policy.spec.ts +0 -256
  175. package/src/service/filter.spec.ts +0 -458
  176. package/src/service/filter.ts +0 -341
  177. package/src/service/filter_bypass.spec.ts +0 -418
  178. package/src/service/filter_integration.spec.ts +0 -1004
  179. package/src/service/freshness.spec.ts +0 -183
  180. package/src/service/freshness.ts +0 -112
  181. package/src/service/gcs_s3_utils.ts +0 -427
  182. package/src/service/given.ts +0 -105
  183. package/src/service/givens_integration.spec.ts +0 -219
  184. package/src/service/manifest_loader.spec.ts +0 -189
  185. package/src/service/manifest_loader.ts +0 -156
  186. package/src/service/materialization_build_session.spec.ts +0 -435
  187. package/src/service/materialization_build_session.ts +0 -681
  188. package/src/service/materialization_config_validation.spec.ts +0 -99
  189. package/src/service/materialization_config_validation.ts +0 -120
  190. package/src/service/materialization_eligibility.spec.ts +0 -158
  191. package/src/service/materialization_eligibility.ts +0 -305
  192. package/src/service/materialization_schedule_surface.spec.ts +0 -256
  193. package/src/service/materialization_scheduler.spec.ts +0 -289
  194. package/src/service/materialization_scheduler.ts +0 -304
  195. package/src/service/materialization_serve_transform.spec.ts +0 -1003
  196. package/src/service/materialization_serve_transform.ts +0 -779
  197. package/src/service/materialization_service.spec.ts +0 -2373
  198. package/src/service/materialization_service.ts +0 -2342
  199. package/src/service/materialization_test_fixtures.ts +0 -249
  200. package/src/service/model.spec.ts +0 -1157
  201. package/src/service/model.ts +0 -3370
  202. package/src/service/model_limits.spec.ts +0 -209
  203. package/src/service/model_limits.ts +0 -131
  204. package/src/service/model_storage_serve.spec.ts +0 -193
  205. package/src/service/model_storage_serve_joins.spec.ts +0 -193
  206. package/src/service/package.spec.ts +0 -612
  207. package/src/service/package.ts +0 -1799
  208. package/src/service/package_manifest.spec.ts +0 -230
  209. package/src/service/package_manifest.ts +0 -241
  210. package/src/service/package_memory_governor.spec.ts +0 -173
  211. package/src/service/package_memory_governor.ts +0 -233
  212. package/src/service/package_quote_bound_tables.spec.ts +0 -171
  213. package/src/service/package_race.spec.ts +0 -208
  214. package/src/service/package_reload_safety.spec.ts +0 -193
  215. package/src/service/package_rollback.spec.ts +0 -190
  216. package/src/service/package_worker_path.spec.ts +0 -645
  217. package/src/service/path_injection.spec.ts +0 -39
  218. package/src/service/persist_annotation_validation.spec.ts +0 -89
  219. package/src/service/persist_annotation_validation.ts +0 -59
  220. package/src/service/persistence_policy.spec.ts +0 -374
  221. package/src/service/proxy.spec.ts +0 -414
  222. package/src/service/proxy.ts +0 -248
  223. package/src/service/query_boundary.spec.ts +0 -470
  224. package/src/service/query_metadata.spec.ts +0 -408
  225. package/src/service/query_metadata.ts +0 -492
  226. package/src/service/query_metadata_identity.spec.ts +0 -149
  227. package/src/service/query_text.spec.ts +0 -79
  228. package/src/service/query_text.ts +0 -43
  229. package/src/service/quoting.spec.ts +0 -135
  230. package/src/service/quoting.ts +0 -78
  231. package/src/service/resolve_environment.ts +0 -15
  232. package/src/service/restricted_mode.spec.ts +0 -299
  233. package/src/service/source_extraction.ts +0 -225
  234. package/src/service/theme_store.ts +0 -199
  235. package/src/storage/DatabaseInterface.ts +0 -198
  236. package/src/storage/StorageManager.ts +0 -113
  237. package/src/storage/duckdb/ConnectionRepository.ts +0 -143
  238. package/src/storage/duckdb/DuckDBConnection.ts +0 -123
  239. package/src/storage/duckdb/DuckDBRepository.ts +0 -208
  240. package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
  241. package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
  242. package/src/storage/duckdb/MaterializationRepository.ts +0 -336
  243. package/src/storage/duckdb/PackageRepository.ts +0 -135
  244. package/src/storage/duckdb/schema.ts +0 -229
  245. package/src/stream_helpers.spec.ts +0 -280
  246. package/src/stream_helpers.ts +0 -162
  247. package/src/telemetry.ts +0 -20
  248. package/src/test_helpers/metrics_harness.ts +0 -166
  249. package/src/theme_key_parity.spec.ts +0 -57
  250. package/src/utils.ts +0 -29
  251. package/tests/fixtures/authorize-compile/model.malloy +0 -9
  252. package/tests/fixtures/authorize-compile/publisher.json +0 -4
  253. package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
  254. package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
  255. package/tests/fixtures/html-pages-test/data.csv +0 -3
  256. package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
  257. package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
  258. package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
  259. package/tests/fixtures/html-pages-test/public/data.json +0 -1
  260. package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
  261. package/tests/fixtures/html-pages-test/public/index.html +0 -9
  262. package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
  263. package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
  264. package/tests/fixtures/html-pages-test/public/slides.html +0 -12
  265. package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
  266. package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
  267. package/tests/fixtures/html-pages-test/publisher.json +0 -5
  268. package/tests/fixtures/html-pages-test/report.malloy +0 -1
  269. package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
  270. package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
  271. package/tests/fixtures/persist-multi-level/publisher.json +0 -5
  272. package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
  273. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
  274. package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
  275. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
  276. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
  277. package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
  278. package/tests/fixtures/persist-test/data/orders.csv +0 -5
  279. package/tests/fixtures/persist-test/persist_test.malloy +0 -11
  280. package/tests/fixtures/persist-test/publisher.json +0 -5
  281. package/tests/fixtures/publisher.config.json +0 -15
  282. package/tests/fixtures/query-givens/data/orders.csv +0 -7
  283. package/tests/fixtures/query-givens/model.malloy +0 -34
  284. package/tests/fixtures/query-givens/publisher.json +0 -5
  285. package/tests/fixtures/xlsx/database.xlsx +0 -0
  286. package/tests/harness/mcp_test_setup.ts +0 -332
  287. package/tests/harness/rest_e2e.ts +0 -68
  288. package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
  289. package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
  290. package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
  291. package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
  292. package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
  293. package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
  294. package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
  295. package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
  296. package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
  297. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
  298. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
  299. package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
  300. package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
  301. package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
  302. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
  303. package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
  304. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
  305. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
  306. package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
  307. package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
  308. package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
  309. package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
  310. package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
  311. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
  312. package/tests/unit/controller/theme.controller.spec.ts +0 -141
  313. package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
  314. package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
  315. package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
  316. package/tests/unit/duckdb/repositories.test.ts +0 -208
  317. package/tests/unit/ducklake/ducklake.test.ts +0 -952
  318. package/tests/unit/service/theme_store.spec.ts +0 -139
  319. package/tsconfig.json +0 -32
@@ -1,2373 +0,0 @@
1
- import type { Connection as MalloyConnection } from "@malloydata/malloy";
2
- import { Manifest } from "@malloydata/malloy";
3
- import { afterEach, beforeEach, describe, expect, it } from "bun:test";
4
- import * as sinon from "sinon";
5
- import {
6
- MaterializationEligibilityError,
7
- BadRequestError,
8
- EnvironmentNotFoundError,
9
- InvalidStateTransitionError,
10
- MaterializationConflictError,
11
- MaterializationNotFoundError,
12
- } from "../errors";
13
- import {
14
- BuildInstruction,
15
- MaterializationStatus,
16
- ResourceRepository,
17
- } from "../storage/DatabaseInterface";
18
- import { DuplicateActiveMaterializationError } from "../storage/duckdb/MaterializationRepository";
19
- import { EnvironmentStore } from "./environment_store";
20
- import {
21
- compiledWith,
22
- fakeSource,
23
- makeBuildPlan,
24
- makeInstruction,
25
- makeMaterialization,
26
- } from "./materialization_test_fixtures";
27
- import {
28
- manifestExcludingStorage,
29
- MaterializationService,
30
- redactConnectionSecrets,
31
- stagingSuffix,
32
- } from "./materialization_service";
33
- import { logger } from "../logger";
34
- import { resetMaterializationTelemetryForTesting } from "../materialization_metrics";
35
- import {
36
- startMetricsHarness,
37
- type MetricsHarness,
38
- } from "../test_helpers/metrics_harness";
39
-
40
- type MockRepo = sinon.SinonStubbedInstance<ResourceRepository>;
41
-
42
- /**
43
- * Turn per-query metadata on for one test and put the environment back after.
44
- * The feature ships dark, so a test that asserts what a statement carries has
45
- * to enable it — and saying so per test keeps the default honest here: a test
46
- * that forgets is testing the shipped default, which is "nothing attached".
47
- */
48
- function withQueryMetadataOn(): void {
49
- const original = process.env.PUBLISHER_QUERY_METADATA;
50
- beforeEach(() => {
51
- process.env.PUBLISHER_QUERY_METADATA = "on";
52
- });
53
- afterEach(() => {
54
- if (original === undefined) delete process.env.PUBLISHER_QUERY_METADATA;
55
- else process.env.PUBLISHER_QUERY_METADATA = original;
56
- });
57
- }
58
-
59
- describe("redactConnectionSecrets", () => {
60
- it("strips credential values but keeps the message legible", () => {
61
- const source = {
62
- name: "orders_pg",
63
- type: "postgres",
64
- postgresConnection: {
65
- host: "db.internal",
66
- userName: "svc",
67
- password: "sup3r-secret-pw",
68
- },
69
- };
70
- const msg =
71
- "IO Error: could not connect password=sup3r-secret-pw to db.internal";
72
- const out = redactConnectionSecrets(msg, source);
73
- expect(out).not.toContain("sup3r-secret-pw");
74
- expect(out).toContain("could not connect");
75
- expect(out).toContain("db.internal"); // host isn't a secret — stays
76
- });
77
-
78
- it("passes a credential-free error (e.g. schema not found) through verbatim", () => {
79
- const source = {
80
- name: "lake",
81
- type: "ducklake",
82
- ducklakeConnection: {
83
- catalog: { postgresConnection: { password: "catpw123" } },
84
- },
85
- };
86
- const msg =
87
- "Catalog Error: Schema 'analytics' not found in DuckLake 'lake'";
88
- expect(redactConnectionSecrets(msg, source)).toBe(msg);
89
- });
90
-
91
- it("redacts a secret in its SQL-escaped form (single quotes doubled)", () => {
92
- // A DuckDB error echoes the offending statement, in which a secret with a
93
- // single quote appears escaped (`'` -> `''`), so the raw value won't match.
94
- const source = {
95
- name: "pg",
96
- type: "postgres",
97
- postgresConnection: { password: "pa'ss'word" },
98
- };
99
- const msg =
100
- "Parser Error: syntax error near 'password=pa''ss''word host=h'";
101
- const out = redactConnectionSecrets(msg, source);
102
- expect(out).not.toContain("pa''ss''word");
103
- expect(out).toContain("***");
104
- });
105
-
106
- // Stubbed at the seam because the leak needs an ATTACH (not CTAS) failure:
107
- // bad catalog creds fast-fail at connection validation, so no black-box test
108
- // can reach this branch.
109
- it("redacts an INFRA chained-build failure, and does not call it a refusal", async () => {
110
- const sandbox = sinon.createSandbox();
111
- try {
112
- const destinationConnection = {
113
- name: "lake",
114
- type: "ducklake",
115
- ducklakeConnection: {
116
- catalog: {
117
- postgresConnection: {
118
- host: "catalog.internal",
119
- userName: "lake_rw",
120
- password: "c4talog-pw",
121
- },
122
- },
123
- },
124
- };
125
- const sourceConnection = {
126
- name: "orders_pg",
127
- type: "postgres",
128
- postgresConnection: { host: "db.internal", password: "src-pw-99" },
129
- };
130
-
131
- // A failed RW ATTACH echoes the statement, connstring included.
132
- sandbox
133
- .stub(
134
- MaterializationService.prototype as unknown as Record<
135
- string,
136
- unknown
137
- >,
138
- "buildDownstreamViaParents" as never,
139
- )
140
- .rejects(
141
- new Error(
142
- "IO Error: failed to ATTACH 'ducklake:postgres:host=catalog.internal " +
143
- "user=lake_rw password=c4talog-pw' (db lake)",
144
- ),
145
- );
146
-
147
- const { service } = createMocks();
148
- const environment = {
149
- getApiConnection: (name: string) =>
150
- name === "lake" ? destinationConnection : sourceConnection,
151
- getEnvironmentPath: () => "/tmp/env",
152
- };
153
-
154
- const call = (
155
- service as unknown as {
156
- buildOneSourceIntoStorage: (...a: unknown[]) => Promise<unknown>;
157
- }
158
- ).buildOneSourceIntoStorage(
159
- { name: "orders_by_month", connectionName: "orders_pg" },
160
- {
161
- sourceEntityId: "sid-1",
162
- physicalTableName: "mz_orders_by_month",
163
- destination: "lake",
164
- },
165
- // strict: throw instead of falling through to recompute-from-raw.
166
- { strict: true, update: () => {} },
167
- environment,
168
- "SELECT 1",
169
- {},
170
- true, // dependsOnStorageUpstream — take the chained path
171
- );
172
-
173
- // A failed ATTACH is infrastructure, not a shape limit: it must NOT
174
- // present as the strict-upstreams refusal (that message means "we could
175
- // have recomputed from raw but you forbade it", which is not what
176
- // happened here).
177
- await expect(call).rejects.toThrow(
178
- /Failed to materialize chained source/i,
179
- );
180
- await expect(call).rejects.not.toThrow(/strict upstreams forbid/i);
181
- const message = await call.then(
182
- () => "",
183
- (e: unknown) => (e instanceof Error ? e.message : String(e)),
184
- );
185
- expect(message).not.toContain("c4talog-pw");
186
- expect(message).toContain("***");
187
- // Redacted but still legible.
188
- expect(message).toContain("failed to ATTACH");
189
- expect(message).toContain("catalog.internal");
190
- expect(message).toContain("orders_by_month");
191
- } finally {
192
- sandbox.restore();
193
- }
194
- });
195
- });
196
-
197
- function createMocks() {
198
- const sandbox = sinon.createSandbox();
199
-
200
- const repository = {
201
- getEnvironmentByName: sandbox.stub(),
202
- listMaterializations: sandbox.stub(),
203
- getMaterializationById: sandbox.stub(),
204
- getActiveMaterialization: sandbox.stub(),
205
- createMaterialization: sandbox.stub(),
206
- updateMaterialization: sandbox.stub(),
207
- deleteMaterialization: sandbox.stub(),
208
- } as unknown as MockRepo;
209
-
210
- const storageManager = { getRepository: () => repository };
211
-
212
- const environmentStore = {
213
- storageManager,
214
- getEnvironment: sandbox.stub(),
215
- } as unknown as EnvironmentStore;
216
-
217
- // Default: environment resolves and yields a package whose build plan is
218
- // empty (no persist sources). Individual tests override getBuildPlan to
219
- // exercise the orchestrated path. getModelPaths()=>[] keeps the background
220
- // build from touching the Malloy runtime.
221
- repository.getEnvironmentByName.resolves({
222
- id: "env-1",
223
- name: "my-env",
224
- path: "/test",
225
- createdAt: new Date(),
226
- updatedAt: new Date(),
227
- });
228
- setPackage(environmentStore, { getBuildPlan: () => null });
229
-
230
- const service = new MaterializationService(environmentStore);
231
-
232
- return { sandbox, repository, environmentStore, service };
233
- }
234
-
235
- /** Point environmentStore.getEnvironment at a package with the given overrides. */
236
- function setPackage(
237
- environmentStore: EnvironmentStore,
238
- pkgOverrides: Record<string, unknown>,
239
- ): void {
240
- const pkg = {
241
- getBuildPlan: () => null,
242
- getModelPaths: () => [],
243
- getPackagePath: () => "/test",
244
- getMalloyConfig: () => ({}),
245
- getMalloyConnection: async () => ({}),
246
- // No bound manifestLocation by default, so seedFromBoundManifest no-ops.
247
- getPackageMetadata: () => ({}),
248
- ...pkgOverrides,
249
- };
250
- (environmentStore.getEnvironment as sinon.SinonStub).resolves({
251
- getPackage: sinon.stub().resolves(pkg),
252
- withPackageLock: async (_name: string, fn: () => Promise<unknown>) =>
253
- fn(),
254
- reloadAllModelsForPackage: sinon.stub().resolves(),
255
- });
256
- }
257
-
258
- describe("MaterializationService", () => {
259
- let ctx: ReturnType<typeof createMocks>;
260
-
261
- beforeEach(() => {
262
- ctx = createMocks();
263
- });
264
-
265
- describe("resolveEnvironmentId", () => {
266
- it("throws EnvironmentNotFoundError when the environment is missing", async () => {
267
- ctx.repository.getEnvironmentByName.resolves(null);
268
- await expect(
269
- ctx.service.listMaterializations("unknown", "pkg"),
270
- ).rejects.toThrow(EnvironmentNotFoundError);
271
- });
272
- });
273
-
274
- describe("deleteMaterialization drop of a storage= table", () => {
275
- it("routes a storage entry to a destination-aware RW drop (best-effort, delete still completes)", async () => {
276
- // getApiConnection returns a non-storage type so the RW drop refuses
277
- // at its destination gate (pre-session), exercising the routing + the
278
- // best-effort failure handling without needing a live DuckDB attach.
279
- const getApiConnection = sinon
280
- .stub()
281
- .returns({ name: "lake", type: "postgres" });
282
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
283
- getPackage: sinon
284
- .stub()
285
- .resolves({ getMalloyConnection: async () => ({}) }),
286
- getApiConnection,
287
- getEnvironmentPath: () => "/test",
288
- });
289
- ctx.repository.getMaterializationById.resolves(
290
- makeMaterialization({
291
- status: "MANIFEST_FILE_READY",
292
- manifest: {
293
- entries: {
294
- se1: {
295
- sourceEntityId: "se1",
296
- sourceName: "daily",
297
- physicalTableName: "daily__mabc123",
298
- connectionName: "wh",
299
- storageConnectionName: "lake",
300
- },
301
- },
302
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
303
- } as any,
304
- }),
305
- );
306
-
307
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
308
- dropTables: true,
309
- });
310
-
311
- // Routed to the storage path (resolved the destination connection for
312
- // an RW drop, not the old skip), and the delete completed despite the
313
- // drop's best-effort failure.
314
- expect(getApiConnection.calledWith("lake")).toBe(true);
315
- expect(
316
- (
317
- ctx.repository.deleteMaterialization as sinon.SinonStub
318
- ).calledWith("mat-1"),
319
- ).toBe(true);
320
- });
321
-
322
- it("skips the drop when another MANIFEST_FILE_READY run still references the table (MED-3)", async () => {
323
- // name=-verbatim: generations of a source share one physical name, so
324
- // dropping a superseded record's table must NOT take out the table a
325
- // remaining run still serves.
326
- const getApiConnection = sinon
327
- .stub()
328
- .returns({ name: "lake", type: "duckdb" });
329
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
330
- getPackage: sinon
331
- .stub()
332
- .resolves({ getMalloyConnection: async () => ({}) }),
333
- getApiConnection,
334
- getEnvironmentPath: () => "/test",
335
- });
336
- const entries = {
337
- se1: {
338
- sourceEntityId: "se1",
339
- sourceName: "daily",
340
- physicalTableName: "daily", // shared logical name
341
- connectionName: "wh",
342
- storageConnectionName: "lake",
343
- },
344
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
345
- } as any;
346
- ctx.repository.getMaterializationById.resolves(
347
- makeMaterialization({
348
- id: "mat-1",
349
- status: "MANIFEST_FILE_READY",
350
- manifest: { entries },
351
- }),
352
- );
353
- // Another remaining successful run references the SAME lake.daily table.
354
- ctx.repository.listMaterializations.resolves([
355
- makeMaterialization({ id: "mat-1", manifest: { entries } }),
356
- makeMaterialization({
357
- id: "mat-2",
358
- status: "MANIFEST_FILE_READY",
359
- manifest: { entries },
360
- }),
361
- ]);
362
-
363
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
364
- dropTables: true,
365
- });
366
-
367
- // The drop was skipped (never resolved the destination connection), and
368
- // the record was still deleted.
369
- expect(getApiConnection.called).toBe(false);
370
- expect(
371
- (
372
- ctx.repository.deleteMaterialization as sinon.SinonStub
373
- ).calledWith("mat-1"),
374
- ).toBe(true);
375
- });
376
- });
377
-
378
- describe("deleteMaterialization rebinds serve bindings (both tiers)", () => {
379
- const storageEntry = {
380
- se1: {
381
- sourceEntityId: "se1",
382
- sourceName: "daily",
383
- physicalTableName: "daily__mabc",
384
- connectionName: "wh",
385
- storageConnectionName: "lake",
386
- schema: [{ name: "a", type: "BIGINT" }],
387
- },
388
- };
389
- // A colocated entry carries NO storageConnectionName.
390
- const colocatedEntry = {
391
- ce1: {
392
- sourceEntityId: "ce1",
393
- sourceName: "daily",
394
- physicalTableName: "daily__cabc",
395
- connectionName: "wh",
396
- },
397
- };
398
-
399
- function setupEnv(): {
400
- storageSpy: sinon.SinonStub;
401
- colocatedSpy: sinon.SinonStub;
402
- } {
403
- const storageSpy = sinon.stub().resolves();
404
- const colocatedSpy = sinon.stub().resolves();
405
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
406
- getPackage: sinon
407
- .stub()
408
- .resolves({ getMalloyConnection: async () => ({}) }),
409
- getApiConnection: () => ({ name: "lake", type: "duckdb" }),
410
- getEnvironmentPath: () => "/test",
411
- bindPackageStorageServeBindings: storageSpy,
412
- bindPackageColocatedServeManifest: colocatedSpy,
413
- });
414
- ctx.repository.getMaterializationById.resolves(
415
- makeMaterialization({
416
- status: "MANIFEST_FILE_READY",
417
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
418
- manifest: { entries: {} } as any,
419
- }),
420
- );
421
- return { storageSpy, colocatedSpy };
422
- }
423
-
424
- it("re-derives storage bindings from the next-latest materialization (on)", async () => {
425
- process.env.PERSIST_STORAGE_MODE = "on";
426
- try {
427
- const { storageSpy, colocatedSpy } = setupEnv();
428
- // The latest REMAINING successful run carries a storage entry.
429
- ctx.repository.listMaterializations.resolves([
430
- makeMaterialization({
431
- id: "m-prev",
432
- status: "MANIFEST_FILE_READY",
433
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
434
- manifest: { entries: storageEntry } as any,
435
- }),
436
- ]);
437
-
438
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
439
-
440
- expect(storageSpy.calledOnce).toBe(true);
441
- expect(Object.keys(storageSpy.firstCall.args[1])).toContain("se1");
442
- // Colocated is re-derived too (with no colocated entries ⇒ cleared).
443
- expect(colocatedSpy.calledOnce).toBe(true);
444
- expect(colocatedSpy.firstCall.args[1]).toEqual({});
445
- } finally {
446
- delete process.env.PERSIST_STORAGE_MODE;
447
- }
448
- });
449
-
450
- it("re-derives colocated bindings regardless of tier mode (off)", async () => {
451
- // PERSIST_STORAGE_MODE unset ⇒ off. Colocated is the v0 path, not gated
452
- // by the storage kill switch, so it is still re-derived; storage is not.
453
- const { storageSpy, colocatedSpy } = setupEnv();
454
- ctx.repository.listMaterializations.resolves([
455
- makeMaterialization({
456
- id: "m-prev",
457
- status: "MANIFEST_FILE_READY",
458
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
459
- manifest: { entries: colocatedEntry } as any,
460
- }),
461
- ]);
462
-
463
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
464
-
465
- expect(colocatedSpy.calledOnce).toBe(true);
466
- expect(Object.keys(colocatedSpy.firstCall.args[1])).toContain("ce1");
467
- // Storage rebind is skipped when the tier is off.
468
- expect(storageSpy.called).toBe(false);
469
- });
470
-
471
- it("clears both tiers' bindings when no successful materialization remains (on)", async () => {
472
- process.env.PERSIST_STORAGE_MODE = "on";
473
- try {
474
- const { storageSpy, colocatedSpy } = setupEnv();
475
- ctx.repository.listMaterializations.resolves([]); // none remain
476
-
477
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
478
-
479
- expect(storageSpy.calledOnce).toBe(true);
480
- expect(storageSpy.firstCall.args[1]).toEqual({}); // empty ⇒ serve live
481
- expect(colocatedSpy.calledOnce).toBe(true);
482
- expect(colocatedSpy.firstCall.args[1]).toEqual({});
483
- } finally {
484
- delete process.env.PERSIST_STORAGE_MODE;
485
- }
486
- });
487
-
488
- it("skips only the storage rebind when the tier is off (dark-ship)", async () => {
489
- const { storageSpy, colocatedSpy } = setupEnv(); // off
490
- ctx.repository.listMaterializations.resolves([]); // none remain
491
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
492
- // Storage is skipped when off; colocated still re-derives (to clear).
493
- expect(storageSpy.called).toBe(false);
494
- expect(colocatedSpy.calledOnce).toBe(true);
495
- expect(colocatedSpy.firstCall.args[1]).toEqual({});
496
- });
497
- });
498
-
499
- describe("queries", () => {
500
- it("lists materializations for a package", async () => {
501
- const rows = [
502
- makeMaterialization(),
503
- makeMaterialization({ id: "m2" }),
504
- ];
505
- ctx.repository.listMaterializations.resolves(rows);
506
- expect(
507
- await ctx.service.listMaterializations("my-env", "pkg"),
508
- ).toEqual(rows);
509
- });
510
-
511
- it("returns a specific materialization", async () => {
512
- const m = makeMaterialization();
513
- ctx.repository.getMaterializationById.resolves(m);
514
- expect(
515
- await ctx.service.getMaterialization("my-env", "pkg", "mat-1"),
516
- ).toEqual(m);
517
- });
518
-
519
- it("throws when the materialization is missing", async () => {
520
- ctx.repository.getMaterializationById.resolves(null);
521
- await expect(
522
- ctx.service.getMaterialization("my-env", "pkg", "nope"),
523
- ).rejects.toThrow(MaterializationNotFoundError);
524
- });
525
-
526
- it("throws when the materialization belongs to another package", async () => {
527
- ctx.repository.getMaterializationById.resolves(
528
- makeMaterialization({ packageName: "other" }),
529
- );
530
- await expect(
531
- ctx.service.getMaterialization("my-env", "pkg", "mat-1"),
532
- ).rejects.toThrow(MaterializationNotFoundError);
533
- });
534
- });
535
-
536
- describe("createMaterialization (auto-run)", () => {
537
- it("creates a PENDING record and persists options with mode=auto", async () => {
538
- ctx.repository.getActiveMaterialization.resolves(null);
539
- const pending = makeMaterialization({ status: "PENDING" });
540
- ctx.repository.createMaterialization.resolves(pending);
541
-
542
- const result = await ctx.service.createMaterialization(
543
- "my-env",
544
- "pkg",
545
- {
546
- forceRefresh: true,
547
- sourceNames: ["orders"],
548
- },
549
- );
550
-
551
- expect(result.status).toBe("PENDING");
552
- expect(ctx.repository.createMaterialization.firstCall.args).toEqual([
553
- "env-1",
554
- "pkg",
555
- "PENDING",
556
- {
557
- forceRefresh: true,
558
- sourceNames: ["orders"],
559
- mode: "auto",
560
- trigger: "ON_DEMAND",
561
- },
562
- ]);
563
- });
564
-
565
- it("defaults to auto mode in metadata", async () => {
566
- ctx.repository.getActiveMaterialization.resolves(null);
567
- ctx.repository.createMaterialization.resolves(
568
- makeMaterialization({ status: "PENDING" }),
569
- );
570
-
571
- await ctx.service.createMaterialization("my-env", "pkg", {});
572
-
573
- expect(ctx.repository.createMaterialization.firstCall.args[3]).toEqual(
574
- {
575
- forceRefresh: false,
576
- sourceNames: null,
577
- mode: "auto",
578
- trigger: "ON_DEMAND",
579
- },
580
- );
581
- });
582
-
583
- it("records trigger=SCHEDULER when the scheduler fires the run", async () => {
584
- ctx.repository.getActiveMaterialization.resolves(null);
585
- ctx.repository.createMaterialization.resolves(
586
- makeMaterialization({ status: "PENDING" }),
587
- );
588
-
589
- await ctx.service.createMaterialization("my-env", "pkg", {
590
- forceRefresh: true,
591
- trigger: "SCHEDULER",
592
- });
593
-
594
- expect(
595
- ctx.repository.createMaterialization.firstCall.args[3],
596
- ).toMatchObject({ mode: "auto", trigger: "SCHEDULER" });
597
- });
598
-
599
- it("rejects when an active materialization already exists", async () => {
600
- ctx.repository.getActiveMaterialization.resolves(
601
- makeMaterialization({
602
- id: "existing",
603
- status: "MANIFEST_ROWS_READY",
604
- }),
605
- );
606
- await expect(
607
- ctx.service.createMaterialization("my-env", "pkg"),
608
- ).rejects.toThrow(MaterializationConflictError);
609
- });
610
-
611
- it("translates a lost create race into a conflict", async () => {
612
- ctx.repository.getActiveMaterialization
613
- .onFirstCall()
614
- .resolves(null)
615
- .onSecondCall()
616
- .resolves(makeMaterialization({ id: "winner" }));
617
- ctx.repository.createMaterialization.rejects(
618
- new DuplicateActiveMaterializationError("env-1", "pkg"),
619
- );
620
- await expect(
621
- ctx.service.createMaterialization("my-env", "pkg"),
622
- ).rejects.toThrow(/winner/);
623
- });
624
- });
625
-
626
- describe("createMaterialization (orchestrated)", () => {
627
- it("creates a PENDING record with mode=orchestrated for valid instructions", async () => {
628
- setPackage(ctx.environmentStore, {
629
- getBuildPlan: () => makeBuildPlan(),
630
- });
631
- ctx.repository.getActiveMaterialization.resolves(null);
632
- ctx.repository.createMaterialization.resolves(
633
- makeMaterialization({ status: "PENDING" }),
634
- );
635
-
636
- const result = await ctx.service.createMaterialization(
637
- "my-env",
638
- "pkg",
639
- { buildInstructions: [makeInstruction()] },
640
- );
641
-
642
- expect(result.status).toBe("PENDING");
643
- expect(
644
- ctx.repository.createMaterialization.firstCall.args[3],
645
- ).toMatchObject({ mode: "orchestrated" });
646
- });
647
-
648
- it("rejects instructions referencing an unknown sourceEntityId before creating", async () => {
649
- setPackage(ctx.environmentStore, {
650
- getBuildPlan: () => makeBuildPlan(),
651
- });
652
- await expect(
653
- ctx.service.createMaterialization("my-env", "pkg", {
654
- buildInstructions: [
655
- makeInstruction({ sourceEntityId: "ghost" }),
656
- ],
657
- }),
658
- ).rejects.toThrow(BadRequestError);
659
- expect(ctx.repository.createMaterialization.called).toBe(false);
660
- });
661
-
662
- it("rejects SNAPSHOT realization", async () => {
663
- setPackage(ctx.environmentStore, {
664
- getBuildPlan: () => makeBuildPlan(),
665
- });
666
- await expect(
667
- ctx.service.createMaterialization("my-env", "pkg", {
668
- buildInstructions: [
669
- makeInstruction({ realization: "SNAPSHOT" }),
670
- ],
671
- }),
672
- ).rejects.toThrow(BadRequestError);
673
- });
674
-
675
- it("rejects buildInstructions when the package has no persist sources", async () => {
676
- setPackage(ctx.environmentStore, { getBuildPlan: () => null });
677
- await expect(
678
- ctx.service.createMaterialization("my-env", "pkg", {
679
- buildInstructions: [makeInstruction()],
680
- }),
681
- ).rejects.toThrow(BadRequestError);
682
- });
683
- });
684
-
685
- describe("stopMaterialization", () => {
686
- const cancellable = ["PENDING", "MANIFEST_ROWS_READY"] as const;
687
-
688
- for (const status of cancellable) {
689
- it(`cancels a ${status} materialization`, async () => {
690
- ctx.repository.getMaterializationById.resolves(
691
- makeMaterialization({ status }),
692
- );
693
- ctx.repository.updateMaterialization.resolves(
694
- makeMaterialization({ status: "CANCELLED" }),
695
- );
696
- const result = await ctx.service.stopMaterialization(
697
- "my-env",
698
- "pkg",
699
- "mat-1",
700
- );
701
- expect(result.status).toBe("CANCELLED");
702
- });
703
- }
704
-
705
- const terminal = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"] as const;
706
- for (const status of terminal) {
707
- it(`rejects stopping a ${status} materialization`, async () => {
708
- ctx.repository.getMaterializationById.resolves(
709
- makeMaterialization({ status }),
710
- );
711
- await expect(
712
- ctx.service.stopMaterialization("my-env", "pkg", "mat-1"),
713
- ).rejects.toThrow(InvalidStateTransitionError);
714
- });
715
- }
716
- });
717
-
718
- describe("deleteMaterialization", () => {
719
- const terminal = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"] as const;
720
- for (const status of terminal) {
721
- it(`deletes a ${status} materialization`, async () => {
722
- ctx.repository.getMaterializationById.resolves(
723
- makeMaterialization({ status }),
724
- );
725
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
726
- expect(
727
- ctx.repository.deleteMaterialization.calledOnceWith("mat-1"),
728
- ).toBe(true);
729
- });
730
- }
731
-
732
- const active = ["PENDING", "MANIFEST_ROWS_READY"] as const;
733
- for (const status of active) {
734
- it(`rejects deleting a ${status} materialization`, async () => {
735
- ctx.repository.getMaterializationById.resolves(
736
- makeMaterialization({ status }),
737
- );
738
- await expect(
739
- ctx.service.deleteMaterialization("my-env", "pkg", "mat-1"),
740
- ).rejects.toThrow(InvalidStateTransitionError);
741
- expect(ctx.repository.deleteMaterialization.called).toBe(false);
742
- });
743
- }
744
-
745
- it("does not drop tables by default (record-only delete)", async () => {
746
- const runSQL = sinon.stub().resolves();
747
- const getMalloyConnection = sinon.stub().resolves({ runSQL });
748
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
749
- getPackage: sinon.stub().resolves({ getMalloyConnection }),
750
- withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
751
- fn(),
752
- });
753
- ctx.repository.getMaterializationById.resolves(
754
- makeMaterialization({
755
- status: "MANIFEST_FILE_READY",
756
- manifest: {
757
- builtAt: new Date().toISOString(),
758
- strict: false,
759
- entries: {
760
- b1: {
761
- sourceEntityId: "b1",
762
- physicalTableName: "orders_mz",
763
- connectionName: "duckdb",
764
- },
765
- },
766
- },
767
- }),
768
- );
769
-
770
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
771
-
772
- expect(getMalloyConnection.called).toBe(false);
773
- expect(runSQL.called).toBe(false);
774
- expect(
775
- ctx.repository.deleteMaterialization.calledOnceWith("mat-1"),
776
- ).toBe(true);
777
- });
778
-
779
- it("drops manifest tables (and staging) when dropTables is set", async () => {
780
- const runSQL = sinon.stub().resolves();
781
- const getMalloyConnection = sinon
782
- .stub()
783
- .resolves({ runSQL, dialectName: "duckdb" });
784
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
785
- getPackage: sinon.stub().resolves({ getMalloyConnection }),
786
- withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
787
- fn(),
788
- });
789
- ctx.repository.getMaterializationById.resolves(
790
- makeMaterialization({
791
- status: "MANIFEST_FILE_READY",
792
- manifest: {
793
- builtAt: new Date().toISOString(),
794
- strict: false,
795
- entries: {
796
- b1: {
797
- sourceEntityId: "b1",
798
- physicalTableName: "orders_mz",
799
- connectionName: "duckdb",
800
- },
801
- },
802
- },
803
- }),
804
- );
805
-
806
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
807
- dropTables: true,
808
- });
809
-
810
- expect(getMalloyConnection.calledOnceWith("duckdb")).toBe(true);
811
- const dropped = runSQL.getCalls().map((c) => c.args[0] as string);
812
- expect(dropped).toContain('DROP TABLE IF EXISTS "orders_mz"');
813
- expect(dropped.some((s) => s.includes('"orders_mz_'))).toBe(true);
814
- expect(
815
- ctx.repository.deleteMaterialization.calledOnceWith("mat-1"),
816
- ).toBe(true);
817
- });
818
-
819
- it("dialect-quotes the drop DDL from the live connection (backtick + container path)", async () => {
820
- const runSQL = sinon.stub().resolves();
821
- const getMalloyConnection = sinon
822
- .stub()
823
- .resolves({ runSQL, dialectName: "standardsql" });
824
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
825
- getPackage: sinon.stub().resolves({ getMalloyConnection }),
826
- withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
827
- fn(),
828
- });
829
- ctx.repository.getMaterializationById.resolves(
830
- makeMaterialization({
831
- status: "MANIFEST_FILE_READY",
832
- manifest: {
833
- builtAt: new Date().toISOString(),
834
- strict: false,
835
- entries: {
836
- b1: {
837
- sourceEntityId: "b1",
838
- // Container path on a hyphenated BigQuery project: each
839
- // segment must be backtick-quoted independently.
840
- physicalTableName: "my-proj.ds.engaged",
841
- connectionName: "bq",
842
- },
843
- },
844
- },
845
- }),
846
- );
847
-
848
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
849
- dropTables: true,
850
- });
851
-
852
- const dropped = runSQL.getCalls().map((c) => c.args[0] as string);
853
- expect(dropped).toContain(
854
- "DROP TABLE IF EXISTS `my-proj`.`ds`.`engaged`",
855
- );
856
- });
857
-
858
- it("still deletes the record when a table drop fails", async () => {
859
- const runSQL = sinon.stub().rejects(new Error("boom"));
860
- const getMalloyConnection = sinon.stub().resolves({ runSQL });
861
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
862
- getPackage: sinon.stub().resolves({ getMalloyConnection }),
863
- withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
864
- fn(),
865
- });
866
- ctx.repository.getMaterializationById.resolves(
867
- makeMaterialization({
868
- status: "FAILED",
869
- manifest: {
870
- builtAt: new Date().toISOString(),
871
- strict: false,
872
- entries: {
873
- b1: {
874
- sourceEntityId: "b1",
875
- physicalTableName: "orders_mz",
876
- connectionName: "duckdb",
877
- },
878
- },
879
- },
880
- }),
881
- );
882
-
883
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
884
- dropTables: true,
885
- });
886
-
887
- expect(
888
- ctx.repository.deleteMaterialization.calledOnceWith("mat-1"),
889
- ).toBe(true);
890
- });
891
- });
892
- });
893
-
894
- describe("deleteMaterialization telemetry", () => {
895
- let ctx: ReturnType<typeof createMocks>;
896
- let harness: MetricsHarness;
897
- withQueryMetadataOn();
898
-
899
- beforeEach(async () => {
900
- ctx = createMocks();
901
- harness = await startMetricsHarness();
902
- // Drop cached instruments so recordDropTables re-binds to this provider.
903
- resetMaterializationTelemetryForTesting();
904
- });
905
-
906
- afterEach(async () => {
907
- resetMaterializationTelemetryForTesting();
908
- await harness.shutdown();
909
- });
910
-
911
- const DROP_COUNTER = "publisher_materialization_drop_tables_total";
912
-
913
- // A terminal materialization whose manifest names one physical table, so
914
- // dropMaterializedTables issues exactly one DROP — the emission under test.
915
- function dropTablesFixture(runSQL: sinon.SinonStub) {
916
- const getMalloyConnection = sinon
917
- .stub()
918
- .resolves({ runSQL, dialectName: "duckdb" });
919
- (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
920
- getPackage: sinon.stub().resolves({ getMalloyConnection }),
921
- withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
922
- fn(),
923
- });
924
- ctx.repository.getMaterializationById.resolves(
925
- makeMaterialization({
926
- status: "MANIFEST_FILE_READY",
927
- manifest: {
928
- builtAt: new Date().toISOString(),
929
- strict: false,
930
- entries: {
931
- b1: {
932
- sourceEntityId: "b1",
933
- physicalTableName: "orders_mz",
934
- connectionName: "duckdb",
935
- },
936
- },
937
- },
938
- }),
939
- );
940
- }
941
-
942
- it("meters a successful physical-table drop as outcome=success", async () => {
943
- dropTablesFixture(sinon.stub().resolves());
944
-
945
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
946
- dropTables: true,
947
- });
948
-
949
- expect(
950
- await harness.collectCounter(DROP_COUNTER, { outcome: "success" }),
951
- ).toBe(1);
952
- expect(
953
- await harness.collectCounter(DROP_COUNTER, { outcome: "failure" }),
954
- ).toBe(0);
955
- });
956
-
957
- it("tags the retiring drops as ops work", async () => {
958
- // A drop is warehouse work someone accounts for later, and it is the one
959
- // statement an operator goes looking for after a table disappears.
960
- const runSQL = sinon.stub().resolves();
961
- dropTablesFixture(runSQL);
962
-
963
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
964
- dropTables: true,
965
- });
966
-
967
- expect(runSQL.callCount).toBeGreaterThan(0);
968
- for (const call of runSQL.getCalls()) {
969
- expect(call.args[1]?.queryMetadata).toMatchObject({
970
- class: "ops",
971
- environment: "my-env",
972
- package: "pkg",
973
- run_id: "mat-1",
974
- });
975
- }
976
- });
977
-
978
- it("meters a failed drop as outcome=failure and still deletes the record", async () => {
979
- dropTablesFixture(sinon.stub().rejects(new Error("drop boom")));
980
-
981
- // The drop is best-effort: a failure is metered but never surfaces to the
982
- // caller, and the record is still removed.
983
- await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
984
- dropTables: true,
985
- });
986
-
987
- expect(
988
- await harness.collectCounter(DROP_COUNTER, { outcome: "failure" }),
989
- ).toBe(1);
990
- expect(ctx.repository.deleteMaterialization.calledOnceWith("mat-1")).toBe(
991
- true,
992
- );
993
- });
994
- });
995
-
996
- describe("stagingSuffix", () => {
997
- it("derives a short, stable suffix from the sourceEntityId", () => {
998
- expect(stagingSuffix("abcdef1234567890")).toBe("_abcdef123456");
999
- });
1000
- });
1001
-
1002
- describe("manifestExcludingStorage (chained-storage inline)", () => {
1003
- it("drops storage entries, keeps colocated entries with the manifest's quoting, preserves strict", () => {
1004
- // The source manifest is already dialect-quoted by the seed loop (#904);
1005
- // manifestExcludingStorage reuses that quoting for the kept entries.
1006
- const manifest = new Manifest();
1007
- manifest.strict = true;
1008
- manifest.update("wh_up", { tableName: '"wh_up_v1"' });
1009
- manifest.update("lake_up", { tableName: '"lake"."lake_up__mabc"' });
1010
- const built: Record<string, unknown> = {
1011
- wh_up: {
1012
- sourceEntityId: "wh_up",
1013
- physicalTableName: "wh_up_v1",
1014
- connectionName: "duckdb",
1015
- // no storageConnectionName -> colocated, keep it
1016
- },
1017
- lake_up: {
1018
- sourceEntityId: "lake_up",
1019
- physicalTableName: "lake_up__mabc",
1020
- connectionName: "wh",
1021
- storageConnectionName: "lake", // storage -> exclude it
1022
- },
1023
- };
1024
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
1025
- const reduced = manifestExcludingStorage(manifest, built as any) as any;
1026
- expect(Object.keys(reduced.entries)).toEqual(["wh_up"]);
1027
- // Kept with the manifest's dialect-quoted name, not the raw physical name.
1028
- expect(reduced.entries.wh_up.tableName).toBe('"wh_up_v1"');
1029
- expect(reduced.entries.lake_up).toBeUndefined();
1030
- expect(reduced.strict).toBe(true);
1031
- });
1032
- });
1033
-
1034
- describe("deriveSelfInstructions", () => {
1035
- let ctx: ReturnType<typeof createMocks>;
1036
- beforeEach(() => {
1037
- ctx = createMocks();
1038
- });
1039
-
1040
- it("carries forward unchanged sourceEntityIds and builds the rest (deduping repeats)", () => {
1041
- const compiled = compiledWith(
1042
- {
1043
- s1: fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" }),
1044
- s2: fakeSource({ name: "s2", sourceEntityId: "b2bbbbbbbbbbbbbb" }),
1045
- },
1046
- [["s1", "s2"], ["s2"]],
1047
- );
1048
- const priorEntries = {
1049
- b1aaaaaaaaaaaaaa: {
1050
- sourceEntityId: "b1aaaaaaaaaaaaaa",
1051
- physicalTableName: "s1_prev",
1052
- connectionName: "duckdb",
1053
- },
1054
- };
1055
-
1056
- const { instructions, carried } = (
1057
- ctx.service as unknown as {
1058
- deriveSelfInstructions: (
1059
- c: unknown,
1060
- n: string[] | undefined,
1061
- p: unknown,
1062
- ) => { instructions: BuildInstruction[]; carried: unknown };
1063
- }
1064
- ).deriveSelfInstructions(compiled, undefined, priorEntries);
1065
-
1066
- expect(instructions).toHaveLength(1);
1067
- expect(instructions[0].sourceEntityId).toBe("b2bbbbbbbbbbbbbb");
1068
- expect(instructions[0].physicalTableName).toBe("s2");
1069
- expect(instructions[0].realization).toBe("COPY");
1070
- expect(instructions[0].materializedTableId).toBe("local-b2bbbbbbbbbb");
1071
- expect(
1072
- (carried as Record<string, unknown>)["b1aaaaaaaaaaaaaa"],
1073
- ).toBeDefined();
1074
- });
1075
-
1076
- it("self-assigns the name= verbatim for a storage= source, exactly as a colocated build", () => {
1077
- process.env.PERSIST_STORAGE_MODE = "on";
1078
- try {
1079
- const compiled = compiledWith(
1080
- {
1081
- lake_src: fakeSource({
1082
- name: "lake_src",
1083
- sourceEntityId: "aa11aa11aa11aa11aa11",
1084
- annotationFields: { storage: "lake", name: "daily" },
1085
- }),
1086
- wh_src: fakeSource({
1087
- name: "wh_src",
1088
- sourceEntityId: "bb22bb22bb22bb22bb22",
1089
- }),
1090
- },
1091
- [["lake_src", "wh_src"]],
1092
- );
1093
- const { instructions } = (
1094
- ctx.service as unknown as {
1095
- deriveSelfInstructions: (
1096
- c: unknown,
1097
- n: string[] | undefined,
1098
- p: unknown,
1099
- ) => { instructions: BuildInstruction[] };
1100
- }
1101
- ).deriveSelfInstructions(compiled, undefined, {});
1102
- const byId = Object.fromEntries(
1103
- instructions.map((i) => [i.sourceEntityId, i]),
1104
- );
1105
- const lake = byId["aa11aa11aa11aa11aa11"];
1106
- const wh = byId["bb22bb22bb22bb22bb22"];
1107
- // storage= → the `name=` value verbatim in the destination (no
1108
- // content-hash decoration); only the destination differs from a colocated build.
1109
- expect(lake.destination).toBe("lake");
1110
- expect(lake.physicalTableName).toBe("daily");
1111
- // colocated → the plain logical (source) name, no destination.
1112
- expect(wh.destination).toBeUndefined();
1113
- expect(wh.physicalTableName).toBe("wh_src");
1114
- } finally {
1115
- delete process.env.PERSIST_STORAGE_MODE;
1116
- }
1117
- });
1118
-
1119
- it("honors the sourceNames filter (excluded sources are neither built nor carried)", () => {
1120
- const compiled = compiledWith(
1121
- {
1122
- s1: fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" }),
1123
- s2: fakeSource({ name: "s2", sourceEntityId: "b2bbbbbbbbbbbbbb" }),
1124
- },
1125
- [["s1", "s2"]],
1126
- );
1127
- const { instructions, carried } = (
1128
- ctx.service as unknown as {
1129
- deriveSelfInstructions: (
1130
- c: unknown,
1131
- n: string[] | undefined,
1132
- p: unknown,
1133
- ) => { instructions: BuildInstruction[]; carried: unknown };
1134
- }
1135
- ).deriveSelfInstructions(compiled, ["s2"], {});
1136
-
1137
- expect(instructions).toHaveLength(1);
1138
- expect(instructions[0].sourceEntityId).toBe("b2bbbbbbbbbbbbbb");
1139
- expect(Object.keys(carried as Record<string, unknown>)).toHaveLength(0);
1140
- });
1141
- });
1142
-
1143
- describe("getMostRecentManifestEntries (skip-if-unchanged)", () => {
1144
- let ctx: ReturnType<typeof createMocks>;
1145
- beforeEach(() => {
1146
- ctx = createMocks();
1147
- });
1148
-
1149
- it("returns entries from the most recent successful run, excluding the in-flight one", async () => {
1150
- const entries = {
1151
- b1: {
1152
- sourceEntityId: "b1",
1153
- physicalTableName: "orders_v1",
1154
- connectionName: "duckdb",
1155
- },
1156
- };
1157
- ctx.repository.listMaterializations.resolves([
1158
- makeMaterialization({
1159
- id: "in-flight",
1160
- status: "MANIFEST_ROWS_READY",
1161
- }),
1162
- makeMaterialization({
1163
- id: "old-success",
1164
- status: "MANIFEST_FILE_READY",
1165
- manifest: { builtAt: "t", strict: false, entries },
1166
- }),
1167
- ]);
1168
-
1169
- const result = await (
1170
- ctx.service as unknown as {
1171
- getMostRecentManifestEntries: (
1172
- e: string,
1173
- p: string,
1174
- x: string,
1175
- ) => Promise<unknown>;
1176
- }
1177
- ).getMostRecentManifestEntries("env-1", "pkg", "in-flight");
1178
-
1179
- expect(result).toEqual(entries);
1180
- });
1181
-
1182
- it("returns {} when the only successful run is the excluded one", async () => {
1183
- ctx.repository.listMaterializations.resolves([
1184
- makeMaterialization({
1185
- id: "self",
1186
- status: "MANIFEST_FILE_READY",
1187
- manifest: {
1188
- builtAt: "t",
1189
- strict: false,
1190
- entries: {
1191
- b1: { sourceEntityId: "b1", physicalTableName: "t1" },
1192
- },
1193
- },
1194
- }),
1195
- ]);
1196
-
1197
- const result = await (
1198
- ctx.service as unknown as {
1199
- getMostRecentManifestEntries: (
1200
- e: string,
1201
- p: string,
1202
- x: string,
1203
- ) => Promise<unknown>;
1204
- }
1205
- ).getMostRecentManifestEntries("env-1", "pkg", "self");
1206
-
1207
- expect(result).toEqual({});
1208
- });
1209
- });
1210
-
1211
- describe("stopMaterialization (in-flight)", () => {
1212
- let ctx: ReturnType<typeof createMocks>;
1213
- beforeEach(() => {
1214
- ctx = createMocks();
1215
- });
1216
-
1217
- it("aborts a running build cooperatively without forcing a transition", async () => {
1218
- ctx.repository.getMaterializationById.resolves(
1219
- makeMaterialization({ status: "MANIFEST_ROWS_READY" }),
1220
- );
1221
- const controller = new AbortController();
1222
- (
1223
- ctx.service as unknown as {
1224
- runningAbortControllers: Map<string, AbortController>;
1225
- }
1226
- ).runningAbortControllers.set("mat-1", controller);
1227
-
1228
- const result = await ctx.service.stopMaterialization(
1229
- "my-env",
1230
- "pkg",
1231
- "mat-1",
1232
- );
1233
-
1234
- expect(controller.signal.aborted).toBe(true);
1235
- // The background run records the terminal state; stop must not also write.
1236
- expect(ctx.repository.updateMaterialization.called).toBe(false);
1237
- expect(result.status).toBe("MANIFEST_ROWS_READY");
1238
- });
1239
- });
1240
-
1241
- describe("executeInstructedBuild", () => {
1242
- let ctx: ReturnType<typeof createMocks>;
1243
- beforeEach(() => {
1244
- ctx = createMocks();
1245
- });
1246
-
1247
- function callExecute(
1248
- compiled: unknown,
1249
- instructions: BuildInstruction[],
1250
- seed: Record<string, unknown>,
1251
- ): Promise<Record<string, { physicalTableName?: string }>> {
1252
- // A stub BuildEnvironment: only the `storage=` branch touches it, and
1253
- // these tests exercise the default in-warehouse path, so it is never read.
1254
- const environment = {
1255
- getApiConnection: () => {
1256
- throw new Error("no connection config in this test");
1257
- },
1258
- getEnvironmentPath: () => "/tmp/env",
1259
- };
1260
- return (
1261
- ctx.service as unknown as {
1262
- executeInstructedBuild: (
1263
- c: unknown,
1264
- env: unknown,
1265
- i: BuildInstruction[],
1266
- s: Record<string, unknown>,
1267
- sig: AbortSignal,
1268
- ) => Promise<Record<string, { physicalTableName?: string }>>;
1269
- }
1270
- ).executeInstructedBuild(
1271
- compiled,
1272
- environment,
1273
- instructions,
1274
- seed,
1275
- new AbortController().signal,
1276
- );
1277
- }
1278
-
1279
- it("builds instructed sources in dependency order and seeds carried entries", async () => {
1280
- const runSQL = sinon.stub().resolves();
1281
- const connection = { runSQL } as unknown as MalloyConnection;
1282
- const s1 = fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" });
1283
- const s2 = fakeSource({ name: "s2", sourceEntityId: "b2bbbbbbbbbbbbbb" });
1284
- const compiled = compiledWith(
1285
- { s1, s2 },
1286
- [["s1"], ["s2"]],
1287
- new Map([["duckdb", connection]]),
1288
- );
1289
-
1290
- const entries = await callExecute(
1291
- compiled,
1292
- [
1293
- {
1294
- sourceEntityId: "b1aaaaaaaaaaaaaa",
1295
- materializedTableId: "mt-1",
1296
- physicalTableName: "s1_v1",
1297
- realization: "COPY",
1298
- },
1299
- {
1300
- sourceEntityId: "b2bbbbbbbbbbbbbb",
1301
- materializedTableId: "mt-2",
1302
- physicalTableName: "s2_v1",
1303
- realization: "COPY",
1304
- },
1305
- ],
1306
- {
1307
- carried0: {
1308
- sourceEntityId: "carried0",
1309
- physicalTableName: "carried_tbl",
1310
- connectionName: "duckdb",
1311
- },
1312
- },
1313
- );
1314
-
1315
- // Both instructed sources built, plus the carried seed entry retained.
1316
- expect(entries["b1aaaaaaaaaaaaaa"].physicalTableName).toBe("s1_v1");
1317
- expect(entries["b2bbbbbbbbbbbbbb"].physicalTableName).toBe("s2_v1");
1318
- expect(entries["carried0"].physicalTableName).toBe("carried_tbl");
1319
- });
1320
-
1321
- it("builds an intermediate persist source nested under a root node", async () => {
1322
- // Mirrors malloy getBuildPlan(): only the terminal `root` is a graph
1323
- // node; its persist dependency `mid` is nested in dependsOn (not a node).
1324
- // Before the iterGraphSources fix, `mid` was silently never built even
1325
- // though the caller instructed it. Both must now build, `mid` first.
1326
- const runSQL = sinon.stub().resolves();
1327
- const connection = { runSQL } as unknown as MalloyConnection;
1328
- const root = fakeSource({
1329
- name: "root",
1330
- sourceEntityId: "brootaaaaaaaaaa",
1331
- });
1332
- const mid = fakeSource({
1333
- name: "mid",
1334
- sourceEntityId: "bmidbbbbbbbbbbb",
1335
- });
1336
- const compiled = {
1337
- graphs: [
1338
- {
1339
- connectionName: "duckdb",
1340
- nodes: [
1341
- [
1342
- {
1343
- sourceID: "root",
1344
- dependsOn: [{ sourceID: "mid", dependsOn: [] }],
1345
- },
1346
- ],
1347
- ],
1348
- },
1349
- ],
1350
- sources: { root, mid },
1351
- connectionDigests: { duckdb: "dig" },
1352
- connections: new Map([["duckdb", connection]]),
1353
- };
1354
-
1355
- const entries = await callExecute(
1356
- compiled,
1357
- [
1358
- {
1359
- sourceEntityId: "brootaaaaaaaaaa",
1360
- materializedTableId: "mt-r",
1361
- physicalTableName: "root_v1",
1362
- realization: "COPY",
1363
- },
1364
- {
1365
- sourceEntityId: "bmidbbbbbbbbbbb",
1366
- materializedTableId: "mt-m",
1367
- physicalTableName: "mid_v1",
1368
- realization: "COPY",
1369
- },
1370
- ],
1371
- {},
1372
- );
1373
-
1374
- expect(entries["bmidbbbbbbbbbbb"].physicalTableName).toBe("mid_v1");
1375
- expect(entries["brootaaaaaaaaaa"].physicalTableName).toBe("root_v1");
1376
- // The dependency's CREATE precedes its dependent's (dependency order).
1377
- const creates = runSQL
1378
- .getCalls()
1379
- .map((c) => c.args[0] as string)
1380
- .filter((s) => s.startsWith("CREATE TABLE"));
1381
- const midIdx = creates.findIndex((s) => s.includes("mid_v1"));
1382
- const rootIdx = creates.findIndex((s) => s.includes("root_v1"));
1383
- expect(midIdx).toBeGreaterThanOrEqual(0);
1384
- expect(rootIdx).toBeGreaterThan(midIdx);
1385
- });
1386
-
1387
- it("seeds a downstream build with the QUOTED upstream reference (case-folding dialect)", async () => {
1388
- // A carried/seeded upstream (built in a prior run or unit, e.g. via
1389
- // referenceManifest) must reach the downstream's SQL generation as the
1390
- // SAME quoted path the builder CREATEd it with; otherwise the downstream
1391
- // CREATE misses the case-preserved table on a case-folding engine
1392
- // (Snowflake). Capture the buildManifest Malloy is handed for the
1393
- // downstream to prove the seed was quoted for its connection's dialect
1394
- // before it becomes a FROM.
1395
- let seen:
1396
- | {
1397
- buildManifest?: {
1398
- entries?: Record<string, { tableName?: string }>;
1399
- };
1400
- }
1401
- | undefined;
1402
- const runSQL = sinon.stub().resolves();
1403
- const connection = {
1404
- runSQL,
1405
- dialectName: "snowflake",
1406
- } as unknown as MalloyConnection;
1407
- const down = fakeSource({
1408
- name: "down",
1409
- sourceEntityId: "bdownaaaaaaaaaa",
1410
- dialectName: "snowflake",
1411
- onGetSQL: (o) => {
1412
- seen = o as typeof seen;
1413
- },
1414
- });
1415
- const compiled = compiledWith(
1416
- { down },
1417
- [["down"]],
1418
- new Map([["duckdb", connection]]),
1419
- );
1420
-
1421
- await callExecute(
1422
- compiled,
1423
- [
1424
- {
1425
- sourceEntityId: "bdownaaaaaaaaaa",
1426
- materializedTableId: "mt-d",
1427
- physicalTableName: "down_v1",
1428
- realization: "COPY",
1429
- },
1430
- ],
1431
- {
1432
- up: {
1433
- sourceEntityId: "up",
1434
- physicalTableName: "schema.orders_base",
1435
- connectionName: "duckdb",
1436
- },
1437
- },
1438
- );
1439
-
1440
- expect(seen?.buildManifest?.entries?.up?.tableName).toBe(
1441
- '"schema"."orders_base"',
1442
- );
1443
- });
1444
-
1445
- it("seeds unquoted when the upstream's connection is absent from the build (older CP / cross-connection)", async () => {
1446
- // No connectionName on the seed (an older control plane) -> bind verbatim,
1447
- // exactly the pre-change behavior. Non-case-folding engines are unaffected;
1448
- // a case-folding engine self-signals by failing the downstream CREATE.
1449
- let seen:
1450
- | {
1451
- buildManifest?: {
1452
- entries?: Record<string, { tableName?: string }>;
1453
- };
1454
- }
1455
- | undefined;
1456
- const runSQL = sinon.stub().resolves();
1457
- const connection = {
1458
- runSQL,
1459
- dialectName: "snowflake",
1460
- } as unknown as MalloyConnection;
1461
- const down = fakeSource({
1462
- name: "down",
1463
- sourceEntityId: "bdownaaaaaaaaaa",
1464
- dialectName: "snowflake",
1465
- onGetSQL: (o) => {
1466
- seen = o as typeof seen;
1467
- },
1468
- });
1469
- const compiled = compiledWith(
1470
- { down },
1471
- [["down"]],
1472
- new Map([["duckdb", connection]]),
1473
- );
1474
-
1475
- await callExecute(
1476
- compiled,
1477
- [
1478
- {
1479
- sourceEntityId: "bdownaaaaaaaaaa",
1480
- materializedTableId: "mt-d",
1481
- physicalTableName: "down_v1",
1482
- realization: "COPY",
1483
- },
1484
- ],
1485
- { up: { sourceEntityId: "up", physicalTableName: "orders_base" } },
1486
- );
1487
-
1488
- expect(seen?.buildManifest?.entries?.up?.tableName).toBe("orders_base");
1489
- });
1490
-
1491
- it("throws when an instructed graph's connection is missing", async () => {
1492
- const s1 = fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" });
1493
- const compiled = compiledWith({ s1 }, [["s1"]], new Map());
1494
-
1495
- await expect(
1496
- callExecute(
1497
- compiled,
1498
- [
1499
- {
1500
- sourceEntityId: "b1aaaaaaaaaaaaaa",
1501
- materializedTableId: "mt-1",
1502
- physicalTableName: "s1_v1",
1503
- realization: "COPY",
1504
- },
1505
- ],
1506
- {},
1507
- ),
1508
- ).rejects.toThrow(BadRequestError);
1509
- });
1510
- });
1511
-
1512
- describe("buildOneSource", () => {
1513
- let ctx: ReturnType<typeof createMocks>;
1514
- withQueryMetadataOn();
1515
- beforeEach(() => {
1516
- ctx = createMocks();
1517
- });
1518
-
1519
- function callBuildOneSource(
1520
- connection: { runSQL: sinon.SinonStub },
1521
- physicalTableName: string,
1522
- dialectName?: string,
1523
- manifest: Manifest = new Manifest(),
1524
- metadata?: {
1525
- source?: Record<string, string>;
1526
- package?: Record<string, string>;
1527
- connection?: Record<string, string>;
1528
- context?: Record<string, string | undefined>;
1529
- },
1530
- ): Promise<{ sourceEntityId: string; physicalTableName: string }> {
1531
- const source = fakeSource({
1532
- name: "orders",
1533
- sourceEntityId: "abcdef1234567890",
1534
- sql: "SELECT * FROM t",
1535
- dialectName,
1536
- queryMetadata: metadata?.source,
1537
- });
1538
- const instruction: BuildInstruction = {
1539
- sourceEntityId: "abcdef1234567890",
1540
- materializedTableId: "mt-1",
1541
- physicalTableName,
1542
- realization: "COPY",
1543
- };
1544
- return (
1545
- ctx.service as unknown as {
1546
- buildOneSource: (
1547
- s: unknown,
1548
- i: BuildInstruction,
1549
- c: unknown,
1550
- d: Record<string, string>,
1551
- m: Manifest,
1552
- env: unknown,
1553
- built: Record<string, unknown>,
1554
- buildMetadata?: unknown,
1555
- ) => Promise<{ sourceEntityId: string; physicalTableName: string }>;
1556
- }
1557
- ).buildOneSource(
1558
- source,
1559
- instruction,
1560
- connection,
1561
- { duckdb: "dig" },
1562
- manifest,
1563
- {
1564
- getApiConnection: () => {
1565
- if (metadata?.connection) {
1566
- return { queryMetadata: metadata.connection };
1567
- }
1568
- throw new Error("no connection config in this colocated test");
1569
- },
1570
- getEnvironmentPath: () => "/tmp/env",
1571
- },
1572
- {}, // builtEntries — colocated build with no prior storage upstreams
1573
- metadata
1574
- ? {
1575
- packageMaterialization: metadata.package
1576
- ? {
1577
- schedule: null,
1578
- freshness: null,
1579
- queryMetadata: metadata.package,
1580
- }
1581
- : null,
1582
- context: metadata.context ?? { queryClass: "materialize" },
1583
- }
1584
- : undefined,
1585
- );
1586
- }
1587
-
1588
- it("tags every statement of the build with the resolved metadata", async () => {
1589
- // One unit of work: the warehouse's query history has to show the staging
1590
- // drop, the CTAS, the swap and the rename as the same build.
1591
- const runSQL = sinon.stub().resolves();
1592
- await callBuildOneSource({ runSQL }, "orders_v1", undefined, undefined, {
1593
- connection: { deployment: "eu" },
1594
- package: { team: "finance", tier: "bronze" },
1595
- source: { tier: "gold" },
1596
- context: { queryClass: "materialize", runId: "run-7" },
1597
- });
1598
-
1599
- const bags = runSQL.getCalls().map((c) => c.args[1]?.queryMetadata);
1600
- expect(bags).toHaveLength(4);
1601
- for (const bag of bags) {
1602
- expect(bag).toEqual({
1603
- deployment: "eu",
1604
- team: "finance",
1605
- // The source's own declaration wins over the package's.
1606
- tier: "gold",
1607
- class: "materialize",
1608
- run_id: "run-7",
1609
- source: "orders",
1610
- });
1611
- }
1612
- });
1613
-
1614
- it("issues untagged statements when nothing declares metadata", async () => {
1615
- // No metadata context (an internal caller) means the statements are exactly
1616
- // what they were before this existed.
1617
- const runSQL = sinon.stub().resolves();
1618
- await callBuildOneSource({ runSQL }, "orders_v1");
1619
- for (const call of runSQL.getCalls()) {
1620
- expect(call.args[1]?.queryMetadata).toBeUndefined();
1621
- }
1622
- });
1623
-
1624
- it("tags the cleanup drop after a failed build", async () => {
1625
- // The cleanup statement belongs to the same build, and it is the one an
1626
- // operator goes looking for after a failure.
1627
- const runSQL = sinon.stub();
1628
- runSQL.onCall(0).resolves();
1629
- runSQL.onCall(1).rejects(new Error("create boom"));
1630
- runSQL.onCall(2).resolves();
1631
- await expect(
1632
- callBuildOneSource({ runSQL }, "orders_v1", undefined, undefined, {
1633
- source: { team: "finance" },
1634
- context: { queryClass: "materialize" },
1635
- }),
1636
- ).rejects.toThrow("create boom");
1637
- expect(runSQL.lastCall.args[1]?.queryMetadata).toEqual({
1638
- team: "finance",
1639
- class: "materialize",
1640
- source: "orders",
1641
- });
1642
- });
1643
-
1644
- it("stages, swaps, and renames in a crash-safe order", async () => {
1645
- const runSQL = sinon.stub().resolves();
1646
- const entry = await callBuildOneSource({ runSQL }, "orders_v1");
1647
-
1648
- const sql = runSQL.getCalls().map((c) => c.args[0] as string);
1649
- expect(sql).toEqual([
1650
- 'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
1651
- 'CREATE TABLE "orders_v1_abcdef123456" AS (SELECT * FROM t)',
1652
- 'DROP TABLE IF EXISTS "orders_v1"',
1653
- 'ALTER TABLE "orders_v1_abcdef123456" RENAME TO "orders_v1"',
1654
- ]);
1655
- expect(entry.physicalTableName).toBe("orders_v1");
1656
- expect(entry.sourceEntityId).toBe("abcdef1234567890");
1657
- });
1658
-
1659
- it("drops the staging table and rethrows when the build SQL fails", async () => {
1660
- const runSQL = sinon.stub();
1661
- runSQL.onCall(0).resolves(); // initial staging drop
1662
- runSQL.onCall(1).rejects(new Error("create boom")); // CREATE TABLE AS
1663
- runSQL.onCall(2).resolves(); // cleanup staging drop
1664
- await expect(callBuildOneSource({ runSQL }, "orders_v1")).rejects.toThrow(
1665
- "create boom",
1666
- );
1667
- expect(runSQL.lastCall.args[0]).toBe(
1668
- 'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
1669
- );
1670
- });
1671
-
1672
- it("rethrows the original build error when staging cleanup also fails", async () => {
1673
- const runSQL = sinon.stub();
1674
- runSQL.onCall(0).resolves(); // initial staging drop
1675
- runSQL.onCall(1).rejects(new Error("create boom")); // CREATE TABLE AS
1676
- runSQL.onCall(2).rejects(new Error("cleanup boom")); // cleanup drop fails
1677
- // The cleanup failure is swallowed (logged as a physical leak); the
1678
- // original build error is what propagates, so the run is classified by its
1679
- // real cause rather than the cleanup noise.
1680
- await expect(callBuildOneSource({ runSQL }, "orders_v1")).rejects.toThrow(
1681
- "create boom",
1682
- );
1683
- expect(runSQL.callCount).toBe(3);
1684
- expect(runSQL.lastCall.args[0]).toBe(
1685
- 'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
1686
- );
1687
- });
1688
-
1689
- it("drops staging when the build is cancelled mid-run (no _staging orphan)", async () => {
1690
- // buildOneSource is cancellation-agnostic: a cancel surfaces as a thrown
1691
- // build SQL and takes the same staging-cleanup path as a failure, so a
1692
- // cancelled build leaves no orphaned _staging table behind.
1693
- const runSQL = sinon.stub();
1694
- runSQL.onCall(0).resolves(); // initial staging drop
1695
- runSQL.onCall(1).rejects(new Error("Build cancelled")); // aborted mid-CREATE
1696
- runSQL.onCall(2).resolves(); // cleanup staging drop
1697
- await expect(callBuildOneSource({ runSQL }, "orders_v1")).rejects.toThrow(
1698
- "Build cancelled",
1699
- );
1700
- expect(runSQL.lastCall.args[0]).toBe(
1701
- 'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
1702
- );
1703
- });
1704
-
1705
- it("quotes each segment of a container path for a backtick dialect", async () => {
1706
- const runSQL = sinon.stub().resolves();
1707
- // Container path (dataset.table) on a backtick dialect (BigQuery): each
1708
- // segment is quoted independently, and the rename targets the bare name.
1709
- const entry = await callBuildOneSource(
1710
- { runSQL },
1711
- "ds.orders_v1",
1712
- "standardsql",
1713
- );
1714
-
1715
- const sql = runSQL.getCalls().map((c) => c.args[0] as string);
1716
- expect(sql).toEqual([
1717
- "DROP TABLE IF EXISTS `ds`.`orders_v1_abcdef123456`",
1718
- "CREATE TABLE `ds`.`orders_v1_abcdef123456` AS (SELECT * FROM t)",
1719
- "DROP TABLE IF EXISTS `ds`.`orders_v1`",
1720
- "ALTER TABLE `ds`.`orders_v1_abcdef123456` RENAME TO `orders_v1`",
1721
- ]);
1722
- expect(entry.physicalTableName).toBe("ds.orders_v1");
1723
- });
1724
-
1725
- it("colocated build excludes a storage upstream from its warehouse manifest (mixed chain)", async () => {
1726
- // A colocated downstream that reads a storage-materialized
1727
- // upstream must NOT get the lake table name in its warehouse build SQL —
1728
- // the warehouse can't resolve it. The storage upstream is excluded (⇒
1729
- // inlined/recomputed); a colocated upstream is kept.
1730
- const runSQL = sinon.stub().resolves();
1731
- let seenManifest:
1732
- | { entries?: Record<string, { tableName?: string }> }
1733
- | undefined;
1734
- const down = fakeSource({
1735
- name: "down",
1736
- sourceEntityId: "downdowndowndown",
1737
- sql: "SELECT 1",
1738
- onGetSQL: (o) => {
1739
- seenManifest = (o as { buildManifest?: typeof seenManifest })
1740
- .buildManifest;
1741
- },
1742
- });
1743
- const manifest = new Manifest();
1744
- manifest.update("up_storage", { tableName: "daily__mabc" });
1745
- manifest.update("up_pathc", { tableName: '"orders_v1"' });
1746
- const builtEntries = {
1747
- up_storage: {
1748
- sourceEntityId: "up_storage",
1749
- physicalTableName: "daily__mabc",
1750
- storageConnectionName: "lake",
1751
- },
1752
- up_pathc: {
1753
- sourceEntityId: "up_pathc",
1754
- physicalTableName: "orders_v1",
1755
- connectionName: "duckdb",
1756
- },
1757
- };
1758
- await (
1759
- ctx.service as unknown as {
1760
- buildOneSource: (...a: unknown[]) => Promise<unknown>;
1761
- }
1762
- ).buildOneSource(
1763
- down,
1764
- {
1765
- sourceEntityId: "downdowndowndown",
1766
- materializedTableId: "mt",
1767
- physicalTableName: "down_v1",
1768
- realization: "COPY",
1769
- }, // no destination ⇒ colocated
1770
- { runSQL },
1771
- { duckdb: "dig" },
1772
- manifest,
1773
- {
1774
- getApiConnection: () => {
1775
- throw new Error("unused in the colocated build");
1776
- },
1777
- getEnvironmentPath: () => "/tmp/env",
1778
- },
1779
- builtEntries,
1780
- );
1781
- expect(seenManifest?.entries?.up_storage).toBeUndefined();
1782
- expect(seenManifest?.entries?.up_pathc?.tableName).toBe('"orders_v1"');
1783
- });
1784
-
1785
- it("records the QUOTED just-built name in the build manifest (matches the CREATE)", async () => {
1786
- // The build manifest feeds a downstream persist's FROM verbatim, so the
1787
- // recorded name must be the SAME quoted path the CREATE used — else a
1788
- // downstream built later in this run misses the case-preserved table on a
1789
- // case-folding engine. buildManifest also passes through Malloy's canonical
1790
- // path validation (requireCanonicalTablePathAnyDialect), so this proves the
1791
- // quoted form is accepted, not just produced.
1792
- const runSQL = sinon.stub().resolves();
1793
- const manifest = new Manifest();
1794
- await callBuildOneSource(
1795
- { runSQL },
1796
- "schema.orders_base",
1797
- "snowflake",
1798
- manifest,
1799
- );
1800
- expect(manifest.buildManifest.entries["abcdef1234567890"].tableName).toBe(
1801
- '"schema"."orders_base"',
1802
- );
1803
- // The logical (unquoted) name is what the CREATE was told to produce, and
1804
- // is exactly what quoteTablePath quoted for the DDL — read mirrors write.
1805
- const created = runSQL
1806
- .getCalls()
1807
- .map((c) => c.args[0] as string)
1808
- .find((s) => s.startsWith("CREATE TABLE"))!;
1809
- expect(created).toContain('"schema"."orders_base_abcdef123456"');
1810
- });
1811
- });
1812
-
1813
- describe("buildOneSourceIntoStorage (chained-build fallback ladder)", () => {
1814
- let ctx: ReturnType<typeof createMocks>;
1815
- beforeEach(() => {
1816
- ctx = createMocks();
1817
- });
1818
-
1819
- function callInto(opts: {
1820
- strict: boolean;
1821
- stackOnParent: "ok" | "throw" | "infra";
1822
- }): Promise<{
1823
- physicalTableName: string;
1824
- storageConnectionName?: string;
1825
- }> {
1826
- const source = fakeSource({
1827
- name: "monthly",
1828
- sourceEntityId: "abcdef1234567890",
1829
- annotationFields: { storage: "lake" },
1830
- });
1831
- const instruction: BuildInstruction = {
1832
- sourceEntityId: "abcdef1234567890",
1833
- materializedTableId: "mt",
1834
- physicalTableName: "monthly__mabc",
1835
- realization: "COPY",
1836
- destination: "lake",
1837
- };
1838
- const manifest = new Manifest();
1839
- manifest.strict = opts.strict;
1840
- const environment = {
1841
- getApiConnection: (n: string) => ({
1842
- name: n,
1843
- type: n === "lake" ? "duckdb" : "postgres",
1844
- }),
1845
- getEnvironmentPath: () => "/tmp/env",
1846
- };
1847
- const svc = ctx.service as unknown as {
1848
- buildDownstreamViaParents: unknown;
1849
- buildOneSourceIntoStorage: (
1850
- s: unknown,
1851
- i: BuildInstruction,
1852
- m: Manifest,
1853
- e: unknown,
1854
- sql: string,
1855
- built: Record<string, unknown>,
1856
- dep: boolean,
1857
- ) => Promise<{
1858
- physicalTableName: string;
1859
- storageConnectionName?: string;
1860
- }>;
1861
- };
1862
- // Stub the stack-on-parent seam so the test exercises the LADDER, not the build.
1863
- svc.buildDownstreamViaParents =
1864
- opts.stackOnParent === "ok"
1865
- ? sinon.stub().resolves({
1866
- storageConnectionName: "lake",
1867
- schema: [
1868
- { name: "order_month", type: "DATE" },
1869
- { name: "monthly_total", type: "DOUBLE" },
1870
- ],
1871
- })
1872
- : opts.stackOnParent === "infra"
1873
- ? // An outage, not a modelling limit: a plain Error, as a failed
1874
- // ATTACH or CTAS would throw.
1875
- sinon.stub().rejects(new Error("IO Error: destination is down"))
1876
- : // A shape limit: the downstream cannot be expressed over its
1877
- // rebound parents.
1878
- sinon.stub().rejects(
1879
- new MaterializationEligibilityError({
1880
- message: "uncarried parent",
1881
- }),
1882
- );
1883
- return svc.buildOneSourceIntoStorage(
1884
- source,
1885
- instruction,
1886
- manifest,
1887
- environment,
1888
- "SELECT should_not_run",
1889
- {
1890
- up: {
1891
- sourceEntityId: "up",
1892
- sourceName: "daily",
1893
- physicalTableName: "daily__mabc",
1894
- storageConnectionName: "lake",
1895
- schema: [{ name: "order_date", type: "DATE" }],
1896
- },
1897
- },
1898
- true, // dependsOnStorageUpstream
1899
- );
1900
- }
1901
-
1902
- it("stacks on the parent: builds by reading it and returns the storage entry", async () => {
1903
- const entry = await callInto({ strict: false, stackOnParent: "ok" });
1904
- expect(entry.storageConnectionName).toBe("lake");
1905
- expect(entry.physicalTableName).toBe("monthly__mabc");
1906
- });
1907
-
1908
- it("strict + cannot stack on the parent: refuses loudly instead of recomputing from raw", async () => {
1909
- await expect(
1910
- callInto({ strict: true, stackOnParent: "throw" }),
1911
- ).rejects.toThrow(/strict upstreams forbid/i);
1912
- });
1913
-
1914
- it("non-strict + an INFRA failure fails rather than recomputing from raw", async () => {
1915
- // Recompute-from-raw writes to the same destination, so retrying an outage
1916
- // there just fails again — and metering it as a fallback files the outage
1917
- // under the same label as a legitimate shape miss. Only a shape failure is
1918
- // a reason to try the other path.
1919
- await expect(
1920
- callInto({ strict: false, stackOnParent: "infra" }),
1921
- ).rejects.toThrow(/Failed to materialize chained source/i);
1922
- });
1923
-
1924
- it("non-strict + a SHAPE failure falls through to recompute-from-raw", async () => {
1925
- // Only the parent-reuse seam is stubbed, so the recompute runs for real and
1926
- // fails for its own reason (no destination file). That is the proof it was
1927
- // reached: the two paths report differently — the chained path says
1928
- // "chained source", the single-source recompute says "source". A shape
1929
- // failure must reach the second; an infra failure must not.
1930
- await expect(
1931
- callInto({ strict: false, stackOnParent: "throw" }),
1932
- ).rejects.toThrow(/Failed to materialize source 'monthly'/i);
1933
- });
1934
- });
1935
-
1936
- describe("runBuild (branch behavior)", () => {
1937
- let ctx: ReturnType<typeof createMocks>;
1938
- beforeEach(() => {
1939
- ctx = createMocks();
1940
- });
1941
-
1942
- // Exercise runBuild's orchestration in isolation: the build engine and
1943
- // manifest commit are stubbed so the test asserts which instructions flow
1944
- // through and whether the manifest is auto-loaded, not how a source is built.
1945
- type RunBuildInternals = {
1946
- executeInstructedBuild: sinon.SinonStub;
1947
- commitManifest: sinon.SinonStub;
1948
- autoLoadManifest: sinon.SinonStub;
1949
- runBuild: (
1950
- id: string,
1951
- env: string,
1952
- pkg: string,
1953
- opts: {
1954
- sourceNames: string[] | undefined;
1955
- forceRefresh: boolean;
1956
- buildInstructions: BuildInstruction[] | undefined;
1957
- referenceManifest?:
1958
- | {
1959
- sourceEntityId: string;
1960
- physicalTableName: string;
1961
- connectionName?: string;
1962
- }[]
1963
- | undefined;
1964
- strictUpstreams?: boolean | undefined;
1965
- },
1966
- signal: AbortSignal,
1967
- ) => Promise<void>;
1968
- };
1969
-
1970
- function stubEngine(): RunBuildInternals {
1971
- const entries = {
1972
- "build-orders": {
1973
- sourceEntityId: "build-orders",
1974
- physicalTableName: '"orders_v1"',
1975
- connectionName: "duckdb",
1976
- },
1977
- };
1978
- const svc = ctx.service as unknown as RunBuildInternals;
1979
- svc.executeInstructedBuild = sinon.stub().resolves(entries);
1980
- svc.commitManifest = sinon.stub().resolves();
1981
- svc.autoLoadManifest = sinon.stub().resolves();
1982
- return svc;
1983
- }
1984
-
1985
- it("orchestrated: builds caller instructions and does not auto-load", async () => {
1986
- const svc = stubEngine();
1987
- const instructions = [makeInstruction()];
1988
-
1989
- await svc.runBuild(
1990
- "mat-1",
1991
- "my-env",
1992
- "pkg",
1993
- {
1994
- sourceNames: undefined,
1995
- forceRefresh: false,
1996
- buildInstructions: instructions,
1997
- },
1998
- new AbortController().signal,
1999
- );
2000
-
2001
- expect(svc.executeInstructedBuild.calledOnce).toBe(true);
2002
- // Caller instructions pass straight through; nothing is carried/reused.
2003
- // (arg[1] is the environment; instructions are arg[2], seed entries arg[3].)
2004
- expect(svc.executeInstructedBuild.firstCall.args[2]).toEqual(
2005
- instructions,
2006
- );
2007
- expect(svc.executeInstructedBuild.firstCall.args[3]).toEqual({});
2008
- expect(svc.commitManifest.firstCall.args[2]).toMatchObject({
2009
- mode: "orchestrated",
2010
- sourcesBuilt: 1,
2011
- sourcesReused: 0,
2012
- });
2013
- // Orchestrated leaves distribution to the caller.
2014
- expect(svc.autoLoadManifest.called).toBe(false);
2015
- });
2016
-
2017
- it("orchestrated: seeds the build from referenceManifest and honors strictUpstreams", async () => {
2018
- const svc = stubEngine();
2019
- const instructions = [makeInstruction()];
2020
-
2021
- await svc.runBuild(
2022
- "mat-1",
2023
- "my-env",
2024
- "pkg",
2025
- {
2026
- sourceNames: undefined,
2027
- forceRefresh: false,
2028
- buildInstructions: instructions,
2029
- referenceManifest: [
2030
- {
2031
- sourceEntityId: "up-1",
2032
- physicalTableName: "upstream_tbl",
2033
- connectionName: "sf",
2034
- },
2035
- ],
2036
- strictUpstreams: true,
2037
- },
2038
- new AbortController().signal,
2039
- );
2040
-
2041
- // The reference manifest becomes the seed (carried) entries so a
2042
- // downstream build resolves its upstream to the existing physical table.
2043
- // arg[1] is environment (the storage build session), so the seed
2044
- // (carried) entries are arg[3]. connectionName is carried through so the
2045
- // seed can be quoted for the upstream's dialect (see quoteSeedTablePath).
2046
- expect(svc.executeInstructedBuild.firstCall.args[3]).toEqual({
2047
- "up-1": {
2048
- sourceEntityId: "up-1",
2049
- physicalTableName: "upstream_tbl",
2050
- connectionName: "sf",
2051
- },
2052
- });
2053
- // strictUpstreams flows through as the strict flag (6th arg).
2054
- expect(svc.executeInstructedBuild.firstCall.args[5]).toBe(true);
2055
- // Reused upstreams are counted as carried, not built.
2056
- expect(svc.commitManifest.firstCall.args[2]).toMatchObject({
2057
- mode: "orchestrated",
2058
- sourcesBuilt: 1,
2059
- sourcesReused: 1,
2060
- });
2061
- });
2062
-
2063
- it("auto-run: derives instructions and auto-loads the manifest", async () => {
2064
- const svc = stubEngine();
2065
-
2066
- await svc.runBuild(
2067
- "mat-1",
2068
- "my-env",
2069
- "pkg",
2070
- {
2071
- sourceNames: undefined,
2072
- forceRefresh: true,
2073
- buildInstructions: undefined,
2074
- },
2075
- new AbortController().signal,
2076
- );
2077
-
2078
- expect(svc.commitManifest.firstCall.args[2]).toMatchObject({
2079
- mode: "auto",
2080
- });
2081
- // Auto-run owns distribution: it loads the fresh manifest into the models.
2082
- expect(svc.autoLoadManifest.calledOnce).toBe(true);
2083
- });
2084
- });
2085
-
2086
- describe("autoLoadManifest", () => {
2087
- let ctx: ReturnType<typeof createMocks>;
2088
- beforeEach(() => {
2089
- ctx = createMocks();
2090
- });
2091
-
2092
- it("loads only entries with a physical table into the package models", async () => {
2093
- const reloadAllModelsForPackage = sinon.stub().resolves();
2094
- const entries = {
2095
- b1: {
2096
- sourceEntityId: "b1",
2097
- physicalTableName: "orders_v1",
2098
- connectionName: "duckdb",
2099
- },
2100
- b2: { sourceEntityId: "b2", physicalTableName: undefined },
2101
- };
2102
-
2103
- await (
2104
- ctx.service as unknown as {
2105
- autoLoadManifest: (
2106
- env: { reloadAllModelsForPackage: sinon.SinonStub },
2107
- pkg: string,
2108
- entries: Record<string, unknown>,
2109
- ) => Promise<void>;
2110
- }
2111
- ).autoLoadManifest({ reloadAllModelsForPackage }, "pkg", entries);
2112
-
2113
- expect(reloadAllModelsForPackage.calledOnce).toBe(true);
2114
- // connectionName is carried alongside tableName so the bind step
2115
- // (Package.quoteBoundTableNames) can quote the path for its dialect.
2116
- expect(reloadAllModelsForPackage.firstCall.args).toEqual([
2117
- "pkg",
2118
- { b1: { tableName: "orders_v1", connectionName: "duckdb" } },
2119
- ]);
2120
- });
2121
-
2122
- it("swallows a reload failure (best-effort; run already succeeded)", async () => {
2123
- const reloadAllModelsForPackage = sinon
2124
- .stub()
2125
- .rejects(new Error("reload boom"));
2126
- await expect(
2127
- (
2128
- ctx.service as unknown as {
2129
- autoLoadManifest: (
2130
- env: { reloadAllModelsForPackage: sinon.SinonStub },
2131
- pkg: string,
2132
- entries: Record<string, unknown>,
2133
- ) => Promise<void>;
2134
- }
2135
- ).autoLoadManifest({ reloadAllModelsForPackage }, "pkg", {
2136
- b1: { sourceEntityId: "b1", physicalTableName: "t" },
2137
- }),
2138
- ).resolves.toBeUndefined();
2139
- });
2140
- });
2141
-
2142
- describe("runInBackground (terminal recording)", () => {
2143
- let ctx: ReturnType<typeof createMocks>;
2144
- beforeEach(() => {
2145
- ctx = createMocks();
2146
- });
2147
-
2148
- const flush = () => new Promise((r) => setTimeout(r, 20));
2149
-
2150
- function background(): {
2151
- runInBackground: (
2152
- id: string,
2153
- run: (signal: AbortSignal) => Promise<void>,
2154
- ) => void;
2155
- runningAbortControllers: Map<string, AbortController>;
2156
- } {
2157
- return ctx.service as unknown as {
2158
- runInBackground: (
2159
- id: string,
2160
- run: (signal: AbortSignal) => Promise<void>,
2161
- ) => void;
2162
- runningAbortControllers: Map<string, AbortController>;
2163
- };
2164
- }
2165
-
2166
- it("records FAILED with the error message when the run rejects", async () => {
2167
- background().runInBackground("bg-1", async () => {
2168
- throw new Error("boom");
2169
- });
2170
- await flush();
2171
-
2172
- expect(ctx.repository.updateMaterialization.calledOnce).toBe(true);
2173
- expect(ctx.repository.updateMaterialization.firstCall.args[0]).toBe(
2174
- "bg-1",
2175
- );
2176
- expect(
2177
- ctx.repository.updateMaterialization.firstCall.args[1],
2178
- ).toMatchObject({ status: "FAILED", error: "boom" });
2179
- });
2180
-
2181
- it("records CANCELLED when the run aborts cooperatively", async () => {
2182
- const bg = background();
2183
- bg.runInBackground("bg-2", async () => {
2184
- bg.runningAbortControllers.get("bg-2")!.abort();
2185
- throw new Error("boom");
2186
- });
2187
- await flush();
2188
-
2189
- expect(
2190
- ctx.repository.updateMaterialization.firstCall.args[1],
2191
- ).toMatchObject({ status: "CANCELLED", error: "Cancelled" });
2192
- });
2193
-
2194
- it("clears the abort controller once the run settles", async () => {
2195
- const bg = background();
2196
- bg.runInBackground("bg-3", async () => {});
2197
- await flush();
2198
- expect(bg.runningAbortControllers.has("bg-3")).toBe(false);
2199
- });
2200
- });
2201
-
2202
- describe("transition (state machine)", () => {
2203
- let ctx: ReturnType<typeof createMocks>;
2204
- beforeEach(() => {
2205
- ctx = createMocks();
2206
- });
2207
-
2208
- function transition(): (
2209
- id: string,
2210
- next: MaterializationStatus,
2211
- ) => Promise<unknown> {
2212
- return (
2213
- ctx.service as unknown as {
2214
- transition: (
2215
- id: string,
2216
- next: MaterializationStatus,
2217
- ) => Promise<unknown>;
2218
- }
2219
- ).transition.bind(ctx.service);
2220
- }
2221
-
2222
- it("throws MaterializationNotFoundError when the record is gone", async () => {
2223
- ctx.repository.getMaterializationById.resolves(undefined);
2224
- await expect(
2225
- transition()("mat-1", "MANIFEST_ROWS_READY"),
2226
- ).rejects.toThrow(MaterializationNotFoundError);
2227
- });
2228
-
2229
- it("rejects a transition the state machine disallows", async () => {
2230
- ctx.repository.getMaterializationById.resolves(
2231
- makeMaterialization({ status: "MANIFEST_FILE_READY" }),
2232
- );
2233
- // MANIFEST_FILE_READY is terminal; nothing follows it.
2234
- await expect(transition()("mat-1", "PENDING")).rejects.toThrow(
2235
- InvalidStateTransitionError,
2236
- );
2237
- expect(ctx.repository.updateMaterialization.called).toBe(false);
2238
- });
2239
-
2240
- it("persists a valid transition", async () => {
2241
- ctx.repository.getMaterializationById.resolves(
2242
- makeMaterialization({ status: "PENDING" }),
2243
- );
2244
- ctx.repository.updateMaterialization.resolves(
2245
- makeMaterialization({ status: "MANIFEST_ROWS_READY" }),
2246
- );
2247
- await transition()("mat-1", "MANIFEST_ROWS_READY");
2248
- expect(
2249
- ctx.repository.updateMaterialization.calledOnceWith("mat-1", {
2250
- status: "MANIFEST_ROWS_READY",
2251
- }),
2252
- ).toBe(true);
2253
- });
2254
- });
2255
-
2256
- // The only path here that issues a DROP against a customer's warehouse, so its
2257
- // guards get assertions rather than a trace. Scenario 64 covers it end to end,
2258
- // but hammer does not run in CI — and the failure mode is silent data loss, not
2259
- // a broken build.
2260
- describe("reclaimStorageTablesFromFailedRun", () => {
2261
- let ctx: ReturnType<typeof createMocks>;
2262
- let infoLog: sinon.SinonStub;
2263
- let warnLog: sinon.SinonStub;
2264
-
2265
- beforeEach(() => {
2266
- ctx = createMocks();
2267
- infoLog = ctx.sandbox.stub(logger, "info");
2268
- warnLog = ctx.sandbox.stub(logger, "warn");
2269
- });
2270
- afterEach(() => ctx.sandbox.restore());
2271
-
2272
- const entry = (table: string) =>
2273
- ({
2274
- sourceEntityId: `eid-${table}`,
2275
- sourceName: "daily",
2276
- physicalTableName: table,
2277
- storageConnectionName: "lake",
2278
- }) as unknown as Parameters<
2279
- MaterializationService["reclaimStorageTablesFromFailedRun"]
2280
- >[0][number];
2281
-
2282
- /** A destination whose drop would fail loudly if one were ever attempted. */
2283
- const environment = {
2284
- getApiConnection: () => ({ name: "lake", type: "duckdb" }),
2285
- getEnvironmentPath: () => "/test",
2286
- } as unknown as Parameters<
2287
- MaterializationService["reclaimStorageTablesFromFailedRun"]
2288
- >[1];
2289
-
2290
- const reclaim = (
2291
- built: ReturnType<typeof entry>[],
2292
- others: unknown[],
2293
- ): Promise<void> => {
2294
- (ctx.repository.listMaterializations as sinon.SinonStub).resolves(others);
2295
- return (
2296
- ctx.service as unknown as {
2297
- reclaimStorageTablesFromFailedRun: (
2298
- b: unknown,
2299
- e: unknown,
2300
- o: unknown,
2301
- ) => Promise<void>;
2302
- }
2303
- ).reclaimStorageTablesFromFailedRun(built, environment, {
2304
- environmentId: "env-1",
2305
- packageName: "pkg",
2306
- });
2307
- };
2308
-
2309
- const said = (stub: sinon.SinonStub, needle: RegExp) =>
2310
- stub.getCalls().some((c) => needle.test(String(c.args[0])));
2311
-
2312
- it("never drops a table a live manifest still serves", async () => {
2313
- // The carried-forward case: a previous successful run's manifest names the
2314
- // same destination + table. Dropping it would delete a table that is being
2315
- // served right now.
2316
- await reclaim(
2317
- [entry("daily_v1")],
2318
- [
2319
- {
2320
- status: "MANIFEST_FILE_READY",
2321
- manifest: {
2322
- entries: {
2323
- other: {
2324
- storageConnectionName: "lake",
2325
- physicalTableName: "daily_v1",
2326
- },
2327
- },
2328
- },
2329
- },
2330
- ],
2331
- );
2332
-
2333
- expect(said(infoLog, /Keeping a table from a failed run/)).toBe(true);
2334
- expect(said(infoLog, /Reclaimed a table/)).toBe(false);
2335
- expect(said(warnLog, /Failed to reclaim/)).toBe(false);
2336
- });
2337
-
2338
- it("does attempt the drop when nothing references the table", async () => {
2339
- // The inverse, so the test above is known to be the still-referenced check
2340
- // doing the work rather than the reclaim being inert. No real destination
2341
- // exists here, so the attempt surfaces as the drop-failure warning.
2342
- await reclaim([entry("daily_v2")], []);
2343
-
2344
- expect(said(infoLog, /Keeping a table from a failed run/)).toBe(false);
2345
- expect(
2346
- said(warnLog, /Failed to reclaim/) ||
2347
- said(infoLog, /Reclaimed a table/),
2348
- ).toBe(true);
2349
- });
2350
-
2351
- it("ignores a superseded run's entries when deciding", async () => {
2352
- // Only MANIFEST_FILE_READY runs count as live. A superseded run naming the
2353
- // same table must not pin it forever.
2354
- await reclaim(
2355
- [entry("daily_v3")],
2356
- [
2357
- {
2358
- status: "SUPERSEDED",
2359
- manifest: {
2360
- entries: {
2361
- old: {
2362
- storageConnectionName: "lake",
2363
- physicalTableName: "daily_v3",
2364
- },
2365
- },
2366
- },
2367
- },
2368
- ],
2369
- );
2370
-
2371
- expect(said(infoLog, /Keeping a table from a failed run/)).toBe(false);
2372
- });
2373
- });