@malloy-publisher/server 0.0.235 → 0.0.237

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (319) hide show
  1. package/dist/app/api-doc.yaml +83 -27
  2. package/dist/app/assets/{EnvironmentPage-BsAnavYN.js → EnvironmentPage-DaP6CpgK.js} +1 -1
  3. package/dist/app/assets/{HomePage-CADE138j.js → HomePage-C9yNw3iu.js} +1 -1
  4. package/dist/app/assets/{LightMode-Cfh7KzN8.js → LightMode-Be4-G-dX.js} +1 -1
  5. package/dist/app/assets/{MainPage-CO3pRlnV.js → MainPage-J_rU2SgI.js} +2 -2
  6. package/dist/app/assets/{MaterializationsPage-p9YjkRXZ.js → MaterializationsPage-DHgPI4MO.js} +1 -1
  7. package/dist/app/assets/ModelPage-BzXfjTu5.js +1 -0
  8. package/dist/app/assets/{PackagePage-e4kN75YR.js → PackagePage-ByoSTLJ4.js} +1 -1
  9. package/dist/app/assets/RouteError-CrSeKN2r.js +1 -0
  10. package/dist/app/assets/{ThemeEditorPage-CciagFTq.js → ThemeEditorPage-Btxnmhtk.js} +1 -1
  11. package/dist/app/assets/{WorkbookPage-DNWmkCXa.js → WorkbookPage-D6JLPnWO.js} +1 -1
  12. package/dist/app/assets/{core-Rj_4rRnA.es-BZyvITuO.js → core-BSRgt1mG.es-Bc_D2Lq4.js} +1 -1
  13. package/dist/app/assets/{index-CH2AcDzc.js → index-BRijWj6K.js} +15 -15
  14. package/dist/app/assets/{index-DQa463gC.js → index-BX4D94xw.js} +1 -1
  15. package/dist/app/assets/{index-VBbcc8s6.js → index-Cvy0xtb7.js} +1 -1
  16. package/dist/app/assets/{index-DQpV7MyA.js → index-DMAhIJNC.js} +1 -1
  17. package/dist/app/index.html +1 -1
  18. package/dist/package_load_worker.mjs +59 -20
  19. package/dist/server.mjs +1342 -616
  20. package/package.json +8 -2
  21. package/.eslintrc.json +0 -44
  22. package/.prettierignore +0 -1
  23. package/.prettierrc +0 -4
  24. package/build.ts +0 -97
  25. package/dist/app/assets/ModelPage-C1OSTv-x.js +0 -1
  26. package/dist/app/assets/RouteError-CzbfOkng.js +0 -1
  27. package/eslint.config.mjs +0 -76
  28. package/k6-tests/README.md +0 -144
  29. package/k6-tests/bun.lock +0 -797
  30. package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
  31. package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
  32. package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
  33. package/k6-tests/load-test/load-test-crud.ts +0 -142
  34. package/k6-tests/load-test/load-test.ts +0 -784
  35. package/k6-tests/package.json +0 -27
  36. package/k6-tests/scripts/run-with-otel.sh +0 -50
  37. package/k6-tests/smoke-test/smoke-test.ts +0 -158
  38. package/k6-tests/tsconfig.json +0 -29
  39. package/k6-tests/utils/client_factory.ts +0 -128
  40. package/k6-tests/utils/common.ts +0 -489
  41. package/k6-tests/utils/logger.ts +0 -62
  42. package/k6-tests/utils/types.d.ts +0 -35
  43. package/malloy_mcp.dxt +0 -0
  44. package/publisher.config.example.bigquery.json +0 -33
  45. package/publisher.config.example.duckdb.json +0 -23
  46. package/publisher.config.json +0 -23
  47. package/scripts/bake-duckdb-extensions.js +0 -110
  48. package/src/config.spec.ts +0 -1535
  49. package/src/config.theme.spec.ts +0 -178
  50. package/src/config.ts +0 -1142
  51. package/src/constants.ts +0 -105
  52. package/src/controller/compile.controller.ts +0 -40
  53. package/src/controller/connection.controller.spec.ts +0 -849
  54. package/src/controller/connection.controller.ts +0 -901
  55. package/src/controller/database.controller.ts +0 -24
  56. package/src/controller/materialization.controller.spec.ts +0 -293
  57. package/src/controller/materialization.controller.ts +0 -319
  58. package/src/controller/model.controller.ts +0 -164
  59. package/src/controller/package.controller.spec.ts +0 -273
  60. package/src/controller/package.controller.ts +0 -306
  61. package/src/controller/query.controller.ts +0 -152
  62. package/src/controller/theme.controller.ts +0 -83
  63. package/src/controller/watch-mode.controller.ts +0 -243
  64. package/src/data_styles.ts +0 -71
  65. package/src/default-publisher.config.json +0 -23
  66. package/src/ducklake_version.spec.ts +0 -163
  67. package/src/ducklake_version.ts +0 -153
  68. package/src/errors.spec.ts +0 -119
  69. package/src/errors.ts +0 -237
  70. package/src/filter_deprecation.spec.ts +0 -64
  71. package/src/filter_deprecation.ts +0 -42
  72. package/src/health.spec.ts +0 -123
  73. package/src/health.ts +0 -233
  74. package/src/heap_check.spec.ts +0 -144
  75. package/src/heap_check.ts +0 -144
  76. package/src/instrumentation.ts +0 -214
  77. package/src/json_utils.spec.ts +0 -51
  78. package/src/json_utils.ts +0 -33
  79. package/src/logger.spec.ts +0 -210
  80. package/src/logger.ts +0 -251
  81. package/src/malloy_pin_prereqs.spec.ts +0 -25
  82. package/src/materialization_metrics.spec.ts +0 -169
  83. package/src/materialization_metrics.ts +0 -346
  84. package/src/mcp/error_messages.spec.ts +0 -35
  85. package/src/mcp/error_messages.ts +0 -183
  86. package/src/mcp/handler_utils.spec.ts +0 -108
  87. package/src/mcp/handler_utils.ts +0 -251
  88. package/src/mcp/mcp_constants.ts +0 -6
  89. package/src/mcp/query_envelope.spec.ts +0 -229
  90. package/src/mcp/query_envelope.ts +0 -240
  91. package/src/mcp/server.protocol.spec.ts +0 -250
  92. package/src/mcp/server.ts +0 -87
  93. package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
  94. package/src/mcp/skills/build_skills_bundle.ts +0 -176
  95. package/src/mcp/skills/skills_bundle.json +0 -1
  96. package/src/mcp/skills/skills_bundle.spec.ts +0 -171
  97. package/src/mcp/tool_response.spec.ts +0 -108
  98. package/src/mcp/tool_response.ts +0 -138
  99. package/src/mcp/tools/compile_tool.spec.ts +0 -315
  100. package/src/mcp/tools/compile_tool.ts +0 -208
  101. package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
  102. package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
  103. package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
  104. package/src/mcp/tools/docs_search_tool.ts +0 -128
  105. package/src/mcp/tools/embedding_index.spec.ts +0 -1236
  106. package/src/mcp/tools/embedding_index.ts +0 -808
  107. package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
  108. package/src/mcp/tools/execute_query_tool.ts +0 -316
  109. package/src/mcp/tools/get_context_eval.ts +0 -275
  110. package/src/mcp/tools/get_context_tool.spec.ts +0 -592
  111. package/src/mcp/tools/get_context_tool.ts +0 -585
  112. package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
  113. package/src/mcp/tools/reload_package_tool.ts +0 -132
  114. package/src/mcp_config.spec.ts +0 -919
  115. package/src/mcp_config.ts +0 -425
  116. package/src/oom_guards.integration.spec.ts +0 -269
  117. package/src/package_load/package_load_pool.spec.ts +0 -252
  118. package/src/package_load/package_load_pool.ts +0 -931
  119. package/src/package_load/package_load_worker.ts +0 -1034
  120. package/src/package_load/protocol.ts +0 -364
  121. package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
  122. package/src/package_load/rpc_wait_accountant.ts +0 -76
  123. package/src/package_load_metrics.spec.ts +0 -114
  124. package/src/package_load_metrics.ts +0 -127
  125. package/src/path_safety.spec.ts +0 -158
  126. package/src/path_safety.ts +0 -146
  127. package/src/pg_helpers.spec.ts +0 -223
  128. package/src/pg_helpers.ts +0 -52
  129. package/src/query_cap_metrics.spec.ts +0 -89
  130. package/src/query_cap_metrics.ts +0 -116
  131. package/src/query_concurrency.spec.ts +0 -247
  132. package/src/query_concurrency.ts +0 -237
  133. package/src/query_metadata_metrics.ts +0 -49
  134. package/src/query_param_utils.ts +0 -29
  135. package/src/query_timeout.spec.ts +0 -224
  136. package/src/query_timeout.ts +0 -179
  137. package/src/runtime/publisher.js +0 -328
  138. package/src/server-old.ts +0 -988
  139. package/src/server.ts +0 -2059
  140. package/src/service/annotations.spec.ts +0 -118
  141. package/src/service/annotations.ts +0 -91
  142. package/src/service/authorize.spec.ts +0 -154
  143. package/src/service/authorize.ts +0 -498
  144. package/src/service/authorize_integration.spec.ts +0 -2041
  145. package/src/service/build_plan.spec.ts +0 -685
  146. package/src/service/build_plan.ts +0 -797
  147. package/src/service/compile_authorize.spec.ts +0 -85
  148. package/src/service/compile_fragment_techniques.spec.ts +0 -156
  149. package/src/service/connection.spec.ts +0 -2360
  150. package/src/service/connection.ts +0 -2187
  151. package/src/service/connection_config.spec.ts +0 -734
  152. package/src/service/connection_config.ts +0 -899
  153. package/src/service/connection_federation.spec.ts +0 -184
  154. package/src/service/connection_fingerprint.spec.ts +0 -102
  155. package/src/service/connection_service.spec.ts +0 -711
  156. package/src/service/connection_service.ts +0 -291
  157. package/src/service/cron_evaluator.spec.ts +0 -79
  158. package/src/service/cron_evaluator.ts +0 -105
  159. package/src/service/db_utils.spec.ts +0 -1085
  160. package/src/service/db_utils.ts +0 -1502
  161. package/src/service/duckdb_instance_isolation.spec.ts +0 -137
  162. package/src/service/ducklake_lazy_attach.spec.ts +0 -110
  163. package/src/service/embedding_provider.spec.ts +0 -329
  164. package/src/service/embedding_provider.ts +0 -236
  165. package/src/service/environment.ts +0 -2191
  166. package/src/service/environment_admission.spec.ts +0 -344
  167. package/src/service/environment_compile.spec.ts +0 -197
  168. package/src/service/environment_store.spec.ts +0 -2274
  169. package/src/service/environment_store.ts +0 -2434
  170. package/src/service/environment_store_anchoring.spec.ts +0 -107
  171. package/src/service/environment_store_clone.spec.ts +0 -350
  172. package/src/service/explore_visibility.spec.ts +0 -434
  173. package/src/service/exports_probe.spec.ts +0 -107
  174. package/src/service/extension_fetch_policy.spec.ts +0 -256
  175. package/src/service/filter.spec.ts +0 -458
  176. package/src/service/filter.ts +0 -341
  177. package/src/service/filter_bypass.spec.ts +0 -418
  178. package/src/service/filter_integration.spec.ts +0 -1004
  179. package/src/service/freshness.spec.ts +0 -183
  180. package/src/service/freshness.ts +0 -112
  181. package/src/service/gcs_s3_utils.ts +0 -427
  182. package/src/service/given.ts +0 -105
  183. package/src/service/givens_integration.spec.ts +0 -219
  184. package/src/service/manifest_loader.spec.ts +0 -189
  185. package/src/service/manifest_loader.ts +0 -156
  186. package/src/service/materialization_build_session.spec.ts +0 -435
  187. package/src/service/materialization_build_session.ts +0 -681
  188. package/src/service/materialization_config_validation.spec.ts +0 -99
  189. package/src/service/materialization_config_validation.ts +0 -120
  190. package/src/service/materialization_eligibility.spec.ts +0 -158
  191. package/src/service/materialization_eligibility.ts +0 -305
  192. package/src/service/materialization_schedule_surface.spec.ts +0 -256
  193. package/src/service/materialization_scheduler.spec.ts +0 -289
  194. package/src/service/materialization_scheduler.ts +0 -304
  195. package/src/service/materialization_serve_transform.spec.ts +0 -1003
  196. package/src/service/materialization_serve_transform.ts +0 -779
  197. package/src/service/materialization_service.spec.ts +0 -2373
  198. package/src/service/materialization_service.ts +0 -2342
  199. package/src/service/materialization_test_fixtures.ts +0 -249
  200. package/src/service/model.spec.ts +0 -1157
  201. package/src/service/model.ts +0 -3370
  202. package/src/service/model_limits.spec.ts +0 -209
  203. package/src/service/model_limits.ts +0 -131
  204. package/src/service/model_storage_serve.spec.ts +0 -193
  205. package/src/service/model_storage_serve_joins.spec.ts +0 -193
  206. package/src/service/package.spec.ts +0 -612
  207. package/src/service/package.ts +0 -1799
  208. package/src/service/package_manifest.spec.ts +0 -230
  209. package/src/service/package_manifest.ts +0 -241
  210. package/src/service/package_memory_governor.spec.ts +0 -173
  211. package/src/service/package_memory_governor.ts +0 -233
  212. package/src/service/package_quote_bound_tables.spec.ts +0 -171
  213. package/src/service/package_race.spec.ts +0 -208
  214. package/src/service/package_reload_safety.spec.ts +0 -193
  215. package/src/service/package_rollback.spec.ts +0 -190
  216. package/src/service/package_worker_path.spec.ts +0 -645
  217. package/src/service/path_injection.spec.ts +0 -39
  218. package/src/service/persist_annotation_validation.spec.ts +0 -89
  219. package/src/service/persist_annotation_validation.ts +0 -59
  220. package/src/service/persistence_policy.spec.ts +0 -374
  221. package/src/service/proxy.spec.ts +0 -414
  222. package/src/service/proxy.ts +0 -248
  223. package/src/service/query_boundary.spec.ts +0 -470
  224. package/src/service/query_metadata.spec.ts +0 -408
  225. package/src/service/query_metadata.ts +0 -492
  226. package/src/service/query_metadata_identity.spec.ts +0 -149
  227. package/src/service/query_text.spec.ts +0 -79
  228. package/src/service/query_text.ts +0 -43
  229. package/src/service/quoting.spec.ts +0 -135
  230. package/src/service/quoting.ts +0 -78
  231. package/src/service/resolve_environment.ts +0 -15
  232. package/src/service/restricted_mode.spec.ts +0 -299
  233. package/src/service/source_extraction.ts +0 -225
  234. package/src/service/theme_store.ts +0 -199
  235. package/src/storage/DatabaseInterface.ts +0 -198
  236. package/src/storage/StorageManager.ts +0 -113
  237. package/src/storage/duckdb/ConnectionRepository.ts +0 -143
  238. package/src/storage/duckdb/DuckDBConnection.ts +0 -123
  239. package/src/storage/duckdb/DuckDBRepository.ts +0 -208
  240. package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
  241. package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
  242. package/src/storage/duckdb/MaterializationRepository.ts +0 -336
  243. package/src/storage/duckdb/PackageRepository.ts +0 -135
  244. package/src/storage/duckdb/schema.ts +0 -229
  245. package/src/stream_helpers.spec.ts +0 -280
  246. package/src/stream_helpers.ts +0 -162
  247. package/src/telemetry.ts +0 -20
  248. package/src/test_helpers/metrics_harness.ts +0 -166
  249. package/src/theme_key_parity.spec.ts +0 -57
  250. package/src/utils.ts +0 -29
  251. package/tests/fixtures/authorize-compile/model.malloy +0 -9
  252. package/tests/fixtures/authorize-compile/publisher.json +0 -4
  253. package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
  254. package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
  255. package/tests/fixtures/html-pages-test/data.csv +0 -3
  256. package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
  257. package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
  258. package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
  259. package/tests/fixtures/html-pages-test/public/data.json +0 -1
  260. package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
  261. package/tests/fixtures/html-pages-test/public/index.html +0 -9
  262. package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
  263. package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
  264. package/tests/fixtures/html-pages-test/public/slides.html +0 -12
  265. package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
  266. package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
  267. package/tests/fixtures/html-pages-test/publisher.json +0 -5
  268. package/tests/fixtures/html-pages-test/report.malloy +0 -1
  269. package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
  270. package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
  271. package/tests/fixtures/persist-multi-level/publisher.json +0 -5
  272. package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
  273. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
  274. package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
  275. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
  276. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
  277. package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
  278. package/tests/fixtures/persist-test/data/orders.csv +0 -5
  279. package/tests/fixtures/persist-test/persist_test.malloy +0 -11
  280. package/tests/fixtures/persist-test/publisher.json +0 -5
  281. package/tests/fixtures/publisher.config.json +0 -15
  282. package/tests/fixtures/query-givens/data/orders.csv +0 -7
  283. package/tests/fixtures/query-givens/model.malloy +0 -34
  284. package/tests/fixtures/query-givens/publisher.json +0 -5
  285. package/tests/fixtures/xlsx/database.xlsx +0 -0
  286. package/tests/harness/mcp_test_setup.ts +0 -332
  287. package/tests/harness/rest_e2e.ts +0 -68
  288. package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
  289. package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
  290. package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
  291. package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
  292. package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
  293. package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
  294. package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
  295. package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
  296. package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
  297. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
  298. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
  299. package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
  300. package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
  301. package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
  302. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
  303. package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
  304. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
  305. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
  306. package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
  307. package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
  308. package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
  309. package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
  310. package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
  311. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
  312. package/tests/unit/controller/theme.controller.spec.ts +0 -141
  313. package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
  314. package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
  315. package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
  316. package/tests/unit/duckdb/repositories.test.ts +0 -208
  317. package/tests/unit/ducklake/ducklake.test.ts +0 -952
  318. package/tests/unit/service/theme_store.spec.ts +0 -139
  319. package/tsconfig.json +0 -32
@@ -1,2360 +0,0 @@
1
- import { DuckDBConnection } from "@malloydata/db-duckdb";
2
- import {
3
- afterAll,
4
- afterEach,
5
- beforeEach,
6
- describe,
7
- expect,
8
- it,
9
- } from "bun:test";
10
- import fs from "fs/promises";
11
- import os from "os";
12
- import path from "path";
13
- import sinon from "sinon";
14
- import { components } from "../api";
15
- import {
16
- attachDuckLakeReadWrite,
17
- buildProxiedSslQuery,
18
- createEnvironmentConnections,
19
- resolveProxiedTls,
20
- testConnectionConfig,
21
- } from "./connection";
22
- import { assembleEnvironmentConnections } from "./connection_config";
23
- import { UnsupportedCatalogFormatError } from "../errors";
24
- import { EnvironmentStore } from "./environment_store";
25
-
26
- type ApiConnection = components["schemas"]["Connection"];
27
- type AttachedDatabase = components["schemas"]["AttachedDatabase"];
28
-
29
- const hasPostgresCredentials = () =>
30
- !!(
31
- process.env.POSTGRES_TEST_HOST &&
32
- process.env.POSTGRES_TEST_USER &&
33
- process.env.POSTGRES_TEST_PASSWORD
34
- );
35
-
36
- const hasBigQueryCredentials = () =>
37
- !!(
38
- process.env.GOOGLE_APPLICATION_CREDENTIALS &&
39
- process.env.BIGQUERY_TEST_PROJECT_ID
40
- );
41
-
42
- const hasSnowflakeCredentials = () =>
43
- !!(
44
- process.env.SNOWFLAKE_TEST_ACCOUNT &&
45
- process.env.SNOWFLAKE_TEST_USER &&
46
- process.env.SNOWFLAKE_TEST_PASSWORD &&
47
- process.env.SNOWFLAKE_TEST_WAREHOUSE
48
- );
49
-
50
- const hasS3Credentials = () =>
51
- !!(
52
- process.env.S3_TEST_ACCESS_KEY_ID && process.env.S3_TEST_SECRET_ACCESS_KEY
53
- );
54
-
55
- const hasGCSCredentials = () =>
56
- !!(process.env.GCS_TEST_KEY_ID && process.env.GCS_TEST_SECRET);
57
-
58
- const readBigQueryServiceAccountJson = async (): Promise<string> =>
59
- fs.readFile(process.env.GOOGLE_APPLICATION_CREDENTIALS!, "utf-8");
60
-
61
- // `BIGQUERY_PUBLIC_DATA_*` env vars are populated by the
62
- // `Setup BigQuery public-data credentials` step in
63
- // `.github/workflows/connection-integration-tests.yml`. Kept separate
64
- // from the existing `BIGQUERY_TEST_*` vars (which point at the
65
- // org-scoped BQ_PRESTO_TRINO_KEY service account) so the two SAs can
66
- // coexist without one overwriting the other.
67
- const hasPublicDataBigQueryCredentials = () =>
68
- !!(
69
- process.env.BIGQUERY_PUBLIC_DATA_CREDENTIALS &&
70
- process.env.BIGQUERY_PUBLIC_DATA_PROJECT_ID
71
- );
72
-
73
- const readPublicDataBigQueryServiceAccountJson = async (): Promise<string> =>
74
- fs.readFile(process.env.BIGQUERY_PUBLIC_DATA_CREDENTIALS!, "utf-8");
75
-
76
- describe("connection integration tests", () => {
77
- const testEnvironmentPath = path.join(
78
- process.cwd(),
79
- "test-environment-connections",
80
- );
81
- let createdConnections: DuckDBConnection[] = [];
82
-
83
- beforeEach(async () => {
84
- await fs.mkdir(testEnvironmentPath, { recursive: true });
85
- });
86
-
87
- afterEach(async () => {
88
- sinon.restore();
89
-
90
- for (const conn of createdConnections) {
91
- try {
92
- await conn.close();
93
- } catch (error) {
94
- console.warn("Error closing connection:", error);
95
- }
96
- }
97
- createdConnections = [];
98
-
99
- const maxRetries = 5;
100
- const delay = 100;
101
- let lastError: unknown;
102
-
103
- for (let attempt = 0; attempt < maxRetries; attempt++) {
104
- try {
105
- await fs.rm(testEnvironmentPath, { recursive: true, force: true });
106
- return;
107
- } catch (error) {
108
- lastError = error;
109
- const errnoError = error as NodeJS.ErrnoException;
110
- if (errnoError.code !== "EBUSY") throw error;
111
- if (attempt < maxRetries - 1) {
112
- await new Promise((resolve) =>
113
- setTimeout(resolve, delay * Math.pow(2, attempt)),
114
- );
115
- }
116
- }
117
- }
118
-
119
- if ((lastError as NodeJS.ErrnoException).code !== "EBUSY") {
120
- throw lastError;
121
- }
122
- });
123
-
124
- describe("createEnvironmentConnections", () => {
125
- describe("DuckDB with PostgreSQL attachment", () => {
126
- it(
127
- "should create DuckDB connection with attached PostgreSQL database",
128
- async () => {
129
- if (!hasPostgresCredentials()) {
130
- console.log(
131
- "Skipping: PostgreSQL credentials not configured",
132
- );
133
- return;
134
- }
135
-
136
- const postgresAttachment: AttachedDatabase = {
137
- name: "pg_test",
138
- type: "postgres",
139
- postgresConnection: {
140
- host: process.env.POSTGRES_TEST_HOST,
141
- port: parseInt(process.env.POSTGRES_TEST_PORT || "5432"),
142
- userName: process.env.POSTGRES_TEST_USER!,
143
- password: process.env.POSTGRES_TEST_PASSWORD!,
144
- databaseName: process.env.POSTGRES_TEST_DATABASE,
145
- },
146
- };
147
-
148
- const duckdbConnection: ApiConnection = {
149
- name: "duckdb_with_postgres",
150
- type: "duckdb",
151
- duckdbConnection: { attachedDatabases: [postgresAttachment] },
152
- };
153
-
154
- const { malloyConnections, apiConnections } =
155
- await createEnvironmentConnections(
156
- [duckdbConnection],
157
- testEnvironmentPath,
158
- );
159
-
160
- expect(malloyConnections.size).toBe(1);
161
- expect(apiConnections.length).toBe(1);
162
-
163
- const connection = malloyConnections.get(
164
- "duckdb_with_postgres",
165
- ) as DuckDBConnection;
166
- expect(connection).toBeDefined();
167
- createdConnections.push(connection);
168
-
169
- const databases = await connection.runSQL("SHOW DATABASES");
170
- const dbNames = databases.rows.map(
171
- (row) => Object.values(row)[0],
172
- );
173
- expect(dbNames).toContain("pg_test");
174
-
175
- const result = await connection.runSQL(
176
- "SELECT 1 as test_value FROM pg_test.information_schema.tables LIMIT 1",
177
- );
178
- expect(result.rows).toBeDefined();
179
- },
180
- { timeout: 30000 },
181
- );
182
-
183
- it(
184
- "should list schemas from attached PostgreSQL database",
185
- async () => {
186
- if (!hasPostgresCredentials()) {
187
- console.log(
188
- "Skipping: PostgreSQL credentials not configured",
189
- );
190
- return;
191
- }
192
-
193
- const duckdbConnection: ApiConnection = {
194
- name: "duckdb_pg_schemas",
195
- type: "duckdb",
196
- duckdbConnection: {
197
- attachedDatabases: [
198
- {
199
- name: "pg_schemas",
200
- type: "postgres",
201
- postgresConnection: {
202
- host: process.env.POSTGRES_TEST_HOST,
203
- port: parseInt(
204
- process.env.POSTGRES_TEST_PORT || "5432",
205
- ),
206
- userName: process.env.POSTGRES_TEST_USER!,
207
- password: process.env.POSTGRES_TEST_PASSWORD!,
208
- databaseName: process.env.POSTGRES_TEST_DATABASE,
209
- },
210
- },
211
- ],
212
- },
213
- };
214
-
215
- const { malloyConnections } = await createEnvironmentConnections(
216
- [duckdbConnection],
217
- testEnvironmentPath,
218
- );
219
-
220
- const connection = malloyConnections.get(
221
- "duckdb_pg_schemas",
222
- ) as DuckDBConnection;
223
- createdConnections.push(connection);
224
-
225
- // listSchemas equivalent
226
- const result = await connection.runSQL(
227
- "SELECT schema_name FROM pg_schemas.information_schema.schemata ORDER BY schema_name",
228
- );
229
- expect(result.rows.length).toBeGreaterThan(0);
230
- const schemaNames = result.rows.map(
231
- (row) => Object.values(row)[0] as string,
232
- );
233
- expect(schemaNames).toContain("public");
234
- },
235
- { timeout: 30000 },
236
- );
237
-
238
- it(
239
- "should list tables from attached PostgreSQL database",
240
- async () => {
241
- if (!hasPostgresCredentials()) {
242
- console.log(
243
- "Skipping: PostgreSQL credentials not configured",
244
- );
245
- return;
246
- }
247
-
248
- const duckdbConnection: ApiConnection = {
249
- name: "duckdb_pg_tables",
250
- type: "duckdb",
251
- duckdbConnection: {
252
- attachedDatabases: [
253
- {
254
- name: "pg_tables_db",
255
- type: "postgres",
256
- postgresConnection: {
257
- host: process.env.POSTGRES_TEST_HOST,
258
- port: parseInt(
259
- process.env.POSTGRES_TEST_PORT || "5432",
260
- ),
261
- userName: process.env.POSTGRES_TEST_USER!,
262
- password: process.env.POSTGRES_TEST_PASSWORD!,
263
- databaseName: process.env.POSTGRES_TEST_DATABASE,
264
- },
265
- },
266
- ],
267
- },
268
- };
269
-
270
- const { malloyConnections } = await createEnvironmentConnections(
271
- [duckdbConnection],
272
- testEnvironmentPath,
273
- );
274
-
275
- const connection = malloyConnections.get(
276
- "duckdb_pg_tables",
277
- ) as DuckDBConnection;
278
- createdConnections.push(connection);
279
-
280
- // listTables equivalent
281
- const result = await connection.runSQL(`
282
- SELECT table_schema, table_name, table_type
283
- FROM pg_tables_db.information_schema.tables
284
- WHERE table_schema NOT IN ('pg_catalog', 'information_schema')
285
- ORDER BY table_schema, table_name
286
- `);
287
- expect(result.rows).toBeDefined();
288
- },
289
- { timeout: 30000 },
290
- );
291
-
292
- it(
293
- "should get table columns from attached PostgreSQL database",
294
- async () => {
295
- if (!hasPostgresCredentials()) {
296
- console.log(
297
- "Skipping: PostgreSQL credentials not configured",
298
- );
299
- return;
300
- }
301
-
302
- const duckdbConnection: ApiConnection = {
303
- name: "duckdb_pg_cols",
304
- type: "duckdb",
305
- duckdbConnection: {
306
- attachedDatabases: [
307
- {
308
- name: "pg_cols",
309
- type: "postgres",
310
- postgresConnection: {
311
- host: process.env.POSTGRES_TEST_HOST,
312
- port: parseInt(
313
- process.env.POSTGRES_TEST_PORT || "5432",
314
- ),
315
- userName: process.env.POSTGRES_TEST_USER!,
316
- password: process.env.POSTGRES_TEST_PASSWORD!,
317
- databaseName: process.env.POSTGRES_TEST_DATABASE,
318
- },
319
- },
320
- ],
321
- },
322
- };
323
-
324
- const { malloyConnections } = await createEnvironmentConnections(
325
- [duckdbConnection],
326
- testEnvironmentPath,
327
- );
328
-
329
- const connection = malloyConnections.get(
330
- "duckdb_pg_cols",
331
- ) as DuckDBConnection;
332
- createdConnections.push(connection);
333
-
334
- const result = await connection.runSQL(`
335
- SELECT column_name, data_type, is_nullable
336
- FROM pg_cols.information_schema.columns
337
- WHERE table_schema = 'information_schema'
338
- AND table_name = 'tables'
339
- ORDER BY ordinal_position
340
- `);
341
- expect(result.rows.length).toBeGreaterThan(0);
342
- const columnNames = result.rows.map(
343
- (row) => (row as Record<string, unknown>)["column_name"],
344
- );
345
- expect(columnNames).toContain("table_name");
346
- expect(columnNames).toContain("table_schema");
347
- },
348
- { timeout: 30000 },
349
- );
350
-
351
- it(
352
- "should handle PostgreSQL connection string format",
353
- async () => {
354
- if (!hasPostgresCredentials()) {
355
- console.log(
356
- "Skipping: PostgreSQL credentials not configured",
357
- );
358
- return;
359
- }
360
-
361
- const connectionString = `host=${process.env.POSTGRES_TEST_HOST} port=${process.env.POSTGRES_TEST_PORT || "5432"} dbname=${process.env.POSTGRES_TEST_DATABASE} user=${process.env.POSTGRES_TEST_USER} password=${process.env.POSTGRES_TEST_PASSWORD}`;
362
-
363
- const duckdbConnection: ApiConnection = {
364
- name: "duckdb_pg_string",
365
- type: "duckdb",
366
- duckdbConnection: {
367
- attachedDatabases: [
368
- {
369
- name: "pg_conn_string",
370
- type: "postgres",
371
- postgresConnection: { connectionString },
372
- },
373
- ],
374
- },
375
- };
376
-
377
- const { malloyConnections } = await createEnvironmentConnections(
378
- [duckdbConnection],
379
- testEnvironmentPath,
380
- );
381
-
382
- const connection = malloyConnections.get(
383
- "duckdb_pg_string",
384
- ) as DuckDBConnection;
385
- createdConnections.push(connection);
386
-
387
- const databases = await connection.runSQL("SHOW DATABASES");
388
- const dbNames = databases.rows.map(
389
- (row) => Object.values(row)[0],
390
- );
391
- expect(dbNames).toContain("pg_conn_string");
392
- },
393
- { timeout: 30000 },
394
- );
395
- });
396
-
397
- describe("DuckDB with BigQuery attachment", () => {
398
- it(
399
- "should create DuckDB connection with attached BigQuery database",
400
- async () => {
401
- if (!hasBigQueryCredentials()) {
402
- console.log("Skipping: BigQuery credentials not configured");
403
- return;
404
- }
405
-
406
- const serviceAccountJson =
407
- await readBigQueryServiceAccountJson();
408
-
409
- const duckdbConnection: ApiConnection = {
410
- name: "duckdb_with_bigquery",
411
- type: "duckdb",
412
- duckdbConnection: {
413
- attachedDatabases: [
414
- {
415
- name: "bq_test",
416
- type: "bigquery",
417
- bigqueryConnection: {
418
- defaultProjectId:
419
- process.env.BIGQUERY_TEST_PROJECT_ID!,
420
- serviceAccountKeyJson: serviceAccountJson,
421
- },
422
- },
423
- ],
424
- },
425
- };
426
-
427
- const { malloyConnections } = await createEnvironmentConnections(
428
- [duckdbConnection],
429
- testEnvironmentPath,
430
- );
431
-
432
- const connection = malloyConnections.get(
433
- "duckdb_with_bigquery",
434
- ) as DuckDBConnection;
435
- expect(connection).toBeDefined();
436
- createdConnections.push(connection);
437
-
438
- const databases = await connection.runSQL("SHOW DATABASES");
439
- const dbNames = databases.rows.map(
440
- (row) => Object.values(row)[0],
441
- );
442
- expect(dbNames).toContain("bq_test");
443
- },
444
- { timeout: 60000 },
445
- );
446
-
447
- it(
448
- "should list datasets (schemas) from attached BigQuery database",
449
- async () => {
450
- if (
451
- !hasBigQueryCredentials() ||
452
- !process.env.BIGQUERY_TEST_DATASET
453
- ) {
454
- console.log(
455
- "Skipping: GOOGLE_APPLICATION_CREDENTIALS, BIGQUERY_TEST_PROJECT_ID, or BIGQUERY_TEST_DATASET not configured",
456
- );
457
- return;
458
- }
459
-
460
- const serviceAccountJson =
461
- await readBigQueryServiceAccountJson();
462
-
463
- const duckdbConnection: ApiConnection = {
464
- name: "duckdb_bq_schemas",
465
- type: "duckdb",
466
- duckdbConnection: {
467
- attachedDatabases: [
468
- {
469
- name: "bq_schemas",
470
- type: "bigquery",
471
- bigqueryConnection: {
472
- defaultProjectId:
473
- process.env.BIGQUERY_TEST_PROJECT_ID!,
474
- serviceAccountKeyJson: serviceAccountJson,
475
- },
476
- },
477
- ],
478
- },
479
- };
480
-
481
- const { malloyConnections } = await createEnvironmentConnections(
482
- [duckdbConnection],
483
- testEnvironmentPath,
484
- );
485
-
486
- const connection = malloyConnections.get(
487
- "duckdb_bq_schemas",
488
- ) as DuckDBConnection;
489
- createdConnections.push(connection);
490
-
491
- const result = await connection.runSQL(
492
- "SELECT schema_name FROM bq_schemas.information_schema.schemata",
493
- );
494
- expect(result.rows.length).toBeGreaterThan(0);
495
- const datasetNames = result.rows.map(
496
- (row) => Object.values(row)[0] as string,
497
- );
498
- expect(datasetNames).toContain(
499
- process.env.BIGQUERY_TEST_DATASET!,
500
- );
501
- },
502
- { timeout: 60000 },
503
- );
504
-
505
- it(
506
- "should list tables from attached BigQuery dataset",
507
- async () => {
508
- if (
509
- !hasBigQueryCredentials() ||
510
- !process.env.BIGQUERY_TEST_DATASET
511
- ) {
512
- console.log(
513
- "Skipping: GOOGLE_APPLICATION_CREDENTIALS, BIGQUERY_TEST_PROJECT_ID, or BIGQUERY_TEST_DATASET not configured",
514
- );
515
- return;
516
- }
517
-
518
- const serviceAccountJson =
519
- await readBigQueryServiceAccountJson();
520
-
521
- const duckdbConnection: ApiConnection = {
522
- name: "duckdb_bq_tables",
523
- type: "duckdb",
524
- duckdbConnection: {
525
- attachedDatabases: [
526
- {
527
- name: "bq_tables",
528
- type: "bigquery",
529
- bigqueryConnection: {
530
- defaultProjectId:
531
- process.env.BIGQUERY_TEST_PROJECT_ID!,
532
- serviceAccountKeyJson: serviceAccountJson,
533
- },
534
- },
535
- ],
536
- },
537
- };
538
-
539
- const { malloyConnections } = await createEnvironmentConnections(
540
- [duckdbConnection],
541
- testEnvironmentPath,
542
- );
543
-
544
- const connection = malloyConnections.get(
545
- "duckdb_bq_tables",
546
- ) as DuckDBConnection;
547
- createdConnections.push(connection);
548
-
549
- const result = await connection.runSQL(`
550
- SELECT table_name, table_type
551
- FROM bq_tables.${process.env.BIGQUERY_TEST_DATASET!}.INFORMATION_SCHEMA.TABLES
552
- ORDER BY table_name
553
- `);
554
- expect(result.rows.length).toBeGreaterThan(0);
555
- },
556
- { timeout: 60000 },
557
- );
558
-
559
- it(
560
- "should get table columns from attached BigQuery table",
561
- async () => {
562
- if (
563
- !hasBigQueryCredentials() ||
564
- !process.env.BIGQUERY_TEST_TABLE
565
- ) {
566
- console.log(
567
- "Skipping: GOOGLE_APPLICATION_CREDENTIALS, BIGQUERY_TEST_PROJECT_ID, or BIGQUERY_TEST_TABLE not configured",
568
- );
569
- return;
570
- }
571
-
572
- const serviceAccountJson =
573
- await readBigQueryServiceAccountJson();
574
- const [dataset, table] =
575
- process.env.BIGQUERY_TEST_TABLE!.split(".");
576
-
577
- const duckdbConnection: ApiConnection = {
578
- name: "duckdb_bq_cols",
579
- type: "duckdb",
580
- duckdbConnection: {
581
- attachedDatabases: [
582
- {
583
- name: "bq_cols",
584
- type: "bigquery",
585
- bigqueryConnection: {
586
- defaultProjectId:
587
- process.env.BIGQUERY_TEST_PROJECT_ID!,
588
- serviceAccountKeyJson: serviceAccountJson,
589
- },
590
- },
591
- ],
592
- },
593
- };
594
-
595
- const { malloyConnections } = await createEnvironmentConnections(
596
- [duckdbConnection],
597
- testEnvironmentPath,
598
- );
599
-
600
- const connection = malloyConnections.get(
601
- "duckdb_bq_cols",
602
- ) as DuckDBConnection;
603
- createdConnections.push(connection);
604
-
605
- // getTable equivalent — fetch column metadata
606
- const result = await connection.runSQL(`
607
- SELECT column_name, data_type, is_nullable
608
- FROM bq_cols.${dataset}.INFORMATION_SCHEMA.COLUMNS
609
- WHERE table_name = '${table}'
610
- ORDER BY ordinal_position
611
- `);
612
- expect(result.rows.length).toBeGreaterThan(0);
613
- },
614
- { timeout: 60000 },
615
- );
616
-
617
- it(
618
- "should query data from attached BigQuery table",
619
- async () => {
620
- if (
621
- !hasBigQueryCredentials() ||
622
- !process.env.BIGQUERY_TEST_TABLE
623
- ) {
624
- console.log(
625
- "Skipping: GOOGLE_APPLICATION_CREDENTIALS, BIGQUERY_TEST_PROJECT_ID, or BIGQUERY_TEST_TABLE not configured",
626
- );
627
- return;
628
- }
629
-
630
- const serviceAccountJson =
631
- await readBigQueryServiceAccountJson();
632
-
633
- const duckdbConnection: ApiConnection = {
634
- name: "duckdb_bq_query",
635
- type: "duckdb",
636
- duckdbConnection: {
637
- attachedDatabases: [
638
- {
639
- name: "bq_query",
640
- type: "bigquery",
641
- bigqueryConnection: {
642
- defaultProjectId:
643
- process.env.BIGQUERY_TEST_PROJECT_ID!,
644
- serviceAccountKeyJson: serviceAccountJson,
645
- },
646
- },
647
- ],
648
- },
649
- };
650
-
651
- const { malloyConnections } = await createEnvironmentConnections(
652
- [duckdbConnection],
653
- testEnvironmentPath,
654
- );
655
-
656
- const connection = malloyConnections.get(
657
- "duckdb_bq_query",
658
- ) as DuckDBConnection;
659
- createdConnections.push(connection);
660
-
661
- const result = await connection.runSQL(
662
- `SELECT * FROM bq_query.${process.env.BIGQUERY_TEST_TABLE!} LIMIT 1`,
663
- );
664
- expect(result.rows).toBeDefined();
665
- expect(result.rows.length).toBeGreaterThan(0);
666
- },
667
- { timeout: 60000 },
668
- );
669
-
670
- it(
671
- "should validate BigQuery service account key format",
672
- async () => {
673
- await expect(
674
- createEnvironmentConnections(
675
- [
676
- {
677
- name: "duckdb_bq_invalid",
678
- type: "duckdb",
679
- duckdbConnection: {
680
- attachedDatabases: [
681
- {
682
- name: "bq_invalid",
683
- type: "bigquery",
684
- bigqueryConnection: {
685
- defaultProjectId: "test-environment",
686
- serviceAccountKeyJson: JSON.stringify({
687
- invalid: "key",
688
- }),
689
- },
690
- },
691
- ],
692
- },
693
- },
694
- ],
695
- testEnvironmentPath,
696
- ),
697
- ).rejects.toThrow(/Invalid service account key/);
698
- },
699
- { timeout: 30000 },
700
- );
701
- });
702
-
703
- describe("BigQuery direct connection (public-data)", () => {
704
- // Single end-to-end check that a real `bigquery`-typed
705
- // connection authenticated by the BIGQUERY_PUBLIC_DATA_SA
706
- // service account can actually round-trip a query against
707
- // bigquery-public-data. Skips locally if the creds aren't
708
- // set; runs in CI under .github/workflows/connection-integration-tests.yml.
709
- //
710
- // Picked `samples.shakespeare` because (a) it's tiny (~6 MB
711
- // / ~165k rows) so the BigQuery byte-scanned cost is well
712
- // under the 1 TB/month free tier even across many PR runs,
713
- // (b) it's a long-stable public table so the assertion stays
714
- // valid across years of reruns. If this assertion ever fails
715
- // the cause is almost certainly auth, not data drift.
716
- it(
717
- "should query bigquery-public-data via real BigQuery connection",
718
- async () => {
719
- if (!hasPublicDataBigQueryCredentials()) {
720
- console.log(
721
- "Skipping: BIGQUERY_PUBLIC_DATA_CREDENTIALS or BIGQUERY_PUBLIC_DATA_PROJECT_ID not configured",
722
- );
723
- return;
724
- }
725
-
726
- const serviceAccountJson =
727
- await readPublicDataBigQueryServiceAccountJson();
728
-
729
- const bqConnection: ApiConnection = {
730
- name: "bq_public_data",
731
- type: "bigquery",
732
- bigqueryConnection: {
733
- // Billing/auth project (the SA's own project_id);
734
- // the queried tables live in bigquery-public-data
735
- // and are referenced explicitly in the SQL below.
736
- defaultProjectId:
737
- process.env.BIGQUERY_PUBLIC_DATA_PROJECT_ID!,
738
- serviceAccountKeyJson: serviceAccountJson,
739
- },
740
- };
741
-
742
- const { malloyConnections } = await createEnvironmentConnections(
743
- [bqConnection],
744
- testEnvironmentPath,
745
- );
746
-
747
- const connection = malloyConnections.get("bq_public_data");
748
- expect(connection).toBeDefined();
749
-
750
- try {
751
- const result = await connection!.runSQL(
752
- "SELECT COUNT(*) AS row_count FROM `bigquery-public-data.samples.shakespeare`",
753
- );
754
- expect(result.rows.length).toBe(1);
755
- // Shakespeare has ~165k rows; bound on both sides so
756
- // we catch both "auth succeeded but query returned
757
- // nothing" and "got a confusingly large value" failure
758
- // modes, while staying tolerant of any minor row-count
759
- // jitter Google might introduce.
760
- const row = result.rows[0] as Record<string, unknown>;
761
- const rowCount = Number(row.row_count);
762
- expect(rowCount).toBeGreaterThan(100_000);
763
- expect(rowCount).toBeLessThan(200_000);
764
- } finally {
765
- // BigQuery driver holds an HTTP/2 client + auth refresh
766
- // state; close it explicitly so we don't leak across
767
- // the rest of the test run. (createdConnections is
768
- // typed for DuckDBConnection, so we can't use the
769
- // shared cleanup array here.)
770
- await connection?.close();
771
- }
772
- },
773
- { timeout: 60000 },
774
- );
775
- });
776
-
777
- describe("BigQuery package end-to-end (bq-hackernews)", () => {
778
- // Step 2 of Sagar's bun-setup-fixes ask: not just "does the
779
- // BigQuery driver round-trip a SQL query" (the previous
780
- // describe block covers that), but "does the publisher's
781
- // package-loading path successfully use the BigQuery
782
- // connection on behalf of a Malloy package."
783
- //
784
- // Mechanism: stand up a real EnvironmentStore against a
785
- // temp publisher.config.json that declares both the BQ
786
- // connection (with the BIGQUERY_PUBLIC_DATA_SA injected via
787
- // serviceAccountKeyJson) AND the bq-hackernews package
788
- // (loaded from credibledata/malloy-samples on GitHub).
789
- // EnvironmentStore.initialize then: clones the malloy-samples
790
- // repo, extracts the bigquery-hackernews subdirectory, parses
791
- // the package's Malloy models, and — crucially — introspects
792
- // their BigQuery table schemas during model compilation. A
793
- // successful Package.listModels() call therefore proves the
794
- // entire package-uses-connection path, not just the bare
795
- // driver auth.
796
- //
797
- // Cost: ~50 MB git clone (one time per test run since
798
- // publisher_data lives under testEnvironmentPath which the
799
- // afterEach cleans up) + ~6 MB BQ schema scan. Both well
800
- // under any meaningful free-tier limit. Test budget: 3 min
801
- // (clone is ~30-60s on GH runners; compile is ~10s).
802
- it(
803
- "should load bq-hackernews package and compile its models via real BQ",
804
- async () => {
805
- if (!hasPublicDataBigQueryCredentials()) {
806
- console.log(
807
- "Skipping: BIGQUERY_PUBLIC_DATA_CREDENTIALS or BIGQUERY_PUBLIC_DATA_PROJECT_ID not configured",
808
- );
809
- return;
810
- }
811
-
812
- const serviceAccountJson =
813
- await readPublicDataBigQueryServiceAccountJson();
814
-
815
- // Each test gets its own serverRoot so publisher_data
816
- // doesn't leak across tests; afterEach removes
817
- // testEnvironmentPath which carries the whole tree
818
- // away.
819
- const tempServerRoot = path.join(
820
- testEnvironmentPath,
821
- "bq-hackernews-pkg-test",
822
- );
823
- await fs.mkdir(tempServerRoot, { recursive: true });
824
-
825
- const config = {
826
- frozenConfig: false,
827
- environments: [
828
- {
829
- name: "malloy-samples",
830
- packages: [
831
- {
832
- name: "bigquery-hackernews",
833
- location:
834
- "https://github.com/credibledata/malloy-samples/tree/main/bigquery-hackernews",
835
- },
836
- ],
837
- connections: [
838
- {
839
- name: "bigquery",
840
- type: "bigquery",
841
- bigqueryConnection: {
842
- defaultProjectId:
843
- process.env
844
- .BIGQUERY_PUBLIC_DATA_PROJECT_ID!,
845
- serviceAccountKeyJson: serviceAccountJson,
846
- },
847
- },
848
- ],
849
- },
850
- ],
851
- };
852
- await fs.writeFile(
853
- path.join(tempServerRoot, "publisher.config.json"),
854
- JSON.stringify(config),
855
- );
856
-
857
- // Force the load-from-config init path. Without this,
858
- // EnvironmentStore.initialize() takes the load-from-DB
859
- // branch and only falls back to config when the DB is
860
- // empty (which it is here, by construction, but relying
861
- // on that is brittle if the test pattern is reused).
862
- //
863
- // SAFETY PRECONDITION: this flag is the SAME one users
864
- // opt into via `--init` / `start:init` to wipe persisted
865
- // storage and re-initialize from config (see CLAUDE.md
866
- // and `environment_store.ts:158`). It is destructive on
867
- // a real serverRoot. We only set it here because
868
- // `tempServerRoot` is a freshly-created empty directory
869
- // (path.join(testEnvironmentPath, "bq-hackernews-pkg-test")
870
- // mkdir'd ~20 lines above), so there is nothing to wipe.
871
- // DO NOT copy this pattern into a test that points
872
- // EnvironmentStore at a non-empty or shared serverRoot
873
- // — it will delete state.
874
- const previousInitializeStorage = process.env.INITIALIZE_STORAGE;
875
- process.env.INITIALIZE_STORAGE = "true";
876
-
877
- try {
878
- const envStore = new EnvironmentStore(tempServerRoot);
879
- await envStore.finishedInitialization;
880
-
881
- // initialize() swallows top-level errors and just calls
882
- // markNotReady(), so finishedInitialization always resolves
883
- // regardless of success; operationalState is what says it
884
- // succeeded. It does NOT imply every configured environment
885
- // and package loaded: a load failure is skipped and still
886
- // reports "serving", which is why loadErrors exists. Assert
887
- // on both, and on the package list below.
888
- const status = await envStore.getStatus();
889
- expect(status.operationalState).toBe("serving");
890
- expect(status.loadErrors).toBeUndefined();
891
-
892
- const env = await envStore.getEnvironment("malloy-samples");
893
- const apiPackages = await env.listPackages();
894
- expect(apiPackages.map((p) => p.name)).toContain(
895
- "bigquery-hackernews",
896
- );
897
-
898
- const apiConnections = env.listApiConnections();
899
- expect(apiConnections.map((c) => c.name)).toContain(
900
- "bigquery",
901
- );
902
-
903
- // The actual integration assertion. Package.listModels()
904
- // collects compile errors per-model and returns ALL
905
- // models in the result (with `error` populated on
906
- // failures) — so models.length>0 alone would pass even
907
- // if every BQ schema introspection failed. Filter for
908
- // models that compiled cleanly: at least one is the
909
- // real proof that BQ-on-behalf-of-a-package works.
910
- const pkg = await env.getPackage("bigquery-hackernews");
911
- const models = await pkg.listModels();
912
- const okModels = models.filter((m) => !m.error);
913
- if (okModels.length === 0) {
914
- console.error(
915
- "All bq-hackernews model compilations failed:",
916
- models.map((m) => `${m.path}: ${m.error}`),
917
- );
918
- }
919
- expect(okModels.length).toBeGreaterThan(0);
920
- } finally {
921
- if (previousInitializeStorage === undefined) {
922
- delete process.env.INITIALIZE_STORAGE;
923
- } else {
924
- process.env.INITIALIZE_STORAGE = previousInitializeStorage;
925
- }
926
- }
927
- },
928
- { timeout: 180000 },
929
- );
930
- });
931
-
932
- describe("DuckDB with Snowflake attachment", () => {
933
- it(
934
- "should create DuckDB connection with attached Snowflake database",
935
- async () => {
936
- if (!hasSnowflakeCredentials()) {
937
- console.log("Skipping: Snowflake credentials not configured");
938
- return;
939
- }
940
-
941
- const duckdbConnection: ApiConnection = {
942
- name: "duckdb_with_snowflake",
943
- type: "duckdb",
944
- duckdbConnection: {
945
- attachedDatabases: [
946
- {
947
- name: "sf_test",
948
- type: "snowflake",
949
- snowflakeConnection: {
950
- account: process.env.SNOWFLAKE_TEST_ACCOUNT!,
951
- username: process.env.SNOWFLAKE_TEST_USER!,
952
- password: process.env.SNOWFLAKE_TEST_PASSWORD!,
953
- warehouse: process.env.SNOWFLAKE_TEST_WAREHOUSE!,
954
- database: process.env.SNOWFLAKE_TEST_DATABASE,
955
- schema: process.env.SNOWFLAKE_TEST_SCHEMA,
956
- },
957
- },
958
- ],
959
- },
960
- };
961
-
962
- const { malloyConnections } = await createEnvironmentConnections(
963
- [duckdbConnection],
964
- testEnvironmentPath,
965
- );
966
-
967
- const connection = malloyConnections.get(
968
- "duckdb_with_snowflake",
969
- ) as DuckDBConnection;
970
- expect(connection).toBeDefined();
971
- createdConnections.push(connection);
972
-
973
- const databases = await connection.runSQL("SHOW DATABASES");
974
- const dbNames = databases.rows.map(
975
- (row) => Object.values(row)[0],
976
- );
977
- expect(dbNames).toContain("sf_test");
978
-
979
- const result = await connection.runSQL(
980
- "SELECT * FROM snowflake_query('SELECT 1 as test', 'sf_test_secret')",
981
- );
982
- expect(result.rows).toBeDefined();
983
- },
984
- { timeout: 30000 },
985
- );
986
-
987
- it("should validate required Snowflake fields", async () => {
988
- await expect(
989
- createEnvironmentConnections(
990
- [
991
- {
992
- name: "duckdb_sf_incomplete",
993
- type: "duckdb",
994
- duckdbConnection: {
995
- attachedDatabases: [
996
- {
997
- name: "sf_incomplete",
998
- type: "snowflake",
999
- snowflakeConnection: {
1000
- account: "test-account",
1001
- username: "test-user",
1002
- },
1003
- },
1004
- ],
1005
- },
1006
- },
1007
- ],
1008
- testEnvironmentPath,
1009
- ),
1010
- ).rejects.toThrow(/required/);
1011
- });
1012
- });
1013
-
1014
- describe("DuckDB with S3 attachment", () => {
1015
- it(
1016
- "should create DuckDB connection with S3 configuration",
1017
- async () => {
1018
- if (!hasS3Credentials()) {
1019
- console.log("Skipping: S3 credentials not configured");
1020
- return;
1021
- }
1022
-
1023
- const duckdbConnection: ApiConnection = {
1024
- name: "duckdb_with_s3",
1025
- type: "duckdb",
1026
- duckdbConnection: {
1027
- attachedDatabases: [
1028
- {
1029
- name: "s3_test",
1030
- type: "s3",
1031
- s3Connection: {
1032
- accessKeyId: process.env.S3_TEST_ACCESS_KEY_ID!,
1033
- secretAccessKey:
1034
- process.env.S3_TEST_SECRET_ACCESS_KEY!,
1035
- region: process.env.S3_TEST_REGION || "us-east-1",
1036
- },
1037
- },
1038
- ],
1039
- },
1040
- };
1041
-
1042
- const { malloyConnections } = await createEnvironmentConnections(
1043
- [duckdbConnection],
1044
- testEnvironmentPath,
1045
- );
1046
-
1047
- const connection = malloyConnections.get(
1048
- "duckdb_with_s3",
1049
- ) as DuckDBConnection;
1050
- expect(connection).toBeDefined();
1051
- createdConnections.push(connection);
1052
-
1053
- const secrets = await connection.runSQL(
1054
- "SELECT name FROM duckdb_secrets()",
1055
- );
1056
- const secretNames = secrets.rows.map(
1057
- (row) => Object.values(row)[0],
1058
- );
1059
- expect(
1060
- secretNames.some((name) => String(name).includes("s3")),
1061
- ).toBe(true);
1062
- },
1063
- { timeout: 30000 },
1064
- );
1065
-
1066
- it(
1067
- "should handle S3 with custom endpoint",
1068
- async () => {
1069
- if (!hasS3Credentials()) {
1070
- console.log("Skipping: S3 credentials not configured");
1071
- return;
1072
- }
1073
-
1074
- const duckdbConnection: ApiConnection = {
1075
- name: "duckdb_s3_custom",
1076
- type: "duckdb",
1077
- duckdbConnection: {
1078
- attachedDatabases: [
1079
- {
1080
- name: "s3_custom",
1081
- type: "s3",
1082
- s3Connection: {
1083
- accessKeyId: process.env.S3_TEST_ACCESS_KEY_ID!,
1084
- secretAccessKey:
1085
- process.env.S3_TEST_SECRET_ACCESS_KEY!,
1086
- region: "us-east-1",
1087
- endpoint: "https://s3.custom-endpoint.com",
1088
- },
1089
- },
1090
- ],
1091
- },
1092
- };
1093
-
1094
- const { malloyConnections } = await createEnvironmentConnections(
1095
- [duckdbConnection],
1096
- testEnvironmentPath,
1097
- );
1098
-
1099
- const connection = malloyConnections.get(
1100
- "duckdb_s3_custom",
1101
- ) as DuckDBConnection;
1102
- createdConnections.push(connection);
1103
- expect(connection).toBeDefined();
1104
- },
1105
- { timeout: 30000 },
1106
- );
1107
- });
1108
-
1109
- describe("DuckDB with GCS attachment", () => {
1110
- it(
1111
- "should create DuckDB connection with GCS configuration",
1112
- async () => {
1113
- if (!hasGCSCredentials()) {
1114
- console.log("Skipping: GCS credentials not configured");
1115
- return;
1116
- }
1117
-
1118
- const duckdbConnection: ApiConnection = {
1119
- name: "duckdb_with_gcs",
1120
- type: "duckdb",
1121
- duckdbConnection: {
1122
- attachedDatabases: [
1123
- {
1124
- name: "gcs_test",
1125
- type: "gcs",
1126
- gcsConnection: {
1127
- keyId: process.env.GCS_TEST_KEY_ID!,
1128
- secret: process.env.GCS_TEST_SECRET!,
1129
- },
1130
- },
1131
- ],
1132
- },
1133
- };
1134
-
1135
- const { malloyConnections } = await createEnvironmentConnections(
1136
- [duckdbConnection],
1137
- testEnvironmentPath,
1138
- );
1139
-
1140
- const connection = malloyConnections.get(
1141
- "duckdb_with_gcs",
1142
- ) as DuckDBConnection;
1143
- expect(connection).toBeDefined();
1144
- createdConnections.push(connection);
1145
-
1146
- const secrets = await connection.runSQL(
1147
- "SELECT name FROM duckdb_secrets()",
1148
- );
1149
- const secretNames = secrets.rows.map(
1150
- (row) => Object.values(row)[0],
1151
- );
1152
- expect(
1153
- secretNames.some((name) => String(name).includes("gcs")),
1154
- ).toBe(true);
1155
- },
1156
- { timeout: 30000 },
1157
- );
1158
- });
1159
-
1160
- describe("DuckDB with multiple attachments", () => {
1161
- it(
1162
- "should attach multiple databases to single DuckDB connection",
1163
- async () => {
1164
- const attachments: AttachedDatabase[] = [];
1165
-
1166
- if (hasPostgresCredentials()) {
1167
- attachments.push({
1168
- name: "pg_multi",
1169
- type: "postgres",
1170
- postgresConnection: {
1171
- host: process.env.POSTGRES_TEST_HOST,
1172
- port: parseInt(
1173
- process.env.POSTGRES_TEST_PORT || "5432",
1174
- ),
1175
- userName: process.env.POSTGRES_TEST_USER!,
1176
- password: process.env.POSTGRES_TEST_PASSWORD!,
1177
- databaseName: process.env.POSTGRES_TEST_DATABASE,
1178
- },
1179
- });
1180
- }
1181
-
1182
- if (hasBigQueryCredentials()) {
1183
- const serviceAccountJson =
1184
- await readBigQueryServiceAccountJson();
1185
- attachments.push({
1186
- name: "bq_multi",
1187
- type: "bigquery",
1188
- bigqueryConnection: {
1189
- defaultProjectId: process.env.BIGQUERY_TEST_PROJECT_ID!,
1190
- serviceAccountKeyJson: serviceAccountJson,
1191
- },
1192
- });
1193
- }
1194
-
1195
- if (hasS3Credentials()) {
1196
- attachments.push({
1197
- name: "s3_multi",
1198
- type: "s3",
1199
- s3Connection: {
1200
- accessKeyId: process.env.S3_TEST_ACCESS_KEY_ID!,
1201
- secretAccessKey: process.env.S3_TEST_SECRET_ACCESS_KEY!,
1202
- region: process.env.S3_TEST_REGION || "us-east-1",
1203
- },
1204
- });
1205
- }
1206
-
1207
- if (attachments.length === 0) {
1208
- console.log("Skipping: No database credentials configured");
1209
- return;
1210
- }
1211
-
1212
- const { malloyConnections } = await createEnvironmentConnections(
1213
- [
1214
- {
1215
- name: "duckdb_multi",
1216
- type: "duckdb",
1217
- duckdbConnection: { attachedDatabases: attachments },
1218
- },
1219
- ],
1220
- testEnvironmentPath,
1221
- );
1222
-
1223
- const connection = malloyConnections.get(
1224
- "duckdb_multi",
1225
- ) as DuckDBConnection;
1226
- expect(connection).toBeDefined();
1227
- createdConnections.push(connection);
1228
-
1229
- const databases = await connection.runSQL("SHOW DATABASES");
1230
- const dbNames = databases.rows.map(
1231
- (row) => Object.values(row)[0],
1232
- );
1233
- attachments.forEach((attachment) => {
1234
- expect(dbNames).toContain(attachment.name!);
1235
- });
1236
- },
1237
- { timeout: 60000 },
1238
- );
1239
- });
1240
-
1241
- describe("error handling", () => {
1242
- describe("DuckLake connection type", () => {
1243
- it(
1244
- "should create DuckLake connection",
1245
- async () => {
1246
- if (!hasPostgresCredentials() || !hasS3Credentials()) {
1247
- console.log(
1248
- "Skipping: PostgreSQL and S3 credentials not configured",
1249
- );
1250
- return;
1251
- }
1252
-
1253
- const { malloyConnections } =
1254
- await createEnvironmentConnections(
1255
- [
1256
- {
1257
- name: "ducklake_test",
1258
- type: "ducklake",
1259
- ducklakeConnection: {
1260
- catalog: {
1261
- postgresConnection: {
1262
- host: process.env.POSTGRES_TEST_HOST,
1263
- port: parseInt(
1264
- process.env.POSTGRES_TEST_PORT ||
1265
- "5432",
1266
- ),
1267
- userName:
1268
- process.env.POSTGRES_TEST_USER!,
1269
- password:
1270
- process.env.POSTGRES_TEST_PASSWORD!,
1271
- databaseName:
1272
- process.env.POSTGRES_TEST_DATABASE,
1273
- },
1274
- },
1275
- storage: {
1276
- bucketUrl:
1277
- process.env.S3_TEST_BUCKET_URL ||
1278
- "s3://test-bucket",
1279
- s3Connection: {
1280
- accessKeyId:
1281
- process.env.S3_TEST_ACCESS_KEY_ID!,
1282
- secretAccessKey:
1283
- process.env
1284
- .S3_TEST_SECRET_ACCESS_KEY!,
1285
- },
1286
- },
1287
- },
1288
- },
1289
- ],
1290
- testEnvironmentPath,
1291
- );
1292
-
1293
- const connection = malloyConnections.get(
1294
- "ducklake_test",
1295
- ) as DuckDBConnection;
1296
- createdConnections.push(connection);
1297
- expect(connection).toBeDefined();
1298
-
1299
- // Verify DuckLake database is attached
1300
- const databases = await connection.runSQL("SHOW DATABASES");
1301
- const dbNames = databases.rows.map(
1302
- (row) => Object.values(row)[0],
1303
- );
1304
- expect(dbNames).toContain("ducklake_test");
1305
- },
1306
- { timeout: 30000 },
1307
- );
1308
-
1309
- it("should throw error if DuckLake catalog connection is missing", async () => {
1310
- await expect(
1311
- createEnvironmentConnections(
1312
- [
1313
- {
1314
- name: "ducklake_no_catalog",
1315
- type: "ducklake",
1316
- ducklakeConnection: {
1317
- storage: {
1318
- bucketUrl: "s3://test-bucket",
1319
- s3Connection: {
1320
- accessKeyId: "test",
1321
- secretAccessKey: "test",
1322
- },
1323
- },
1324
- },
1325
- } as ApiConnection,
1326
- ],
1327
- testEnvironmentPath,
1328
- ),
1329
- ).rejects.toThrow(
1330
- /PostgreSQL connection configuration is required/,
1331
- );
1332
- });
1333
-
1334
- it("should throw error if DuckLake connection config is missing", async () => {
1335
- await expect(
1336
- createEnvironmentConnections(
1337
- [
1338
- {
1339
- name: "ducklake_missing_config",
1340
- type: "ducklake",
1341
- },
1342
- ],
1343
- testEnvironmentPath,
1344
- ),
1345
- ).rejects.toThrow(
1346
- /DuckLake connection configuration is missing/,
1347
- );
1348
- });
1349
-
1350
- // ── catalog.metadataSchema ──────────────────────────────────────
1351
- //
1352
- // Several independent DuckLake catalogs may share ONE catalog
1353
- // database when each attaches its own metadata schema. These use a
1354
- // LOCAL directory as DATA_PATH rather than S3/GCS, so they need only
1355
- // the Postgres service and are not skipped for want of object-store
1356
- // credentials.
1357
- describe("catalog.metadataSchema", () => {
1358
- const pgCatalog = () => ({
1359
- host: process.env.POSTGRES_TEST_HOST,
1360
- port: parseInt(process.env.POSTGRES_TEST_PORT || "5432"),
1361
- userName: process.env.POSTGRES_TEST_USER!,
1362
- password: process.env.POSTGRES_TEST_PASSWORD!,
1363
- databaseName: process.env.POSTGRES_TEST_DATABASE,
1364
- });
1365
-
1366
- const pgConnString = () => {
1367
- const pg = pgCatalog();
1368
- return (
1369
- `host=${pg.host} port=${pg.port} user=${pg.userName} ` +
1370
- `password=${pg.password}` +
1371
- (pg.databaseName ? ` dbname=${pg.databaseName}` : "")
1372
- );
1373
- };
1374
-
1375
- // A unique schema per test AND per run: the catalog database is
1376
- // shared across the specs in this file, and (locally) across runs,
1377
- // so a fixed name would let one test observe another's metadata.
1378
- const runId = Math.random().toString(36).slice(2, 8);
1379
- let schemaSeq = 0;
1380
- const createdSchemas: string[] = [];
1381
- const uniqueSchema = (label: string) => {
1382
- const schema = `dl_meta_${label}_${runId}_${++schemaSeq}`;
1383
- createdSchemas.push(schema);
1384
- return schema;
1385
- };
1386
-
1387
- // Leave the catalog database as we found it. These schemas are
1388
- // created by DuckLake inside a database shared with the other
1389
- // specs in this file, and enough of them makes a sibling test's
1390
- // schema enumeration miss `public`.
1391
- afterAll(async () => {
1392
- if (
1393
- !hasPostgresCredentials() ||
1394
- createdSchemas.length === 0
1395
- ) {
1396
- return;
1397
- }
1398
- const workDir = await fs.mkdtemp(
1399
- path.join(os.tmpdir(), "dl-cleanup-"),
1400
- );
1401
- const conn = new DuckDBConnection(
1402
- "dl_cleanup",
1403
- ":memory:",
1404
- workDir,
1405
- );
1406
- try {
1407
- await conn.runSQL(`INSTALL postgres`);
1408
- await conn.runSQL(`LOAD postgres`);
1409
- await conn.runSQL(
1410
- `ATTACH '${pgConnString()}' AS cleanup (TYPE postgres);`,
1411
- );
1412
- for (const schema of createdSchemas) {
1413
- // Quoted, because at least one test registers a mixed-case
1414
- // schema and an unquoted identifier would not match it. And
1415
- // reported rather than swallowed: a silently failing drop
1416
- // leaks one schema per run into the shared catalog database,
1417
- // which is the accumulation this hook exists to prevent.
1418
- await conn
1419
- .runSQL(`DROP SCHEMA cleanup."${schema}" CASCADE;`)
1420
- .catch((error: unknown) =>
1421
- console.warn(
1422
- `Failed to drop test schema "${schema}":`,
1423
- error,
1424
- ),
1425
- );
1426
- }
1427
- } catch (error) {
1428
- console.warn("DuckLake schema cleanup failed:", error);
1429
- } finally {
1430
- await conn.close().catch(() => undefined);
1431
- }
1432
- });
1433
-
1434
- const duckLakeConfig = async (
1435
- name: string,
1436
- metadataSchema?: string,
1437
- ) => {
1438
- const dataDir = path.join(
1439
- testEnvironmentPath,
1440
- `${name}_data`,
1441
- );
1442
- await fs.mkdir(dataDir, { recursive: true });
1443
- return {
1444
- catalog: {
1445
- postgresConnection: pgCatalog(),
1446
- ...(metadataSchema ? { metadataSchema } : {}),
1447
- },
1448
- storage: { bucketUrl: `${dataDir}/` },
1449
- } as components["schemas"]["DucklakeConnection"];
1450
- };
1451
-
1452
- // Bootstrap a catalog the way a BUILD does — a read-WRITE attach.
1453
- // The read-only serve attach cannot create one ("creating a new
1454
- // DuckLake is explicitly disabled"), so anything exercising the
1455
- // serve path needs this first. Also the natural place to assert
1456
- // that the build path carries METADATA_SCHEMA.
1457
- const bootstrapCatalog = async (
1458
- dbName: string,
1459
- cfg: components["schemas"]["DucklakeConnection"],
1460
- ): Promise<DuckDBConnection> => {
1461
- const workDir = await fs.mkdtemp(
1462
- path.join(os.tmpdir(), `dl-boot-${dbName}-`),
1463
- );
1464
- const conn = new DuckDBConnection(
1465
- `bootstrap_${dbName}`,
1466
- ":memory:",
1467
- workDir,
1468
- );
1469
- createdConnections.push(conn);
1470
- await attachDuckLakeReadWrite(conn, dbName, cfg);
1471
- return conn;
1472
- };
1473
-
1474
- const connect = async (defs: ApiConnection[]) => {
1475
- const { malloyConnections } =
1476
- await createEnvironmentConnections(
1477
- defs,
1478
- testEnvironmentPath,
1479
- );
1480
- const out = new Map<string, DuckDBConnection>();
1481
- for (const def of defs) {
1482
- const c = malloyConnections.get(
1483
- def.name!,
1484
- ) as DuckDBConnection;
1485
- createdConnections.push(c);
1486
- out.set(def.name!, c);
1487
- }
1488
- return out;
1489
- };
1490
-
1491
- const firstValue = (rows: Record<string, unknown>[]) =>
1492
- Number(Object.values(rows[0])[0]);
1493
-
1494
- it(
1495
- "puts DuckLake metadata in the configured schema, creating it",
1496
- async () => {
1497
- if (!hasPostgresCredentials()) {
1498
- console.log("Skipping: PostgreSQL not configured");
1499
- return;
1500
- }
1501
- const schema = uniqueSchema("lands");
1502
- const cfg = await duckLakeConfig("dl_lands", schema);
1503
- const c = await bootstrapCatalog("dl_lands", cfg);
1504
- await c.runSQL(
1505
- `CREATE OR REPLACE TABLE dl_lands.t AS SELECT 1 AS x`,
1506
- );
1507
-
1508
- // Read the catalog database directly: the ducklake_*
1509
- // bookkeeping tables must be in `schema`, which DuckLake
1510
- // created on attach (this test never created the schema).
1511
- await c.runSQL(
1512
- `ATTACH '${pgConnString()}' AS insp (TYPE postgres, READ_ONLY);`,
1513
- );
1514
- const found = await c.runSQL(
1515
- `SELECT count(*) AS n FROM insp.${schema}.ducklake_metadata;`,
1516
- );
1517
- expect(firstValue(found.rows)).toBeGreaterThan(0);
1518
- },
1519
- { timeout: 60000 },
1520
- );
1521
-
1522
- it(
1523
- "isolates two catalogs sharing one catalog database",
1524
- async () => {
1525
- if (!hasPostgresCredentials()) {
1526
- console.log("Skipping: PostgreSQL not configured");
1527
- return;
1528
- }
1529
- const a = await bootstrapCatalog(
1530
- "dl_a",
1531
- await duckLakeConfig("dl_a", uniqueSchema("a")),
1532
- );
1533
- const b = await bootstrapCatalog(
1534
- "dl_b",
1535
- await duckLakeConfig("dl_b", uniqueSchema("b")),
1536
- );
1537
-
1538
- await a.runSQL(
1539
- `CREATE OR REPLACE TABLE dl_a.only_in_a AS SELECT 1`,
1540
- );
1541
- await b.runSQL(
1542
- `CREATE OR REPLACE TABLE dl_b.only_in_b AS SELECT 2`,
1543
- );
1544
-
1545
- const tablesOf = async (
1546
- conn: DuckDBConnection,
1547
- db: string,
1548
- ) => {
1549
- const r = await conn.runSQL(
1550
- `SELECT table_name FROM duckdb_tables() WHERE database_name = '${db}';`,
1551
- );
1552
- return r.rows.map((row) => Object.values(row)[0]);
1553
- };
1554
-
1555
- // Each catalog sees its own table and NOT the other's —
1556
- // the whole point of a per-catalog metadata schema. Both
1557
- // live in ONE catalog database.
1558
- expect(await tablesOf(a, "dl_a")).toEqual(["only_in_a"]);
1559
- expect(await tablesOf(b, "dl_b")).toEqual(["only_in_b"]);
1560
- },
1561
- { timeout: 60000 },
1562
- );
1563
-
1564
- it(
1565
- "serves read-only from a catalog in a non-default schema",
1566
- async () => {
1567
- if (!hasPostgresCredentials()) {
1568
- console.log("Skipping: PostgreSQL not configured");
1569
- return;
1570
- }
1571
- // The read-only serve attach must pass METADATA_SCHEMA too:
1572
- // without it the attach looks in the catalog's default
1573
- // schema, finds nothing, and fails outright, because a
1574
- // read-only attach may not create a catalog.
1575
- const schema = uniqueSchema("serve");
1576
- const cfg = await duckLakeConfig("dl_serve", schema);
1577
- const boot = await bootstrapCatalog("dl_serve", cfg);
1578
- await boot.runSQL(
1579
- `CREATE OR REPLACE TABLE dl_serve.t AS SELECT 7 AS x`,
1580
- );
1581
-
1582
- const conns = await connect([
1583
- {
1584
- name: "dl_serve",
1585
- type: "ducklake",
1586
- ducklakeConnection: cfg,
1587
- } as ApiConnection,
1588
- ]);
1589
- const served = conns.get("dl_serve")!;
1590
- const r = await served.runSQL(`SELECT x FROM dl_serve.t;`);
1591
- expect(firstValue(r.rows)).toBe(7);
1592
- },
1593
- { timeout: 60000 },
1594
- );
1595
-
1596
- it(
1597
- "keeps the catalog-format preflight working for a non-default schema",
1598
- async () => {
1599
- if (!hasPostgresCredentials()) {
1600
- console.log("Skipping: PostgreSQL not configured");
1601
- return;
1602
- }
1603
- // The preflight reads `ducklake_metadata`, which lives in
1604
- // the configured schema. Read unqualified it would MISS,
1605
- // and the preflight fails SOFT (logs and returns) — so the
1606
- // range check would silently stop protecting exactly the
1607
- // catalogs setting this option. Poisoning the recorded
1608
- // version proves the read resolves: an unqualified read
1609
- // finds nothing and so raises nothing.
1610
- const schema = uniqueSchema("preflight");
1611
- const cfg = await duckLakeConfig("dl_pf", schema);
1612
- const boot = await bootstrapCatalog("dl_pf", cfg);
1613
- await boot.runSQL(
1614
- `ATTACH '${pgConnString()}' AS poison (TYPE postgres);`,
1615
- );
1616
- await boot.runSQL(
1617
- `UPDATE poison.${schema}.ducklake_metadata ` +
1618
- `SET value = '9999.0' WHERE key = 'version';`,
1619
- );
1620
-
1621
- // A fresh attach of the same catalog must now refuse with
1622
- // the clean typed range error rather than proceeding.
1623
- let err: unknown;
1624
- try {
1625
- await bootstrapCatalog("dl_pf_again", cfg);
1626
- } catch (e) {
1627
- err = e;
1628
- }
1629
- // The TYPE matters: converting a deep DuckDB failure into
1630
- // this typed refusal is the preflight's whole job, so a
1631
- // message match alone could not tell the two apart.
1632
- expect(err).toBeInstanceOf(UnsupportedCatalogFormatError);
1633
- // And it must be THIS catalog's poisoned format. Asserting
1634
- // the value keeps the test honest if some other schema in
1635
- // the shared catalog database holds a valid catalog: an
1636
- // unqualified read would report that one's format instead,
1637
- // which is exactly the bug being guarded against.
1638
- expect((err as Error).message).toContain("9999.0");
1639
- },
1640
- { timeout: 60000 },
1641
- );
1642
-
1643
- it(
1644
- "round-trips a mixed-case metadataSchema through the preflight",
1645
- async () => {
1646
- if (!hasPostgresCredentials()) {
1647
- console.log("Skipping: PostgreSQL not configured");
1648
- return;
1649
- }
1650
- // The validator accepts mixed case but nothing exercised it, so
1651
- // this covers the create -> preflight -> attach round-trip for
1652
- // such a name. Honest scope: it does NOT discriminate on the
1653
- // preflight's identifier quoting — measured, unquoted resolves
1654
- // too, because DuckDB matches identifiers case-insensitively.
1655
- // Poisoning the version is what proves the preflight read THIS
1656
- // catalog's metadata rather than missing silently.
1657
- const schema = `DlMixedCase_${runId}`;
1658
- createdSchemas.push(schema);
1659
- const cfg = await duckLakeConfig("dl_mixed", schema);
1660
- const boot = await bootstrapCatalog("dl_mixed", cfg);
1661
- await boot.runSQL(
1662
- `ATTACH '${pgConnString()}' AS poison_mixed (TYPE postgres);`,
1663
- );
1664
- await boot.runSQL(
1665
- `UPDATE poison_mixed."${schema}".ducklake_metadata ` +
1666
- `SET value = '9999.0' WHERE key = 'version';`,
1667
- );
1668
-
1669
- let err: unknown;
1670
- try {
1671
- await bootstrapCatalog("dl_mixed_again", cfg);
1672
- } catch (e) {
1673
- err = e;
1674
- }
1675
- expect(err).toBeInstanceOf(UnsupportedCatalogFormatError);
1676
- expect((err as Error).message).toContain("9999.0");
1677
- },
1678
- { timeout: 60000 },
1679
- );
1680
-
1681
- it(
1682
- "attaches without a metadataSchema exactly as before",
1683
- async () => {
1684
- if (!hasPostgresCredentials()) {
1685
- console.log("Skipping: PostgreSQL not configured");
1686
- return;
1687
- }
1688
- // Back-compat: the option is a conditional append, so an
1689
- // absent value emits the pre-existing ATTACH verbatim and
1690
- // DuckLake uses the catalog connection's default schema.
1691
- const cfg = await duckLakeConfig("dl_default");
1692
- const c = await bootstrapCatalog("dl_default", cfg);
1693
- await c.runSQL(
1694
- `CREATE OR REPLACE TABLE dl_default.t AS SELECT 7 AS x`,
1695
- );
1696
- const r = await c.runSQL(`SELECT x FROM dl_default.t;`);
1697
- expect(firstValue(r.rows)).toBe(7);
1698
-
1699
- await c.runSQL(
1700
- `ATTACH '${pgConnString()}' AS insp2 (TYPE postgres, READ_ONLY);`,
1701
- );
1702
- const inPublic = await c.runSQL(
1703
- `SELECT count(*) AS n FROM insp2.public.ducklake_metadata;`,
1704
- );
1705
- expect(firstValue(inPublic.rows)).toBeGreaterThan(0);
1706
- },
1707
- { timeout: 60000 },
1708
- );
1709
- });
1710
- });
1711
-
1712
- it("should throw error if DuckDB connection name conflicts with attached database", async () => {
1713
- await expect(
1714
- createEnvironmentConnections(
1715
- [
1716
- {
1717
- name: "conflict_db",
1718
- type: "duckdb",
1719
- duckdbConnection: {
1720
- attachedDatabases: [
1721
- {
1722
- name: "conflict_db",
1723
- type: "postgres",
1724
- postgresConnection: {
1725
- connectionString:
1726
- "postgresql://localhost/test",
1727
- },
1728
- },
1729
- ],
1730
- },
1731
- },
1732
- ],
1733
- testEnvironmentPath,
1734
- ),
1735
- ).rejects.toThrow(/cannot conflict/);
1736
- });
1737
-
1738
- it("should throw error if connection name is 'duckdb'", async () => {
1739
- await expect(
1740
- createEnvironmentConnections(
1741
- [
1742
- {
1743
- name: "duckdb",
1744
- type: "duckdb",
1745
- duckdbConnection: { attachedDatabases: [] },
1746
- },
1747
- ],
1748
- testEnvironmentPath,
1749
- ),
1750
- ).rejects.toThrow(/'duckdb' is reserved/);
1751
- });
1752
-
1753
- it("should reject DuckDB connections with no attachments", async () => {
1754
- // Env-level DuckDB connections must declare at least one
1755
- // attached foreign database; the empty-array case is operator
1756
- // confusion (the per-package "duckdb" sandbox already covers
1757
- // the plain-in-memory use case).
1758
- await expect(
1759
- createEnvironmentConnections(
1760
- [
1761
- {
1762
- name: "empty_duckdb",
1763
- type: "duckdb",
1764
- duckdbConnection: { attachedDatabases: [] },
1765
- },
1766
- ],
1767
- testEnvironmentPath,
1768
- ),
1769
- ).rejects.toThrow(/has no attached databases/);
1770
- });
1771
-
1772
- it("should reject unsupported DuckDB connector fields", async () => {
1773
- await expect(
1774
- createEnvironmentConnections(
1775
- [
1776
- {
1777
- name: "duckdb_with_setup_sql",
1778
- type: "duckdb",
1779
- duckdbConnection: {
1780
- attachedDatabases: [],
1781
- setupSQL: "INSTALL httpfs",
1782
- },
1783
- } as unknown as ApiConnection,
1784
- ],
1785
- testEnvironmentPath,
1786
- ),
1787
- ).rejects.toThrow(/Unsupported DuckDB connection field/);
1788
- });
1789
-
1790
- it("should reject environment-authored DuckDB policy fields", async () => {
1791
- await expect(
1792
- createEnvironmentConnections(
1793
- [
1794
- {
1795
- name: "duckdb_with_policy",
1796
- type: "duckdb",
1797
- duckdbConnection: {
1798
- attachedDatabases: [],
1799
- securityPolicy: "sandboxed",
1800
- },
1801
- } as unknown as ApiConnection,
1802
- ],
1803
- testEnvironmentPath,
1804
- ),
1805
- ).rejects.toThrow(/Unsupported DuckDB connection field/);
1806
- });
1807
-
1808
- it("should preserve Snowflake private-key auth options", async () => {
1809
- const { malloyConnections, releaseConnections } =
1810
- await createEnvironmentConnections(
1811
- [
1812
- {
1813
- name: "sf_private_key",
1814
- type: "snowflake",
1815
- snowflakeConnection: {
1816
- account: "test-account",
1817
- username: "test-user",
1818
- privateKey:
1819
- "-----BEGIN PRIVATE KEY-----MIIB-----END PRIVATE KEY-----",
1820
- warehouse: "test-warehouse",
1821
- },
1822
- },
1823
- ],
1824
- testEnvironmentPath,
1825
- );
1826
-
1827
- try {
1828
- const connection = malloyConnections.get(
1829
- "sf_private_key",
1830
- ) as unknown as { connOptions: Record<string, unknown> };
1831
- expect(connection.connOptions.authenticator).toBe(
1832
- "SNOWFLAKE_JWT",
1833
- );
1834
- expect(connection.connOptions.privateKey).toContain(
1835
- "BEGIN PRIVATE KEY",
1836
- );
1837
- } finally {
1838
- await releaseConnections();
1839
- }
1840
- });
1841
-
1842
- it("should translate Trino Peaka credentials to core extraCredential", () => {
1843
- const assembled = assembleEnvironmentConnections(
1844
- [
1845
- {
1846
- name: "trino_peaka",
1847
- type: "trino",
1848
- trinoConnection: {
1849
- server: "https://example.com",
1850
- port: 443,
1851
- catalog: "catalog",
1852
- schema: "schema",
1853
- user: "user",
1854
- peakaKey: "peaka-secret",
1855
- },
1856
- },
1857
- ],
1858
- testEnvironmentPath,
1859
- );
1860
-
1861
- expect(
1862
- assembled.pojo.connections.trino_peaka.extraCredential,
1863
- ).toEqual({ peakaKey: "peaka-secret" });
1864
- expect(
1865
- assembled.pojo.connections.trino_peaka.extraConfig,
1866
- ).toBeUndefined();
1867
- });
1868
-
1869
- it("should validate environment-level BigQuery service account keys", () => {
1870
- expect(() =>
1871
- assembleEnvironmentConnections(
1872
- [
1873
- {
1874
- name: "bq_invalid",
1875
- type: "bigquery",
1876
- bigqueryConnection: {
1877
- defaultProjectId: "test-environment",
1878
- serviceAccountKeyJson: '{"invalid":"key"}',
1879
- },
1880
- },
1881
- ],
1882
- testEnvironmentPath,
1883
- ),
1884
- ).toThrow(/missing "type" field/);
1885
- });
1886
-
1887
- it("should preserve PGSSLMODE for environment-level Postgres", () => {
1888
- const previousPgSslMode = process.env.PGSSLMODE;
1889
- process.env.PGSSLMODE = "require";
1890
- try {
1891
- const assembled = assembleEnvironmentConnections(
1892
- [
1893
- {
1894
- name: "pg_ssl",
1895
- type: "postgres",
1896
- postgresConnection: {
1897
- host: "localhost",
1898
- port: 5432,
1899
- userName: "user",
1900
- password: "pass",
1901
- databaseName: "db",
1902
- },
1903
- },
1904
- ],
1905
- testEnvironmentPath,
1906
- );
1907
-
1908
- expect(
1909
- assembled.pojo.connections.pg_ssl.connectionString,
1910
- ).toContain("sslmode=require");
1911
- } finally {
1912
- if (previousPgSslMode === undefined) {
1913
- delete process.env.PGSSLMODE;
1914
- } else {
1915
- process.env.PGSSLMODE = previousPgSslMode;
1916
- }
1917
- }
1918
- });
1919
-
1920
- it("should use environment-root-relative file paths for environment-level DuckDB", async () => {
1921
- const insideCsvPath = path.join(testEnvironmentPath, "inside.csv");
1922
- await fs.writeFile(insideCsvPath, "id\n1\n");
1923
-
1924
- const minimalGcsAttachment = {
1925
- name: "test_gcs",
1926
- type: "gcs" as const,
1927
- gcsConnection: {
1928
- keyId: "test-key-id",
1929
- secret: "test-secret",
1930
- },
1931
- };
1932
-
1933
- const { malloyConnections } = await createEnvironmentConnections(
1934
- [
1935
- {
1936
- name: "environment_scoped_duckdb",
1937
- type: "duckdb",
1938
- duckdbConnection: {
1939
- attachedDatabases: [minimalGcsAttachment],
1940
- },
1941
- },
1942
- ],
1943
- testEnvironmentPath,
1944
- );
1945
-
1946
- const connection = malloyConnections.get(
1947
- "environment_scoped_duckdb",
1948
- ) as DuckDBConnection;
1949
- createdConnections.push(connection);
1950
-
1951
- const assembled = assembleEnvironmentConnections(
1952
- [
1953
- {
1954
- name: "environment_scoped_duckdb",
1955
- type: "duckdb",
1956
- duckdbConnection: {
1957
- attachedDatabases: [minimalGcsAttachment],
1958
- },
1959
- },
1960
- ],
1961
- testEnvironmentPath,
1962
- );
1963
- expect(
1964
- assembled.pojo.connections.environment_scoped_duckdb
1965
- .workingDirectory,
1966
- ).toBeUndefined();
1967
- expect(
1968
- assembled.pojo.connections.environment_scoped_duckdb
1969
- .securityPolicy,
1970
- ).toBeUndefined();
1971
-
1972
- await expect(
1973
- connection.runSQL("SELECT * FROM read_csv_auto('inside.csv')"),
1974
- ).resolves.toBeDefined();
1975
- });
1976
-
1977
- it("should keep external access available for federated DuckDB entries", () => {
1978
- const assembled = assembleEnvironmentConnections(
1979
- [
1980
- {
1981
- name: "federated_duckdb",
1982
- type: "duckdb",
1983
- duckdbConnection: {
1984
- attachedDatabases: [
1985
- {
1986
- name: "pg",
1987
- type: "postgres",
1988
- postgresConnection: {
1989
- host: "localhost",
1990
- port: 5432,
1991
- userName: "user",
1992
- password: "pass",
1993
- databaseName: "db",
1994
- },
1995
- },
1996
- ],
1997
- },
1998
- },
1999
- ],
2000
- testEnvironmentPath,
2001
- );
2002
-
2003
- const entry = assembled.pojo.connections.federated_duckdb;
2004
- expect(entry.securityPolicy).toBeUndefined();
2005
- expect(entry.enableExternalAccess).toBeUndefined();
2006
- expect(entry.allowedDirectories).toBeUndefined();
2007
- });
2008
-
2009
- it("should keep external access available for MotherDuck entries", () => {
2010
- const assembled = assembleEnvironmentConnections(
2011
- [
2012
- {
2013
- name: "md",
2014
- type: "motherduck",
2015
- motherduckConnection: {
2016
- accessToken: "token",
2017
- database: "db",
2018
- },
2019
- },
2020
- ],
2021
- testEnvironmentPath,
2022
- );
2023
-
2024
- const entry = assembled.pojo.connections.md;
2025
- expect(entry.securityPolicy).toBeUndefined();
2026
- expect(entry.enableExternalAccess).toBeUndefined();
2027
- expect(entry.allowedDirectories).toBeUndefined();
2028
- });
2029
-
2030
- it("should handle already attached database gracefully", async () => {
2031
- if (!hasPostgresCredentials()) {
2032
- console.log("Skipping: PostgreSQL credentials not configured");
2033
- return;
2034
- }
2035
-
2036
- const postgresAttachment: AttachedDatabase = {
2037
- name: "pg_duplicate",
2038
- type: "postgres",
2039
- postgresConnection: {
2040
- host: process.env.POSTGRES_TEST_HOST,
2041
- port: parseInt(process.env.POSTGRES_TEST_PORT || "5432"),
2042
- userName: process.env.POSTGRES_TEST_USER!,
2043
- password: process.env.POSTGRES_TEST_PASSWORD!,
2044
- databaseName: process.env.POSTGRES_TEST_DATABASE,
2045
- },
2046
- };
2047
-
2048
- const { malloyConnections } = await createEnvironmentConnections(
2049
- [
2050
- {
2051
- name: "duckdb_duplicate_test",
2052
- type: "duckdb",
2053
- duckdbConnection: {
2054
- attachedDatabases: [
2055
- postgresAttachment,
2056
- postgresAttachment,
2057
- ],
2058
- },
2059
- },
2060
- ],
2061
- testEnvironmentPath,
2062
- );
2063
-
2064
- const connection = malloyConnections.get(
2065
- "duckdb_duplicate_test",
2066
- ) as DuckDBConnection;
2067
- createdConnections.push(connection);
2068
- expect(connection).toBeDefined();
2069
- });
2070
- });
2071
- });
2072
-
2073
- describe("testConnectionConfig", () => {
2074
- it(
2075
- "should successfully test valid PostgreSQL connection",
2076
- async () => {
2077
- if (!hasPostgresCredentials()) {
2078
- console.log("Skipping: PostgreSQL credentials not configured");
2079
- return;
2080
- }
2081
-
2082
- const result = await testConnectionConfig({
2083
- name: "test_postgres",
2084
- type: "postgres",
2085
- postgresConnection: {
2086
- host: process.env.POSTGRES_TEST_HOST,
2087
- port: parseInt(process.env.POSTGRES_TEST_PORT || "5432"),
2088
- userName: process.env.POSTGRES_TEST_USER!,
2089
- password: process.env.POSTGRES_TEST_PASSWORD!,
2090
- databaseName: process.env.POSTGRES_TEST_DATABASE,
2091
- },
2092
- });
2093
-
2094
- expect(result.status).toBe("ok");
2095
- expect(result.errorMessage).toBe("");
2096
- },
2097
- { timeout: 30000 },
2098
- );
2099
-
2100
- it(
2101
- "should fail for invalid PostgreSQL credentials",
2102
- async () => {
2103
- const result = await testConnectionConfig({
2104
- name: "test_postgres_invalid",
2105
- type: "postgres",
2106
- postgresConnection: {
2107
- host: process.env.POSTGRES_TEST_HOST,
2108
- port: parseInt(process.env.POSTGRES_TEST_PORT || "5432"),
2109
- userName: "invalid_user",
2110
- password: "invalid_password",
2111
- databaseName: "nonexistent",
2112
- },
2113
- });
2114
-
2115
- expect(result.status).toBe("failed");
2116
- expect(result.errorMessage).toBeDefined();
2117
- },
2118
- { timeout: 30000 },
2119
- );
2120
-
2121
- it("should fail for missing connection name", async () => {
2122
- const result = await testConnectionConfig({
2123
- name: "",
2124
- type: "postgres",
2125
- postgresConnection: {},
2126
- });
2127
-
2128
- expect(result.status).toBe("failed");
2129
- expect(result.errorMessage).toContain("name is required");
2130
- });
2131
- });
2132
-
2133
- describe("SQL injection prevention", () => {
2134
- it("should properly escape special characters in credentials", async () => {
2135
- if (!hasPostgresCredentials()) {
2136
- console.log("Skipping: PostgreSQL credentials not configured");
2137
- return;
2138
- }
2139
-
2140
- const { malloyConnections } = await createEnvironmentConnections(
2141
- [
2142
- {
2143
- name: "duckdb_special_chars",
2144
- type: "duckdb",
2145
- duckdbConnection: {
2146
- attachedDatabases: [
2147
- {
2148
- name: "pg_special",
2149
- type: "postgres",
2150
- postgresConnection: {
2151
- host: process.env.POSTGRES_TEST_HOST,
2152
- port: parseInt(
2153
- process.env.POSTGRES_TEST_PORT || "5432",
2154
- ),
2155
- userName: process.env.POSTGRES_TEST_USER!,
2156
- password: process.env.POSTGRES_TEST_PASSWORD!,
2157
- databaseName: process.env.POSTGRES_TEST_DATABASE,
2158
- },
2159
- },
2160
- ],
2161
- },
2162
- },
2163
- ],
2164
- testEnvironmentPath,
2165
- );
2166
-
2167
- const connection = malloyConnections.get(
2168
- "duckdb_special_chars",
2169
- ) as DuckDBConnection;
2170
- createdConnections.push(connection);
2171
- expect(connection).toBeDefined();
2172
- });
2173
- });
2174
-
2175
- describe("connection attributes", () => {
2176
- it(
2177
- "should return correct attributes for DuckDB connection",
2178
- async () => {
2179
- if (!hasPostgresCredentials()) {
2180
- console.log("Skipping: PostgreSQL credentials not configured");
2181
- return;
2182
- }
2183
-
2184
- const { apiConnections } = await createEnvironmentConnections(
2185
- [
2186
- {
2187
- name: "duckdb_attrs",
2188
- type: "duckdb",
2189
- duckdbConnection: {
2190
- attachedDatabases: [
2191
- {
2192
- name: "pg_attrs",
2193
- type: "postgres",
2194
- postgresConnection: {
2195
- host: process.env.POSTGRES_TEST_HOST,
2196
- port: parseInt(
2197
- process.env.POSTGRES_TEST_PORT || "5432",
2198
- ),
2199
- userName: process.env.POSTGRES_TEST_USER!,
2200
- password: process.env.POSTGRES_TEST_PASSWORD!,
2201
- databaseName:
2202
- process.env.POSTGRES_TEST_DATABASE,
2203
- },
2204
- },
2205
- ],
2206
- },
2207
- },
2208
- ],
2209
- testEnvironmentPath,
2210
- );
2211
-
2212
- const connection = apiConnections[0];
2213
- expect(connection.attributes).toBeDefined();
2214
- expect(connection.attributes?.dialectName).toBe("duckdb");
2215
- expect(typeof connection.attributes?.canPersist).toBe("boolean");
2216
- expect(typeof connection.attributes?.canStream).toBe("boolean");
2217
- expect(typeof connection.attributes?.isPool).toBe("boolean");
2218
- },
2219
- { timeout: 30000 },
2220
- );
2221
- });
2222
- });
2223
-
2224
- describe("buildProxiedSslQuery", () => {
2225
- const ORIG_CA = process.env.NODE_EXTRA_CA_CERTS;
2226
- afterEach(() => {
2227
- if (ORIG_CA === undefined) delete process.env.NODE_EXTRA_CA_CERTS;
2228
- else process.env.NODE_EXTRA_CA_CERTS = ORIG_CA;
2229
- });
2230
-
2231
- it("defaults to no-verify when sslmode is unset (force-SSL target isn't rejected for plaintext)", () => {
2232
- expect(buildProxiedSslQuery("conn")).toBe("?sslmode=no-verify");
2233
- });
2234
-
2235
- it("returns no-verify for explicit no-verify", () => {
2236
- expect(buildProxiedSslQuery("conn", "no-verify")).toBe(
2237
- "?sslmode=no-verify",
2238
- );
2239
- });
2240
-
2241
- it("emits an explicit sslmode=disable, never an empty query", () => {
2242
- // Regression: an empty query lets pg fall back to the deployment's PGSSLMODE
2243
- // env, which would verify and fail the 127.0.0.1 hostname check.
2244
- expect(buildProxiedSslQuery("conn", "disable")).toBe("?sslmode=disable");
2245
- });
2246
-
2247
- it("builds a chain-verifying, hostname-skipped query for verify-ca", () => {
2248
- process.env.NODE_EXTRA_CA_CERTS = "/etc/ssl/certs/rds bundle.pem";
2249
- const q = buildProxiedSslQuery("conn", "verify-ca");
2250
- expect(q).toContain("uselibpqcompat=true");
2251
- expect(q).toContain("sslmode=verify-ca");
2252
- expect(q).toContain(
2253
- `sslrootcert=${encodeURIComponent("/etc/ssl/certs/rds bundle.pem")}`,
2254
- );
2255
- });
2256
-
2257
- it("throws for verify-ca when no CA bundle is available", () => {
2258
- delete process.env.NODE_EXTRA_CA_CERTS;
2259
- expect(() => buildProxiedSslQuery("conn", "verify-ca")).toThrow(
2260
- /no trusted CA bundle/i,
2261
- );
2262
- });
2263
-
2264
- it("throws for an unsupported sslmode", () => {
2265
- expect(() => buildProxiedSslQuery("conn", "require")).toThrow(
2266
- /unsupported sslmode/i,
2267
- );
2268
- });
2269
-
2270
- it("verify-ca connectionString parses to an ssl config (pg-connection-string version guard)", async () => {
2271
- // uselibpqcompat=true (pg-connection-string >= 2.7) is what turns verify-ca
2272
- // into "verify chain, skip hostname". If pg is ever downgraded below that
2273
- // floor, sslmode would be ignored — this guard fails loudly instead.
2274
- const caPath = path.join(os.tmpdir(), `proxy-ca-${process.pid}.pem`);
2275
- await fs.writeFile(
2276
- caPath,
2277
- "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n",
2278
- );
2279
- process.env.NODE_EXTRA_CA_CERTS = caPath;
2280
- try {
2281
- const q = buildProxiedSslQuery("conn", "verify-ca");
2282
- const { parse } = await import("pg-connection-string");
2283
- const parsed = parse(`postgresql://127.0.0.1:5432/db${q}`);
2284
- expect(parsed.ssl).toBeTruthy();
2285
- expect(typeof parsed.ssl).toBe("object");
2286
- } finally {
2287
- await fs.rm(caPath, { force: true });
2288
- }
2289
- });
2290
- });
2291
-
2292
- describe("resolveProxiedTls", () => {
2293
- it("routes verify-full to a raw ssl object with servername = the real host", () => {
2294
- // verify-full (chain + hostname) can't ride the connectionString through a
2295
- // tunnel — it needs ssl.servername set to the real DB host so pg checks the
2296
- // cert's SAN against it, not against the 127.0.0.1 socket address.
2297
- const { query, ssl } = resolveProxiedTls(
2298
- "conn",
2299
- "db.internal.example.com",
2300
- "verify-full",
2301
- );
2302
- expect(query).toBe("");
2303
- expect(ssl).toEqual({
2304
- servername: "db.internal.example.com",
2305
- rejectUnauthorized: true,
2306
- });
2307
- });
2308
-
2309
- it("throws for verify-full with no host (servername would be undefined)", () => {
2310
- expect(() => resolveProxiedTls("conn", undefined, "verify-full")).toThrow(
2311
- /verify-full.*no host/i,
2312
- );
2313
- });
2314
-
2315
- it("routes the connectionString-param modes through buildProxiedSslQuery with no ssl object", () => {
2316
- // verify-ca needs a CA bundle to build its query, so set one for the loop.
2317
- const prior = process.env.NODE_EXTRA_CA_CERTS;
2318
- process.env.NODE_EXTRA_CA_CERTS = "/etc/ssl/certs/bundle.pem";
2319
- try {
2320
- for (const mode of ["disable", "no-verify", "verify-ca", undefined]) {
2321
- const { query, ssl } = resolveProxiedTls("conn", "127.0.0.1", mode);
2322
- expect(query).toBe(buildProxiedSslQuery("conn", mode));
2323
- expect(ssl).toBeUndefined();
2324
- }
2325
- } finally {
2326
- if (prior === undefined) delete process.env.NODE_EXTRA_CA_CERTS;
2327
- else process.env.NODE_EXTRA_CA_CERTS = prior;
2328
- }
2329
- });
2330
-
2331
- it("verify-full: a bare connectionString does not clobber the explicit ssl object (pg merge guard)", async () => {
2332
- // Load-bearing invariant: pg's ConnectionParameters does
2333
- // Object.assign({}, config, parse(connectionString)). A verify-full
2334
- // connectionString carries no sslmode, so pg-connection-string adds no
2335
- // `ssl` key and our explicit { servername, rejectUnauthorized } survives
2336
- // the merge. If a future pg bump changes this — or we accidentally emit an
2337
- // sslmode into the query — this guard fails loudly instead of silently
2338
- // downgrading verify-full.
2339
- const { query, ssl } = resolveProxiedTls(
2340
- "conn",
2341
- "db.internal.example.com",
2342
- "verify-full",
2343
- );
2344
- const { default: ConnectionParameters } = await import(
2345
- "pg/lib/connection-parameters.js"
2346
- );
2347
- const params = new (ConnectionParameters as unknown as new (
2348
- c: unknown,
2349
- ) => {
2350
- ssl: unknown;
2351
- })({
2352
- connectionString: `postgresql://u:p@127.0.0.1:5432/db${query}`,
2353
- ssl,
2354
- });
2355
- expect(params.ssl).toEqual({
2356
- servername: "db.internal.example.com",
2357
- rejectUnauthorized: true,
2358
- });
2359
- });
2360
- });