@malloy-publisher/server 0.0.235 → 0.0.236

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (301) hide show
  1. package/dist/app/api-doc.yaml +47 -17
  2. package/dist/package_load_worker.mjs +8 -1
  3. package/dist/server.mjs +471 -54
  4. package/package.json +5 -2
  5. package/.eslintrc.json +0 -44
  6. package/.prettierignore +0 -1
  7. package/.prettierrc +0 -4
  8. package/build.ts +0 -97
  9. package/eslint.config.mjs +0 -76
  10. package/k6-tests/README.md +0 -144
  11. package/k6-tests/bun.lock +0 -797
  12. package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
  13. package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
  14. package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
  15. package/k6-tests/load-test/load-test-crud.ts +0 -142
  16. package/k6-tests/load-test/load-test.ts +0 -784
  17. package/k6-tests/package.json +0 -27
  18. package/k6-tests/scripts/run-with-otel.sh +0 -50
  19. package/k6-tests/smoke-test/smoke-test.ts +0 -158
  20. package/k6-tests/tsconfig.json +0 -29
  21. package/k6-tests/utils/client_factory.ts +0 -128
  22. package/k6-tests/utils/common.ts +0 -489
  23. package/k6-tests/utils/logger.ts +0 -62
  24. package/k6-tests/utils/types.d.ts +0 -35
  25. package/malloy_mcp.dxt +0 -0
  26. package/publisher.config.example.bigquery.json +0 -33
  27. package/publisher.config.example.duckdb.json +0 -23
  28. package/publisher.config.json +0 -23
  29. package/scripts/bake-duckdb-extensions.js +0 -110
  30. package/src/config.spec.ts +0 -1535
  31. package/src/config.theme.spec.ts +0 -178
  32. package/src/config.ts +0 -1142
  33. package/src/constants.ts +0 -105
  34. package/src/controller/compile.controller.ts +0 -40
  35. package/src/controller/connection.controller.spec.ts +0 -849
  36. package/src/controller/connection.controller.ts +0 -901
  37. package/src/controller/database.controller.ts +0 -24
  38. package/src/controller/materialization.controller.spec.ts +0 -293
  39. package/src/controller/materialization.controller.ts +0 -319
  40. package/src/controller/model.controller.ts +0 -164
  41. package/src/controller/package.controller.spec.ts +0 -273
  42. package/src/controller/package.controller.ts +0 -306
  43. package/src/controller/query.controller.ts +0 -152
  44. package/src/controller/theme.controller.ts +0 -83
  45. package/src/controller/watch-mode.controller.ts +0 -243
  46. package/src/data_styles.ts +0 -71
  47. package/src/default-publisher.config.json +0 -23
  48. package/src/ducklake_version.spec.ts +0 -163
  49. package/src/ducklake_version.ts +0 -153
  50. package/src/errors.spec.ts +0 -119
  51. package/src/errors.ts +0 -237
  52. package/src/filter_deprecation.spec.ts +0 -64
  53. package/src/filter_deprecation.ts +0 -42
  54. package/src/health.spec.ts +0 -123
  55. package/src/health.ts +0 -233
  56. package/src/heap_check.spec.ts +0 -144
  57. package/src/heap_check.ts +0 -144
  58. package/src/instrumentation.ts +0 -214
  59. package/src/json_utils.spec.ts +0 -51
  60. package/src/json_utils.ts +0 -33
  61. package/src/logger.spec.ts +0 -210
  62. package/src/logger.ts +0 -251
  63. package/src/malloy_pin_prereqs.spec.ts +0 -25
  64. package/src/materialization_metrics.spec.ts +0 -169
  65. package/src/materialization_metrics.ts +0 -346
  66. package/src/mcp/error_messages.spec.ts +0 -35
  67. package/src/mcp/error_messages.ts +0 -183
  68. package/src/mcp/handler_utils.spec.ts +0 -108
  69. package/src/mcp/handler_utils.ts +0 -251
  70. package/src/mcp/mcp_constants.ts +0 -6
  71. package/src/mcp/query_envelope.spec.ts +0 -229
  72. package/src/mcp/query_envelope.ts +0 -240
  73. package/src/mcp/server.protocol.spec.ts +0 -250
  74. package/src/mcp/server.ts +0 -87
  75. package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
  76. package/src/mcp/skills/build_skills_bundle.ts +0 -176
  77. package/src/mcp/skills/skills_bundle.json +0 -1
  78. package/src/mcp/skills/skills_bundle.spec.ts +0 -171
  79. package/src/mcp/tool_response.spec.ts +0 -108
  80. package/src/mcp/tool_response.ts +0 -138
  81. package/src/mcp/tools/compile_tool.spec.ts +0 -315
  82. package/src/mcp/tools/compile_tool.ts +0 -208
  83. package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
  84. package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
  85. package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
  86. package/src/mcp/tools/docs_search_tool.ts +0 -128
  87. package/src/mcp/tools/embedding_index.spec.ts +0 -1236
  88. package/src/mcp/tools/embedding_index.ts +0 -808
  89. package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
  90. package/src/mcp/tools/execute_query_tool.ts +0 -316
  91. package/src/mcp/tools/get_context_eval.ts +0 -275
  92. package/src/mcp/tools/get_context_tool.spec.ts +0 -592
  93. package/src/mcp/tools/get_context_tool.ts +0 -585
  94. package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
  95. package/src/mcp/tools/reload_package_tool.ts +0 -132
  96. package/src/mcp_config.spec.ts +0 -919
  97. package/src/mcp_config.ts +0 -425
  98. package/src/oom_guards.integration.spec.ts +0 -269
  99. package/src/package_load/package_load_pool.spec.ts +0 -252
  100. package/src/package_load/package_load_pool.ts +0 -931
  101. package/src/package_load/package_load_worker.ts +0 -1034
  102. package/src/package_load/protocol.ts +0 -364
  103. package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
  104. package/src/package_load/rpc_wait_accountant.ts +0 -76
  105. package/src/package_load_metrics.spec.ts +0 -114
  106. package/src/package_load_metrics.ts +0 -127
  107. package/src/path_safety.spec.ts +0 -158
  108. package/src/path_safety.ts +0 -146
  109. package/src/pg_helpers.spec.ts +0 -223
  110. package/src/pg_helpers.ts +0 -52
  111. package/src/query_cap_metrics.spec.ts +0 -89
  112. package/src/query_cap_metrics.ts +0 -116
  113. package/src/query_concurrency.spec.ts +0 -247
  114. package/src/query_concurrency.ts +0 -237
  115. package/src/query_metadata_metrics.ts +0 -49
  116. package/src/query_param_utils.ts +0 -29
  117. package/src/query_timeout.spec.ts +0 -224
  118. package/src/query_timeout.ts +0 -179
  119. package/src/runtime/publisher.js +0 -328
  120. package/src/server-old.ts +0 -988
  121. package/src/server.ts +0 -2059
  122. package/src/service/annotations.spec.ts +0 -118
  123. package/src/service/annotations.ts +0 -91
  124. package/src/service/authorize.spec.ts +0 -154
  125. package/src/service/authorize.ts +0 -498
  126. package/src/service/authorize_integration.spec.ts +0 -2041
  127. package/src/service/build_plan.spec.ts +0 -685
  128. package/src/service/build_plan.ts +0 -797
  129. package/src/service/compile_authorize.spec.ts +0 -85
  130. package/src/service/compile_fragment_techniques.spec.ts +0 -156
  131. package/src/service/connection.spec.ts +0 -2360
  132. package/src/service/connection.ts +0 -2187
  133. package/src/service/connection_config.spec.ts +0 -734
  134. package/src/service/connection_config.ts +0 -899
  135. package/src/service/connection_federation.spec.ts +0 -184
  136. package/src/service/connection_fingerprint.spec.ts +0 -102
  137. package/src/service/connection_service.spec.ts +0 -711
  138. package/src/service/connection_service.ts +0 -291
  139. package/src/service/cron_evaluator.spec.ts +0 -79
  140. package/src/service/cron_evaluator.ts +0 -105
  141. package/src/service/db_utils.spec.ts +0 -1085
  142. package/src/service/db_utils.ts +0 -1502
  143. package/src/service/duckdb_instance_isolation.spec.ts +0 -137
  144. package/src/service/ducklake_lazy_attach.spec.ts +0 -110
  145. package/src/service/embedding_provider.spec.ts +0 -329
  146. package/src/service/embedding_provider.ts +0 -236
  147. package/src/service/environment.ts +0 -2191
  148. package/src/service/environment_admission.spec.ts +0 -344
  149. package/src/service/environment_compile.spec.ts +0 -197
  150. package/src/service/environment_store.spec.ts +0 -2274
  151. package/src/service/environment_store.ts +0 -2434
  152. package/src/service/environment_store_anchoring.spec.ts +0 -107
  153. package/src/service/environment_store_clone.spec.ts +0 -350
  154. package/src/service/explore_visibility.spec.ts +0 -434
  155. package/src/service/exports_probe.spec.ts +0 -107
  156. package/src/service/extension_fetch_policy.spec.ts +0 -256
  157. package/src/service/filter.spec.ts +0 -458
  158. package/src/service/filter.ts +0 -341
  159. package/src/service/filter_bypass.spec.ts +0 -418
  160. package/src/service/filter_integration.spec.ts +0 -1004
  161. package/src/service/freshness.spec.ts +0 -183
  162. package/src/service/freshness.ts +0 -112
  163. package/src/service/gcs_s3_utils.ts +0 -427
  164. package/src/service/given.ts +0 -105
  165. package/src/service/givens_integration.spec.ts +0 -219
  166. package/src/service/manifest_loader.spec.ts +0 -189
  167. package/src/service/manifest_loader.ts +0 -156
  168. package/src/service/materialization_build_session.spec.ts +0 -435
  169. package/src/service/materialization_build_session.ts +0 -681
  170. package/src/service/materialization_config_validation.spec.ts +0 -99
  171. package/src/service/materialization_config_validation.ts +0 -120
  172. package/src/service/materialization_eligibility.spec.ts +0 -158
  173. package/src/service/materialization_eligibility.ts +0 -305
  174. package/src/service/materialization_schedule_surface.spec.ts +0 -256
  175. package/src/service/materialization_scheduler.spec.ts +0 -289
  176. package/src/service/materialization_scheduler.ts +0 -304
  177. package/src/service/materialization_serve_transform.spec.ts +0 -1003
  178. package/src/service/materialization_serve_transform.ts +0 -779
  179. package/src/service/materialization_service.spec.ts +0 -2373
  180. package/src/service/materialization_service.ts +0 -2342
  181. package/src/service/materialization_test_fixtures.ts +0 -249
  182. package/src/service/model.spec.ts +0 -1157
  183. package/src/service/model.ts +0 -3370
  184. package/src/service/model_limits.spec.ts +0 -209
  185. package/src/service/model_limits.ts +0 -131
  186. package/src/service/model_storage_serve.spec.ts +0 -193
  187. package/src/service/model_storage_serve_joins.spec.ts +0 -193
  188. package/src/service/package.spec.ts +0 -612
  189. package/src/service/package.ts +0 -1799
  190. package/src/service/package_manifest.spec.ts +0 -230
  191. package/src/service/package_manifest.ts +0 -241
  192. package/src/service/package_memory_governor.spec.ts +0 -173
  193. package/src/service/package_memory_governor.ts +0 -233
  194. package/src/service/package_quote_bound_tables.spec.ts +0 -171
  195. package/src/service/package_race.spec.ts +0 -208
  196. package/src/service/package_reload_safety.spec.ts +0 -193
  197. package/src/service/package_rollback.spec.ts +0 -190
  198. package/src/service/package_worker_path.spec.ts +0 -645
  199. package/src/service/path_injection.spec.ts +0 -39
  200. package/src/service/persist_annotation_validation.spec.ts +0 -89
  201. package/src/service/persist_annotation_validation.ts +0 -59
  202. package/src/service/persistence_policy.spec.ts +0 -374
  203. package/src/service/proxy.spec.ts +0 -414
  204. package/src/service/proxy.ts +0 -248
  205. package/src/service/query_boundary.spec.ts +0 -470
  206. package/src/service/query_metadata.spec.ts +0 -408
  207. package/src/service/query_metadata.ts +0 -492
  208. package/src/service/query_metadata_identity.spec.ts +0 -149
  209. package/src/service/query_text.spec.ts +0 -79
  210. package/src/service/query_text.ts +0 -43
  211. package/src/service/quoting.spec.ts +0 -135
  212. package/src/service/quoting.ts +0 -78
  213. package/src/service/resolve_environment.ts +0 -15
  214. package/src/service/restricted_mode.spec.ts +0 -299
  215. package/src/service/source_extraction.ts +0 -225
  216. package/src/service/theme_store.ts +0 -199
  217. package/src/storage/DatabaseInterface.ts +0 -198
  218. package/src/storage/StorageManager.ts +0 -113
  219. package/src/storage/duckdb/ConnectionRepository.ts +0 -143
  220. package/src/storage/duckdb/DuckDBConnection.ts +0 -123
  221. package/src/storage/duckdb/DuckDBRepository.ts +0 -208
  222. package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
  223. package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
  224. package/src/storage/duckdb/MaterializationRepository.ts +0 -336
  225. package/src/storage/duckdb/PackageRepository.ts +0 -135
  226. package/src/storage/duckdb/schema.ts +0 -229
  227. package/src/stream_helpers.spec.ts +0 -280
  228. package/src/stream_helpers.ts +0 -162
  229. package/src/telemetry.ts +0 -20
  230. package/src/test_helpers/metrics_harness.ts +0 -166
  231. package/src/theme_key_parity.spec.ts +0 -57
  232. package/src/utils.ts +0 -29
  233. package/tests/fixtures/authorize-compile/model.malloy +0 -9
  234. package/tests/fixtures/authorize-compile/publisher.json +0 -4
  235. package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
  236. package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
  237. package/tests/fixtures/html-pages-test/data.csv +0 -3
  238. package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
  239. package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
  240. package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
  241. package/tests/fixtures/html-pages-test/public/data.json +0 -1
  242. package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
  243. package/tests/fixtures/html-pages-test/public/index.html +0 -9
  244. package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
  245. package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
  246. package/tests/fixtures/html-pages-test/public/slides.html +0 -12
  247. package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
  248. package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
  249. package/tests/fixtures/html-pages-test/publisher.json +0 -5
  250. package/tests/fixtures/html-pages-test/report.malloy +0 -1
  251. package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
  252. package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
  253. package/tests/fixtures/persist-multi-level/publisher.json +0 -5
  254. package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
  255. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
  256. package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
  257. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
  258. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
  259. package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
  260. package/tests/fixtures/persist-test/data/orders.csv +0 -5
  261. package/tests/fixtures/persist-test/persist_test.malloy +0 -11
  262. package/tests/fixtures/persist-test/publisher.json +0 -5
  263. package/tests/fixtures/publisher.config.json +0 -15
  264. package/tests/fixtures/query-givens/data/orders.csv +0 -7
  265. package/tests/fixtures/query-givens/model.malloy +0 -34
  266. package/tests/fixtures/query-givens/publisher.json +0 -5
  267. package/tests/fixtures/xlsx/database.xlsx +0 -0
  268. package/tests/harness/mcp_test_setup.ts +0 -332
  269. package/tests/harness/rest_e2e.ts +0 -68
  270. package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
  271. package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
  272. package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
  273. package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
  274. package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
  275. package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
  276. package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
  277. package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
  278. package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
  279. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
  280. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
  281. package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
  282. package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
  283. package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
  284. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
  285. package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
  286. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
  287. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
  288. package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
  289. package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
  290. package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
  291. package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
  292. package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
  293. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
  294. package/tests/unit/controller/theme.controller.spec.ts +0 -141
  295. package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
  296. package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
  297. package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
  298. package/tests/unit/duckdb/repositories.test.ts +0 -208
  299. package/tests/unit/ducklake/ducklake.test.ts +0 -952
  300. package/tests/unit/service/theme_store.spec.ts +0 -139
  301. package/tsconfig.json +0 -32
@@ -1,2360 +0,0 @@
1
- import { DuckDBConnection } from "@malloydata/db-duckdb";
2
- import {
3
- afterAll,
4
- afterEach,
5
- beforeEach,
6
- describe,
7
- expect,
8
- it,
9
- } from "bun:test";
10
- import fs from "fs/promises";
11
- import os from "os";
12
- import path from "path";
13
- import sinon from "sinon";
14
- import { components } from "../api";
15
- import {
16
- attachDuckLakeReadWrite,
17
- buildProxiedSslQuery,
18
- createEnvironmentConnections,
19
- resolveProxiedTls,
20
- testConnectionConfig,
21
- } from "./connection";
22
- import { assembleEnvironmentConnections } from "./connection_config";
23
- import { UnsupportedCatalogFormatError } from "../errors";
24
- import { EnvironmentStore } from "./environment_store";
25
-
26
- type ApiConnection = components["schemas"]["Connection"];
27
- type AttachedDatabase = components["schemas"]["AttachedDatabase"];
28
-
29
- const hasPostgresCredentials = () =>
30
- !!(
31
- process.env.POSTGRES_TEST_HOST &&
32
- process.env.POSTGRES_TEST_USER &&
33
- process.env.POSTGRES_TEST_PASSWORD
34
- );
35
-
36
- const hasBigQueryCredentials = () =>
37
- !!(
38
- process.env.GOOGLE_APPLICATION_CREDENTIALS &&
39
- process.env.BIGQUERY_TEST_PROJECT_ID
40
- );
41
-
42
- const hasSnowflakeCredentials = () =>
43
- !!(
44
- process.env.SNOWFLAKE_TEST_ACCOUNT &&
45
- process.env.SNOWFLAKE_TEST_USER &&
46
- process.env.SNOWFLAKE_TEST_PASSWORD &&
47
- process.env.SNOWFLAKE_TEST_WAREHOUSE
48
- );
49
-
50
- const hasS3Credentials = () =>
51
- !!(
52
- process.env.S3_TEST_ACCESS_KEY_ID && process.env.S3_TEST_SECRET_ACCESS_KEY
53
- );
54
-
55
- const hasGCSCredentials = () =>
56
- !!(process.env.GCS_TEST_KEY_ID && process.env.GCS_TEST_SECRET);
57
-
58
- const readBigQueryServiceAccountJson = async (): Promise<string> =>
59
- fs.readFile(process.env.GOOGLE_APPLICATION_CREDENTIALS!, "utf-8");
60
-
61
- // `BIGQUERY_PUBLIC_DATA_*` env vars are populated by the
62
- // `Setup BigQuery public-data credentials` step in
63
- // `.github/workflows/connection-integration-tests.yml`. Kept separate
64
- // from the existing `BIGQUERY_TEST_*` vars (which point at the
65
- // org-scoped BQ_PRESTO_TRINO_KEY service account) so the two SAs can
66
- // coexist without one overwriting the other.
67
- const hasPublicDataBigQueryCredentials = () =>
68
- !!(
69
- process.env.BIGQUERY_PUBLIC_DATA_CREDENTIALS &&
70
- process.env.BIGQUERY_PUBLIC_DATA_PROJECT_ID
71
- );
72
-
73
- const readPublicDataBigQueryServiceAccountJson = async (): Promise<string> =>
74
- fs.readFile(process.env.BIGQUERY_PUBLIC_DATA_CREDENTIALS!, "utf-8");
75
-
76
- describe("connection integration tests", () => {
77
- const testEnvironmentPath = path.join(
78
- process.cwd(),
79
- "test-environment-connections",
80
- );
81
- let createdConnections: DuckDBConnection[] = [];
82
-
83
- beforeEach(async () => {
84
- await fs.mkdir(testEnvironmentPath, { recursive: true });
85
- });
86
-
87
- afterEach(async () => {
88
- sinon.restore();
89
-
90
- for (const conn of createdConnections) {
91
- try {
92
- await conn.close();
93
- } catch (error) {
94
- console.warn("Error closing connection:", error);
95
- }
96
- }
97
- createdConnections = [];
98
-
99
- const maxRetries = 5;
100
- const delay = 100;
101
- let lastError: unknown;
102
-
103
- for (let attempt = 0; attempt < maxRetries; attempt++) {
104
- try {
105
- await fs.rm(testEnvironmentPath, { recursive: true, force: true });
106
- return;
107
- } catch (error) {
108
- lastError = error;
109
- const errnoError = error as NodeJS.ErrnoException;
110
- if (errnoError.code !== "EBUSY") throw error;
111
- if (attempt < maxRetries - 1) {
112
- await new Promise((resolve) =>
113
- setTimeout(resolve, delay * Math.pow(2, attempt)),
114
- );
115
- }
116
- }
117
- }
118
-
119
- if ((lastError as NodeJS.ErrnoException).code !== "EBUSY") {
120
- throw lastError;
121
- }
122
- });
123
-
124
- describe("createEnvironmentConnections", () => {
125
- describe("DuckDB with PostgreSQL attachment", () => {
126
- it(
127
- "should create DuckDB connection with attached PostgreSQL database",
128
- async () => {
129
- if (!hasPostgresCredentials()) {
130
- console.log(
131
- "Skipping: PostgreSQL credentials not configured",
132
- );
133
- return;
134
- }
135
-
136
- const postgresAttachment: AttachedDatabase = {
137
- name: "pg_test",
138
- type: "postgres",
139
- postgresConnection: {
140
- host: process.env.POSTGRES_TEST_HOST,
141
- port: parseInt(process.env.POSTGRES_TEST_PORT || "5432"),
142
- userName: process.env.POSTGRES_TEST_USER!,
143
- password: process.env.POSTGRES_TEST_PASSWORD!,
144
- databaseName: process.env.POSTGRES_TEST_DATABASE,
145
- },
146
- };
147
-
148
- const duckdbConnection: ApiConnection = {
149
- name: "duckdb_with_postgres",
150
- type: "duckdb",
151
- duckdbConnection: { attachedDatabases: [postgresAttachment] },
152
- };
153
-
154
- const { malloyConnections, apiConnections } =
155
- await createEnvironmentConnections(
156
- [duckdbConnection],
157
- testEnvironmentPath,
158
- );
159
-
160
- expect(malloyConnections.size).toBe(1);
161
- expect(apiConnections.length).toBe(1);
162
-
163
- const connection = malloyConnections.get(
164
- "duckdb_with_postgres",
165
- ) as DuckDBConnection;
166
- expect(connection).toBeDefined();
167
- createdConnections.push(connection);
168
-
169
- const databases = await connection.runSQL("SHOW DATABASES");
170
- const dbNames = databases.rows.map(
171
- (row) => Object.values(row)[0],
172
- );
173
- expect(dbNames).toContain("pg_test");
174
-
175
- const result = await connection.runSQL(
176
- "SELECT 1 as test_value FROM pg_test.information_schema.tables LIMIT 1",
177
- );
178
- expect(result.rows).toBeDefined();
179
- },
180
- { timeout: 30000 },
181
- );
182
-
183
- it(
184
- "should list schemas from attached PostgreSQL database",
185
- async () => {
186
- if (!hasPostgresCredentials()) {
187
- console.log(
188
- "Skipping: PostgreSQL credentials not configured",
189
- );
190
- return;
191
- }
192
-
193
- const duckdbConnection: ApiConnection = {
194
- name: "duckdb_pg_schemas",
195
- type: "duckdb",
196
- duckdbConnection: {
197
- attachedDatabases: [
198
- {
199
- name: "pg_schemas",
200
- type: "postgres",
201
- postgresConnection: {
202
- host: process.env.POSTGRES_TEST_HOST,
203
- port: parseInt(
204
- process.env.POSTGRES_TEST_PORT || "5432",
205
- ),
206
- userName: process.env.POSTGRES_TEST_USER!,
207
- password: process.env.POSTGRES_TEST_PASSWORD!,
208
- databaseName: process.env.POSTGRES_TEST_DATABASE,
209
- },
210
- },
211
- ],
212
- },
213
- };
214
-
215
- const { malloyConnections } = await createEnvironmentConnections(
216
- [duckdbConnection],
217
- testEnvironmentPath,
218
- );
219
-
220
- const connection = malloyConnections.get(
221
- "duckdb_pg_schemas",
222
- ) as DuckDBConnection;
223
- createdConnections.push(connection);
224
-
225
- // listSchemas equivalent
226
- const result = await connection.runSQL(
227
- "SELECT schema_name FROM pg_schemas.information_schema.schemata ORDER BY schema_name",
228
- );
229
- expect(result.rows.length).toBeGreaterThan(0);
230
- const schemaNames = result.rows.map(
231
- (row) => Object.values(row)[0] as string,
232
- );
233
- expect(schemaNames).toContain("public");
234
- },
235
- { timeout: 30000 },
236
- );
237
-
238
- it(
239
- "should list tables from attached PostgreSQL database",
240
- async () => {
241
- if (!hasPostgresCredentials()) {
242
- console.log(
243
- "Skipping: PostgreSQL credentials not configured",
244
- );
245
- return;
246
- }
247
-
248
- const duckdbConnection: ApiConnection = {
249
- name: "duckdb_pg_tables",
250
- type: "duckdb",
251
- duckdbConnection: {
252
- attachedDatabases: [
253
- {
254
- name: "pg_tables_db",
255
- type: "postgres",
256
- postgresConnection: {
257
- host: process.env.POSTGRES_TEST_HOST,
258
- port: parseInt(
259
- process.env.POSTGRES_TEST_PORT || "5432",
260
- ),
261
- userName: process.env.POSTGRES_TEST_USER!,
262
- password: process.env.POSTGRES_TEST_PASSWORD!,
263
- databaseName: process.env.POSTGRES_TEST_DATABASE,
264
- },
265
- },
266
- ],
267
- },
268
- };
269
-
270
- const { malloyConnections } = await createEnvironmentConnections(
271
- [duckdbConnection],
272
- testEnvironmentPath,
273
- );
274
-
275
- const connection = malloyConnections.get(
276
- "duckdb_pg_tables",
277
- ) as DuckDBConnection;
278
- createdConnections.push(connection);
279
-
280
- // listTables equivalent
281
- const result = await connection.runSQL(`
282
- SELECT table_schema, table_name, table_type
283
- FROM pg_tables_db.information_schema.tables
284
- WHERE table_schema NOT IN ('pg_catalog', 'information_schema')
285
- ORDER BY table_schema, table_name
286
- `);
287
- expect(result.rows).toBeDefined();
288
- },
289
- { timeout: 30000 },
290
- );
291
-
292
- it(
293
- "should get table columns from attached PostgreSQL database",
294
- async () => {
295
- if (!hasPostgresCredentials()) {
296
- console.log(
297
- "Skipping: PostgreSQL credentials not configured",
298
- );
299
- return;
300
- }
301
-
302
- const duckdbConnection: ApiConnection = {
303
- name: "duckdb_pg_cols",
304
- type: "duckdb",
305
- duckdbConnection: {
306
- attachedDatabases: [
307
- {
308
- name: "pg_cols",
309
- type: "postgres",
310
- postgresConnection: {
311
- host: process.env.POSTGRES_TEST_HOST,
312
- port: parseInt(
313
- process.env.POSTGRES_TEST_PORT || "5432",
314
- ),
315
- userName: process.env.POSTGRES_TEST_USER!,
316
- password: process.env.POSTGRES_TEST_PASSWORD!,
317
- databaseName: process.env.POSTGRES_TEST_DATABASE,
318
- },
319
- },
320
- ],
321
- },
322
- };
323
-
324
- const { malloyConnections } = await createEnvironmentConnections(
325
- [duckdbConnection],
326
- testEnvironmentPath,
327
- );
328
-
329
- const connection = malloyConnections.get(
330
- "duckdb_pg_cols",
331
- ) as DuckDBConnection;
332
- createdConnections.push(connection);
333
-
334
- const result = await connection.runSQL(`
335
- SELECT column_name, data_type, is_nullable
336
- FROM pg_cols.information_schema.columns
337
- WHERE table_schema = 'information_schema'
338
- AND table_name = 'tables'
339
- ORDER BY ordinal_position
340
- `);
341
- expect(result.rows.length).toBeGreaterThan(0);
342
- const columnNames = result.rows.map(
343
- (row) => (row as Record<string, unknown>)["column_name"],
344
- );
345
- expect(columnNames).toContain("table_name");
346
- expect(columnNames).toContain("table_schema");
347
- },
348
- { timeout: 30000 },
349
- );
350
-
351
- it(
352
- "should handle PostgreSQL connection string format",
353
- async () => {
354
- if (!hasPostgresCredentials()) {
355
- console.log(
356
- "Skipping: PostgreSQL credentials not configured",
357
- );
358
- return;
359
- }
360
-
361
- const connectionString = `host=${process.env.POSTGRES_TEST_HOST} port=${process.env.POSTGRES_TEST_PORT || "5432"} dbname=${process.env.POSTGRES_TEST_DATABASE} user=${process.env.POSTGRES_TEST_USER} password=${process.env.POSTGRES_TEST_PASSWORD}`;
362
-
363
- const duckdbConnection: ApiConnection = {
364
- name: "duckdb_pg_string",
365
- type: "duckdb",
366
- duckdbConnection: {
367
- attachedDatabases: [
368
- {
369
- name: "pg_conn_string",
370
- type: "postgres",
371
- postgresConnection: { connectionString },
372
- },
373
- ],
374
- },
375
- };
376
-
377
- const { malloyConnections } = await createEnvironmentConnections(
378
- [duckdbConnection],
379
- testEnvironmentPath,
380
- );
381
-
382
- const connection = malloyConnections.get(
383
- "duckdb_pg_string",
384
- ) as DuckDBConnection;
385
- createdConnections.push(connection);
386
-
387
- const databases = await connection.runSQL("SHOW DATABASES");
388
- const dbNames = databases.rows.map(
389
- (row) => Object.values(row)[0],
390
- );
391
- expect(dbNames).toContain("pg_conn_string");
392
- },
393
- { timeout: 30000 },
394
- );
395
- });
396
-
397
- describe("DuckDB with BigQuery attachment", () => {
398
- it(
399
- "should create DuckDB connection with attached BigQuery database",
400
- async () => {
401
- if (!hasBigQueryCredentials()) {
402
- console.log("Skipping: BigQuery credentials not configured");
403
- return;
404
- }
405
-
406
- const serviceAccountJson =
407
- await readBigQueryServiceAccountJson();
408
-
409
- const duckdbConnection: ApiConnection = {
410
- name: "duckdb_with_bigquery",
411
- type: "duckdb",
412
- duckdbConnection: {
413
- attachedDatabases: [
414
- {
415
- name: "bq_test",
416
- type: "bigquery",
417
- bigqueryConnection: {
418
- defaultProjectId:
419
- process.env.BIGQUERY_TEST_PROJECT_ID!,
420
- serviceAccountKeyJson: serviceAccountJson,
421
- },
422
- },
423
- ],
424
- },
425
- };
426
-
427
- const { malloyConnections } = await createEnvironmentConnections(
428
- [duckdbConnection],
429
- testEnvironmentPath,
430
- );
431
-
432
- const connection = malloyConnections.get(
433
- "duckdb_with_bigquery",
434
- ) as DuckDBConnection;
435
- expect(connection).toBeDefined();
436
- createdConnections.push(connection);
437
-
438
- const databases = await connection.runSQL("SHOW DATABASES");
439
- const dbNames = databases.rows.map(
440
- (row) => Object.values(row)[0],
441
- );
442
- expect(dbNames).toContain("bq_test");
443
- },
444
- { timeout: 60000 },
445
- );
446
-
447
- it(
448
- "should list datasets (schemas) from attached BigQuery database",
449
- async () => {
450
- if (
451
- !hasBigQueryCredentials() ||
452
- !process.env.BIGQUERY_TEST_DATASET
453
- ) {
454
- console.log(
455
- "Skipping: GOOGLE_APPLICATION_CREDENTIALS, BIGQUERY_TEST_PROJECT_ID, or BIGQUERY_TEST_DATASET not configured",
456
- );
457
- return;
458
- }
459
-
460
- const serviceAccountJson =
461
- await readBigQueryServiceAccountJson();
462
-
463
- const duckdbConnection: ApiConnection = {
464
- name: "duckdb_bq_schemas",
465
- type: "duckdb",
466
- duckdbConnection: {
467
- attachedDatabases: [
468
- {
469
- name: "bq_schemas",
470
- type: "bigquery",
471
- bigqueryConnection: {
472
- defaultProjectId:
473
- process.env.BIGQUERY_TEST_PROJECT_ID!,
474
- serviceAccountKeyJson: serviceAccountJson,
475
- },
476
- },
477
- ],
478
- },
479
- };
480
-
481
- const { malloyConnections } = await createEnvironmentConnections(
482
- [duckdbConnection],
483
- testEnvironmentPath,
484
- );
485
-
486
- const connection = malloyConnections.get(
487
- "duckdb_bq_schemas",
488
- ) as DuckDBConnection;
489
- createdConnections.push(connection);
490
-
491
- const result = await connection.runSQL(
492
- "SELECT schema_name FROM bq_schemas.information_schema.schemata",
493
- );
494
- expect(result.rows.length).toBeGreaterThan(0);
495
- const datasetNames = result.rows.map(
496
- (row) => Object.values(row)[0] as string,
497
- );
498
- expect(datasetNames).toContain(
499
- process.env.BIGQUERY_TEST_DATASET!,
500
- );
501
- },
502
- { timeout: 60000 },
503
- );
504
-
505
- it(
506
- "should list tables from attached BigQuery dataset",
507
- async () => {
508
- if (
509
- !hasBigQueryCredentials() ||
510
- !process.env.BIGQUERY_TEST_DATASET
511
- ) {
512
- console.log(
513
- "Skipping: GOOGLE_APPLICATION_CREDENTIALS, BIGQUERY_TEST_PROJECT_ID, or BIGQUERY_TEST_DATASET not configured",
514
- );
515
- return;
516
- }
517
-
518
- const serviceAccountJson =
519
- await readBigQueryServiceAccountJson();
520
-
521
- const duckdbConnection: ApiConnection = {
522
- name: "duckdb_bq_tables",
523
- type: "duckdb",
524
- duckdbConnection: {
525
- attachedDatabases: [
526
- {
527
- name: "bq_tables",
528
- type: "bigquery",
529
- bigqueryConnection: {
530
- defaultProjectId:
531
- process.env.BIGQUERY_TEST_PROJECT_ID!,
532
- serviceAccountKeyJson: serviceAccountJson,
533
- },
534
- },
535
- ],
536
- },
537
- };
538
-
539
- const { malloyConnections } = await createEnvironmentConnections(
540
- [duckdbConnection],
541
- testEnvironmentPath,
542
- );
543
-
544
- const connection = malloyConnections.get(
545
- "duckdb_bq_tables",
546
- ) as DuckDBConnection;
547
- createdConnections.push(connection);
548
-
549
- const result = await connection.runSQL(`
550
- SELECT table_name, table_type
551
- FROM bq_tables.${process.env.BIGQUERY_TEST_DATASET!}.INFORMATION_SCHEMA.TABLES
552
- ORDER BY table_name
553
- `);
554
- expect(result.rows.length).toBeGreaterThan(0);
555
- },
556
- { timeout: 60000 },
557
- );
558
-
559
- it(
560
- "should get table columns from attached BigQuery table",
561
- async () => {
562
- if (
563
- !hasBigQueryCredentials() ||
564
- !process.env.BIGQUERY_TEST_TABLE
565
- ) {
566
- console.log(
567
- "Skipping: GOOGLE_APPLICATION_CREDENTIALS, BIGQUERY_TEST_PROJECT_ID, or BIGQUERY_TEST_TABLE not configured",
568
- );
569
- return;
570
- }
571
-
572
- const serviceAccountJson =
573
- await readBigQueryServiceAccountJson();
574
- const [dataset, table] =
575
- process.env.BIGQUERY_TEST_TABLE!.split(".");
576
-
577
- const duckdbConnection: ApiConnection = {
578
- name: "duckdb_bq_cols",
579
- type: "duckdb",
580
- duckdbConnection: {
581
- attachedDatabases: [
582
- {
583
- name: "bq_cols",
584
- type: "bigquery",
585
- bigqueryConnection: {
586
- defaultProjectId:
587
- process.env.BIGQUERY_TEST_PROJECT_ID!,
588
- serviceAccountKeyJson: serviceAccountJson,
589
- },
590
- },
591
- ],
592
- },
593
- };
594
-
595
- const { malloyConnections } = await createEnvironmentConnections(
596
- [duckdbConnection],
597
- testEnvironmentPath,
598
- );
599
-
600
- const connection = malloyConnections.get(
601
- "duckdb_bq_cols",
602
- ) as DuckDBConnection;
603
- createdConnections.push(connection);
604
-
605
- // getTable equivalent — fetch column metadata
606
- const result = await connection.runSQL(`
607
- SELECT column_name, data_type, is_nullable
608
- FROM bq_cols.${dataset}.INFORMATION_SCHEMA.COLUMNS
609
- WHERE table_name = '${table}'
610
- ORDER BY ordinal_position
611
- `);
612
- expect(result.rows.length).toBeGreaterThan(0);
613
- },
614
- { timeout: 60000 },
615
- );
616
-
617
- it(
618
- "should query data from attached BigQuery table",
619
- async () => {
620
- if (
621
- !hasBigQueryCredentials() ||
622
- !process.env.BIGQUERY_TEST_TABLE
623
- ) {
624
- console.log(
625
- "Skipping: GOOGLE_APPLICATION_CREDENTIALS, BIGQUERY_TEST_PROJECT_ID, or BIGQUERY_TEST_TABLE not configured",
626
- );
627
- return;
628
- }
629
-
630
- const serviceAccountJson =
631
- await readBigQueryServiceAccountJson();
632
-
633
- const duckdbConnection: ApiConnection = {
634
- name: "duckdb_bq_query",
635
- type: "duckdb",
636
- duckdbConnection: {
637
- attachedDatabases: [
638
- {
639
- name: "bq_query",
640
- type: "bigquery",
641
- bigqueryConnection: {
642
- defaultProjectId:
643
- process.env.BIGQUERY_TEST_PROJECT_ID!,
644
- serviceAccountKeyJson: serviceAccountJson,
645
- },
646
- },
647
- ],
648
- },
649
- };
650
-
651
- const { malloyConnections } = await createEnvironmentConnections(
652
- [duckdbConnection],
653
- testEnvironmentPath,
654
- );
655
-
656
- const connection = malloyConnections.get(
657
- "duckdb_bq_query",
658
- ) as DuckDBConnection;
659
- createdConnections.push(connection);
660
-
661
- const result = await connection.runSQL(
662
- `SELECT * FROM bq_query.${process.env.BIGQUERY_TEST_TABLE!} LIMIT 1`,
663
- );
664
- expect(result.rows).toBeDefined();
665
- expect(result.rows.length).toBeGreaterThan(0);
666
- },
667
- { timeout: 60000 },
668
- );
669
-
670
- it(
671
- "should validate BigQuery service account key format",
672
- async () => {
673
- await expect(
674
- createEnvironmentConnections(
675
- [
676
- {
677
- name: "duckdb_bq_invalid",
678
- type: "duckdb",
679
- duckdbConnection: {
680
- attachedDatabases: [
681
- {
682
- name: "bq_invalid",
683
- type: "bigquery",
684
- bigqueryConnection: {
685
- defaultProjectId: "test-environment",
686
- serviceAccountKeyJson: JSON.stringify({
687
- invalid: "key",
688
- }),
689
- },
690
- },
691
- ],
692
- },
693
- },
694
- ],
695
- testEnvironmentPath,
696
- ),
697
- ).rejects.toThrow(/Invalid service account key/);
698
- },
699
- { timeout: 30000 },
700
- );
701
- });
702
-
703
- describe("BigQuery direct connection (public-data)", () => {
704
- // Single end-to-end check that a real `bigquery`-typed
705
- // connection authenticated by the BIGQUERY_PUBLIC_DATA_SA
706
- // service account can actually round-trip a query against
707
- // bigquery-public-data. Skips locally if the creds aren't
708
- // set; runs in CI under .github/workflows/connection-integration-tests.yml.
709
- //
710
- // Picked `samples.shakespeare` because (a) it's tiny (~6 MB
711
- // / ~165k rows) so the BigQuery byte-scanned cost is well
712
- // under the 1 TB/month free tier even across many PR runs,
713
- // (b) it's a long-stable public table so the assertion stays
714
- // valid across years of reruns. If this assertion ever fails
715
- // the cause is almost certainly auth, not data drift.
716
- it(
717
- "should query bigquery-public-data via real BigQuery connection",
718
- async () => {
719
- if (!hasPublicDataBigQueryCredentials()) {
720
- console.log(
721
- "Skipping: BIGQUERY_PUBLIC_DATA_CREDENTIALS or BIGQUERY_PUBLIC_DATA_PROJECT_ID not configured",
722
- );
723
- return;
724
- }
725
-
726
- const serviceAccountJson =
727
- await readPublicDataBigQueryServiceAccountJson();
728
-
729
- const bqConnection: ApiConnection = {
730
- name: "bq_public_data",
731
- type: "bigquery",
732
- bigqueryConnection: {
733
- // Billing/auth project (the SA's own project_id);
734
- // the queried tables live in bigquery-public-data
735
- // and are referenced explicitly in the SQL below.
736
- defaultProjectId:
737
- process.env.BIGQUERY_PUBLIC_DATA_PROJECT_ID!,
738
- serviceAccountKeyJson: serviceAccountJson,
739
- },
740
- };
741
-
742
- const { malloyConnections } = await createEnvironmentConnections(
743
- [bqConnection],
744
- testEnvironmentPath,
745
- );
746
-
747
- const connection = malloyConnections.get("bq_public_data");
748
- expect(connection).toBeDefined();
749
-
750
- try {
751
- const result = await connection!.runSQL(
752
- "SELECT COUNT(*) AS row_count FROM `bigquery-public-data.samples.shakespeare`",
753
- );
754
- expect(result.rows.length).toBe(1);
755
- // Shakespeare has ~165k rows; bound on both sides so
756
- // we catch both "auth succeeded but query returned
757
- // nothing" and "got a confusingly large value" failure
758
- // modes, while staying tolerant of any minor row-count
759
- // jitter Google might introduce.
760
- const row = result.rows[0] as Record<string, unknown>;
761
- const rowCount = Number(row.row_count);
762
- expect(rowCount).toBeGreaterThan(100_000);
763
- expect(rowCount).toBeLessThan(200_000);
764
- } finally {
765
- // BigQuery driver holds an HTTP/2 client + auth refresh
766
- // state; close it explicitly so we don't leak across
767
- // the rest of the test run. (createdConnections is
768
- // typed for DuckDBConnection, so we can't use the
769
- // shared cleanup array here.)
770
- await connection?.close();
771
- }
772
- },
773
- { timeout: 60000 },
774
- );
775
- });
776
-
777
- describe("BigQuery package end-to-end (bq-hackernews)", () => {
778
- // Step 2 of Sagar's bun-setup-fixes ask: not just "does the
779
- // BigQuery driver round-trip a SQL query" (the previous
780
- // describe block covers that), but "does the publisher's
781
- // package-loading path successfully use the BigQuery
782
- // connection on behalf of a Malloy package."
783
- //
784
- // Mechanism: stand up a real EnvironmentStore against a
785
- // temp publisher.config.json that declares both the BQ
786
- // connection (with the BIGQUERY_PUBLIC_DATA_SA injected via
787
- // serviceAccountKeyJson) AND the bq-hackernews package
788
- // (loaded from credibledata/malloy-samples on GitHub).
789
- // EnvironmentStore.initialize then: clones the malloy-samples
790
- // repo, extracts the bigquery-hackernews subdirectory, parses
791
- // the package's Malloy models, and — crucially — introspects
792
- // their BigQuery table schemas during model compilation. A
793
- // successful Package.listModels() call therefore proves the
794
- // entire package-uses-connection path, not just the bare
795
- // driver auth.
796
- //
797
- // Cost: ~50 MB git clone (one time per test run since
798
- // publisher_data lives under testEnvironmentPath which the
799
- // afterEach cleans up) + ~6 MB BQ schema scan. Both well
800
- // under any meaningful free-tier limit. Test budget: 3 min
801
- // (clone is ~30-60s on GH runners; compile is ~10s).
802
- it(
803
- "should load bq-hackernews package and compile its models via real BQ",
804
- async () => {
805
- if (!hasPublicDataBigQueryCredentials()) {
806
- console.log(
807
- "Skipping: BIGQUERY_PUBLIC_DATA_CREDENTIALS or BIGQUERY_PUBLIC_DATA_PROJECT_ID not configured",
808
- );
809
- return;
810
- }
811
-
812
- const serviceAccountJson =
813
- await readPublicDataBigQueryServiceAccountJson();
814
-
815
- // Each test gets its own serverRoot so publisher_data
816
- // doesn't leak across tests; afterEach removes
817
- // testEnvironmentPath which carries the whole tree
818
- // away.
819
- const tempServerRoot = path.join(
820
- testEnvironmentPath,
821
- "bq-hackernews-pkg-test",
822
- );
823
- await fs.mkdir(tempServerRoot, { recursive: true });
824
-
825
- const config = {
826
- frozenConfig: false,
827
- environments: [
828
- {
829
- name: "malloy-samples",
830
- packages: [
831
- {
832
- name: "bigquery-hackernews",
833
- location:
834
- "https://github.com/credibledata/malloy-samples/tree/main/bigquery-hackernews",
835
- },
836
- ],
837
- connections: [
838
- {
839
- name: "bigquery",
840
- type: "bigquery",
841
- bigqueryConnection: {
842
- defaultProjectId:
843
- process.env
844
- .BIGQUERY_PUBLIC_DATA_PROJECT_ID!,
845
- serviceAccountKeyJson: serviceAccountJson,
846
- },
847
- },
848
- ],
849
- },
850
- ],
851
- };
852
- await fs.writeFile(
853
- path.join(tempServerRoot, "publisher.config.json"),
854
- JSON.stringify(config),
855
- );
856
-
857
- // Force the load-from-config init path. Without this,
858
- // EnvironmentStore.initialize() takes the load-from-DB
859
- // branch and only falls back to config when the DB is
860
- // empty (which it is here, by construction, but relying
861
- // on that is brittle if the test pattern is reused).
862
- //
863
- // SAFETY PRECONDITION: this flag is the SAME one users
864
- // opt into via `--init` / `start:init` to wipe persisted
865
- // storage and re-initialize from config (see CLAUDE.md
866
- // and `environment_store.ts:158`). It is destructive on
867
- // a real serverRoot. We only set it here because
868
- // `tempServerRoot` is a freshly-created empty directory
869
- // (path.join(testEnvironmentPath, "bq-hackernews-pkg-test")
870
- // mkdir'd ~20 lines above), so there is nothing to wipe.
871
- // DO NOT copy this pattern into a test that points
872
- // EnvironmentStore at a non-empty or shared serverRoot
873
- // — it will delete state.
874
- const previousInitializeStorage = process.env.INITIALIZE_STORAGE;
875
- process.env.INITIALIZE_STORAGE = "true";
876
-
877
- try {
878
- const envStore = new EnvironmentStore(tempServerRoot);
879
- await envStore.finishedInitialization;
880
-
881
- // initialize() swallows top-level errors and just calls
882
- // markNotReady(), so finishedInitialization always resolves
883
- // regardless of success; operationalState is what says it
884
- // succeeded. It does NOT imply every configured environment
885
- // and package loaded: a load failure is skipped and still
886
- // reports "serving", which is why loadErrors exists. Assert
887
- // on both, and on the package list below.
888
- const status = await envStore.getStatus();
889
- expect(status.operationalState).toBe("serving");
890
- expect(status.loadErrors).toBeUndefined();
891
-
892
- const env = await envStore.getEnvironment("malloy-samples");
893
- const apiPackages = await env.listPackages();
894
- expect(apiPackages.map((p) => p.name)).toContain(
895
- "bigquery-hackernews",
896
- );
897
-
898
- const apiConnections = env.listApiConnections();
899
- expect(apiConnections.map((c) => c.name)).toContain(
900
- "bigquery",
901
- );
902
-
903
- // The actual integration assertion. Package.listModels()
904
- // collects compile errors per-model and returns ALL
905
- // models in the result (with `error` populated on
906
- // failures) — so models.length>0 alone would pass even
907
- // if every BQ schema introspection failed. Filter for
908
- // models that compiled cleanly: at least one is the
909
- // real proof that BQ-on-behalf-of-a-package works.
910
- const pkg = await env.getPackage("bigquery-hackernews");
911
- const models = await pkg.listModels();
912
- const okModels = models.filter((m) => !m.error);
913
- if (okModels.length === 0) {
914
- console.error(
915
- "All bq-hackernews model compilations failed:",
916
- models.map((m) => `${m.path}: ${m.error}`),
917
- );
918
- }
919
- expect(okModels.length).toBeGreaterThan(0);
920
- } finally {
921
- if (previousInitializeStorage === undefined) {
922
- delete process.env.INITIALIZE_STORAGE;
923
- } else {
924
- process.env.INITIALIZE_STORAGE = previousInitializeStorage;
925
- }
926
- }
927
- },
928
- { timeout: 180000 },
929
- );
930
- });
931
-
932
- describe("DuckDB with Snowflake attachment", () => {
933
- it(
934
- "should create DuckDB connection with attached Snowflake database",
935
- async () => {
936
- if (!hasSnowflakeCredentials()) {
937
- console.log("Skipping: Snowflake credentials not configured");
938
- return;
939
- }
940
-
941
- const duckdbConnection: ApiConnection = {
942
- name: "duckdb_with_snowflake",
943
- type: "duckdb",
944
- duckdbConnection: {
945
- attachedDatabases: [
946
- {
947
- name: "sf_test",
948
- type: "snowflake",
949
- snowflakeConnection: {
950
- account: process.env.SNOWFLAKE_TEST_ACCOUNT!,
951
- username: process.env.SNOWFLAKE_TEST_USER!,
952
- password: process.env.SNOWFLAKE_TEST_PASSWORD!,
953
- warehouse: process.env.SNOWFLAKE_TEST_WAREHOUSE!,
954
- database: process.env.SNOWFLAKE_TEST_DATABASE,
955
- schema: process.env.SNOWFLAKE_TEST_SCHEMA,
956
- },
957
- },
958
- ],
959
- },
960
- };
961
-
962
- const { malloyConnections } = await createEnvironmentConnections(
963
- [duckdbConnection],
964
- testEnvironmentPath,
965
- );
966
-
967
- const connection = malloyConnections.get(
968
- "duckdb_with_snowflake",
969
- ) as DuckDBConnection;
970
- expect(connection).toBeDefined();
971
- createdConnections.push(connection);
972
-
973
- const databases = await connection.runSQL("SHOW DATABASES");
974
- const dbNames = databases.rows.map(
975
- (row) => Object.values(row)[0],
976
- );
977
- expect(dbNames).toContain("sf_test");
978
-
979
- const result = await connection.runSQL(
980
- "SELECT * FROM snowflake_query('SELECT 1 as test', 'sf_test_secret')",
981
- );
982
- expect(result.rows).toBeDefined();
983
- },
984
- { timeout: 30000 },
985
- );
986
-
987
- it("should validate required Snowflake fields", async () => {
988
- await expect(
989
- createEnvironmentConnections(
990
- [
991
- {
992
- name: "duckdb_sf_incomplete",
993
- type: "duckdb",
994
- duckdbConnection: {
995
- attachedDatabases: [
996
- {
997
- name: "sf_incomplete",
998
- type: "snowflake",
999
- snowflakeConnection: {
1000
- account: "test-account",
1001
- username: "test-user",
1002
- },
1003
- },
1004
- ],
1005
- },
1006
- },
1007
- ],
1008
- testEnvironmentPath,
1009
- ),
1010
- ).rejects.toThrow(/required/);
1011
- });
1012
- });
1013
-
1014
- describe("DuckDB with S3 attachment", () => {
1015
- it(
1016
- "should create DuckDB connection with S3 configuration",
1017
- async () => {
1018
- if (!hasS3Credentials()) {
1019
- console.log("Skipping: S3 credentials not configured");
1020
- return;
1021
- }
1022
-
1023
- const duckdbConnection: ApiConnection = {
1024
- name: "duckdb_with_s3",
1025
- type: "duckdb",
1026
- duckdbConnection: {
1027
- attachedDatabases: [
1028
- {
1029
- name: "s3_test",
1030
- type: "s3",
1031
- s3Connection: {
1032
- accessKeyId: process.env.S3_TEST_ACCESS_KEY_ID!,
1033
- secretAccessKey:
1034
- process.env.S3_TEST_SECRET_ACCESS_KEY!,
1035
- region: process.env.S3_TEST_REGION || "us-east-1",
1036
- },
1037
- },
1038
- ],
1039
- },
1040
- };
1041
-
1042
- const { malloyConnections } = await createEnvironmentConnections(
1043
- [duckdbConnection],
1044
- testEnvironmentPath,
1045
- );
1046
-
1047
- const connection = malloyConnections.get(
1048
- "duckdb_with_s3",
1049
- ) as DuckDBConnection;
1050
- expect(connection).toBeDefined();
1051
- createdConnections.push(connection);
1052
-
1053
- const secrets = await connection.runSQL(
1054
- "SELECT name FROM duckdb_secrets()",
1055
- );
1056
- const secretNames = secrets.rows.map(
1057
- (row) => Object.values(row)[0],
1058
- );
1059
- expect(
1060
- secretNames.some((name) => String(name).includes("s3")),
1061
- ).toBe(true);
1062
- },
1063
- { timeout: 30000 },
1064
- );
1065
-
1066
- it(
1067
- "should handle S3 with custom endpoint",
1068
- async () => {
1069
- if (!hasS3Credentials()) {
1070
- console.log("Skipping: S3 credentials not configured");
1071
- return;
1072
- }
1073
-
1074
- const duckdbConnection: ApiConnection = {
1075
- name: "duckdb_s3_custom",
1076
- type: "duckdb",
1077
- duckdbConnection: {
1078
- attachedDatabases: [
1079
- {
1080
- name: "s3_custom",
1081
- type: "s3",
1082
- s3Connection: {
1083
- accessKeyId: process.env.S3_TEST_ACCESS_KEY_ID!,
1084
- secretAccessKey:
1085
- process.env.S3_TEST_SECRET_ACCESS_KEY!,
1086
- region: "us-east-1",
1087
- endpoint: "https://s3.custom-endpoint.com",
1088
- },
1089
- },
1090
- ],
1091
- },
1092
- };
1093
-
1094
- const { malloyConnections } = await createEnvironmentConnections(
1095
- [duckdbConnection],
1096
- testEnvironmentPath,
1097
- );
1098
-
1099
- const connection = malloyConnections.get(
1100
- "duckdb_s3_custom",
1101
- ) as DuckDBConnection;
1102
- createdConnections.push(connection);
1103
- expect(connection).toBeDefined();
1104
- },
1105
- { timeout: 30000 },
1106
- );
1107
- });
1108
-
1109
- describe("DuckDB with GCS attachment", () => {
1110
- it(
1111
- "should create DuckDB connection with GCS configuration",
1112
- async () => {
1113
- if (!hasGCSCredentials()) {
1114
- console.log("Skipping: GCS credentials not configured");
1115
- return;
1116
- }
1117
-
1118
- const duckdbConnection: ApiConnection = {
1119
- name: "duckdb_with_gcs",
1120
- type: "duckdb",
1121
- duckdbConnection: {
1122
- attachedDatabases: [
1123
- {
1124
- name: "gcs_test",
1125
- type: "gcs",
1126
- gcsConnection: {
1127
- keyId: process.env.GCS_TEST_KEY_ID!,
1128
- secret: process.env.GCS_TEST_SECRET!,
1129
- },
1130
- },
1131
- ],
1132
- },
1133
- };
1134
-
1135
- const { malloyConnections } = await createEnvironmentConnections(
1136
- [duckdbConnection],
1137
- testEnvironmentPath,
1138
- );
1139
-
1140
- const connection = malloyConnections.get(
1141
- "duckdb_with_gcs",
1142
- ) as DuckDBConnection;
1143
- expect(connection).toBeDefined();
1144
- createdConnections.push(connection);
1145
-
1146
- const secrets = await connection.runSQL(
1147
- "SELECT name FROM duckdb_secrets()",
1148
- );
1149
- const secretNames = secrets.rows.map(
1150
- (row) => Object.values(row)[0],
1151
- );
1152
- expect(
1153
- secretNames.some((name) => String(name).includes("gcs")),
1154
- ).toBe(true);
1155
- },
1156
- { timeout: 30000 },
1157
- );
1158
- });
1159
-
1160
- describe("DuckDB with multiple attachments", () => {
1161
- it(
1162
- "should attach multiple databases to single DuckDB connection",
1163
- async () => {
1164
- const attachments: AttachedDatabase[] = [];
1165
-
1166
- if (hasPostgresCredentials()) {
1167
- attachments.push({
1168
- name: "pg_multi",
1169
- type: "postgres",
1170
- postgresConnection: {
1171
- host: process.env.POSTGRES_TEST_HOST,
1172
- port: parseInt(
1173
- process.env.POSTGRES_TEST_PORT || "5432",
1174
- ),
1175
- userName: process.env.POSTGRES_TEST_USER!,
1176
- password: process.env.POSTGRES_TEST_PASSWORD!,
1177
- databaseName: process.env.POSTGRES_TEST_DATABASE,
1178
- },
1179
- });
1180
- }
1181
-
1182
- if (hasBigQueryCredentials()) {
1183
- const serviceAccountJson =
1184
- await readBigQueryServiceAccountJson();
1185
- attachments.push({
1186
- name: "bq_multi",
1187
- type: "bigquery",
1188
- bigqueryConnection: {
1189
- defaultProjectId: process.env.BIGQUERY_TEST_PROJECT_ID!,
1190
- serviceAccountKeyJson: serviceAccountJson,
1191
- },
1192
- });
1193
- }
1194
-
1195
- if (hasS3Credentials()) {
1196
- attachments.push({
1197
- name: "s3_multi",
1198
- type: "s3",
1199
- s3Connection: {
1200
- accessKeyId: process.env.S3_TEST_ACCESS_KEY_ID!,
1201
- secretAccessKey: process.env.S3_TEST_SECRET_ACCESS_KEY!,
1202
- region: process.env.S3_TEST_REGION || "us-east-1",
1203
- },
1204
- });
1205
- }
1206
-
1207
- if (attachments.length === 0) {
1208
- console.log("Skipping: No database credentials configured");
1209
- return;
1210
- }
1211
-
1212
- const { malloyConnections } = await createEnvironmentConnections(
1213
- [
1214
- {
1215
- name: "duckdb_multi",
1216
- type: "duckdb",
1217
- duckdbConnection: { attachedDatabases: attachments },
1218
- },
1219
- ],
1220
- testEnvironmentPath,
1221
- );
1222
-
1223
- const connection = malloyConnections.get(
1224
- "duckdb_multi",
1225
- ) as DuckDBConnection;
1226
- expect(connection).toBeDefined();
1227
- createdConnections.push(connection);
1228
-
1229
- const databases = await connection.runSQL("SHOW DATABASES");
1230
- const dbNames = databases.rows.map(
1231
- (row) => Object.values(row)[0],
1232
- );
1233
- attachments.forEach((attachment) => {
1234
- expect(dbNames).toContain(attachment.name!);
1235
- });
1236
- },
1237
- { timeout: 60000 },
1238
- );
1239
- });
1240
-
1241
- describe("error handling", () => {
1242
- describe("DuckLake connection type", () => {
1243
- it(
1244
- "should create DuckLake connection",
1245
- async () => {
1246
- if (!hasPostgresCredentials() || !hasS3Credentials()) {
1247
- console.log(
1248
- "Skipping: PostgreSQL and S3 credentials not configured",
1249
- );
1250
- return;
1251
- }
1252
-
1253
- const { malloyConnections } =
1254
- await createEnvironmentConnections(
1255
- [
1256
- {
1257
- name: "ducklake_test",
1258
- type: "ducklake",
1259
- ducklakeConnection: {
1260
- catalog: {
1261
- postgresConnection: {
1262
- host: process.env.POSTGRES_TEST_HOST,
1263
- port: parseInt(
1264
- process.env.POSTGRES_TEST_PORT ||
1265
- "5432",
1266
- ),
1267
- userName:
1268
- process.env.POSTGRES_TEST_USER!,
1269
- password:
1270
- process.env.POSTGRES_TEST_PASSWORD!,
1271
- databaseName:
1272
- process.env.POSTGRES_TEST_DATABASE,
1273
- },
1274
- },
1275
- storage: {
1276
- bucketUrl:
1277
- process.env.S3_TEST_BUCKET_URL ||
1278
- "s3://test-bucket",
1279
- s3Connection: {
1280
- accessKeyId:
1281
- process.env.S3_TEST_ACCESS_KEY_ID!,
1282
- secretAccessKey:
1283
- process.env
1284
- .S3_TEST_SECRET_ACCESS_KEY!,
1285
- },
1286
- },
1287
- },
1288
- },
1289
- ],
1290
- testEnvironmentPath,
1291
- );
1292
-
1293
- const connection = malloyConnections.get(
1294
- "ducklake_test",
1295
- ) as DuckDBConnection;
1296
- createdConnections.push(connection);
1297
- expect(connection).toBeDefined();
1298
-
1299
- // Verify DuckLake database is attached
1300
- const databases = await connection.runSQL("SHOW DATABASES");
1301
- const dbNames = databases.rows.map(
1302
- (row) => Object.values(row)[0],
1303
- );
1304
- expect(dbNames).toContain("ducklake_test");
1305
- },
1306
- { timeout: 30000 },
1307
- );
1308
-
1309
- it("should throw error if DuckLake catalog connection is missing", async () => {
1310
- await expect(
1311
- createEnvironmentConnections(
1312
- [
1313
- {
1314
- name: "ducklake_no_catalog",
1315
- type: "ducklake",
1316
- ducklakeConnection: {
1317
- storage: {
1318
- bucketUrl: "s3://test-bucket",
1319
- s3Connection: {
1320
- accessKeyId: "test",
1321
- secretAccessKey: "test",
1322
- },
1323
- },
1324
- },
1325
- } as ApiConnection,
1326
- ],
1327
- testEnvironmentPath,
1328
- ),
1329
- ).rejects.toThrow(
1330
- /PostgreSQL connection configuration is required/,
1331
- );
1332
- });
1333
-
1334
- it("should throw error if DuckLake connection config is missing", async () => {
1335
- await expect(
1336
- createEnvironmentConnections(
1337
- [
1338
- {
1339
- name: "ducklake_missing_config",
1340
- type: "ducklake",
1341
- },
1342
- ],
1343
- testEnvironmentPath,
1344
- ),
1345
- ).rejects.toThrow(
1346
- /DuckLake connection configuration is missing/,
1347
- );
1348
- });
1349
-
1350
- // ── catalog.metadataSchema ──────────────────────────────────────
1351
- //
1352
- // Several independent DuckLake catalogs may share ONE catalog
1353
- // database when each attaches its own metadata schema. These use a
1354
- // LOCAL directory as DATA_PATH rather than S3/GCS, so they need only
1355
- // the Postgres service and are not skipped for want of object-store
1356
- // credentials.
1357
- describe("catalog.metadataSchema", () => {
1358
- const pgCatalog = () => ({
1359
- host: process.env.POSTGRES_TEST_HOST,
1360
- port: parseInt(process.env.POSTGRES_TEST_PORT || "5432"),
1361
- userName: process.env.POSTGRES_TEST_USER!,
1362
- password: process.env.POSTGRES_TEST_PASSWORD!,
1363
- databaseName: process.env.POSTGRES_TEST_DATABASE,
1364
- });
1365
-
1366
- const pgConnString = () => {
1367
- const pg = pgCatalog();
1368
- return (
1369
- `host=${pg.host} port=${pg.port} user=${pg.userName} ` +
1370
- `password=${pg.password}` +
1371
- (pg.databaseName ? ` dbname=${pg.databaseName}` : "")
1372
- );
1373
- };
1374
-
1375
- // A unique schema per test AND per run: the catalog database is
1376
- // shared across the specs in this file, and (locally) across runs,
1377
- // so a fixed name would let one test observe another's metadata.
1378
- const runId = Math.random().toString(36).slice(2, 8);
1379
- let schemaSeq = 0;
1380
- const createdSchemas: string[] = [];
1381
- const uniqueSchema = (label: string) => {
1382
- const schema = `dl_meta_${label}_${runId}_${++schemaSeq}`;
1383
- createdSchemas.push(schema);
1384
- return schema;
1385
- };
1386
-
1387
- // Leave the catalog database as we found it. These schemas are
1388
- // created by DuckLake inside a database shared with the other
1389
- // specs in this file, and enough of them makes a sibling test's
1390
- // schema enumeration miss `public`.
1391
- afterAll(async () => {
1392
- if (
1393
- !hasPostgresCredentials() ||
1394
- createdSchemas.length === 0
1395
- ) {
1396
- return;
1397
- }
1398
- const workDir = await fs.mkdtemp(
1399
- path.join(os.tmpdir(), "dl-cleanup-"),
1400
- );
1401
- const conn = new DuckDBConnection(
1402
- "dl_cleanup",
1403
- ":memory:",
1404
- workDir,
1405
- );
1406
- try {
1407
- await conn.runSQL(`INSTALL postgres`);
1408
- await conn.runSQL(`LOAD postgres`);
1409
- await conn.runSQL(
1410
- `ATTACH '${pgConnString()}' AS cleanup (TYPE postgres);`,
1411
- );
1412
- for (const schema of createdSchemas) {
1413
- // Quoted, because at least one test registers a mixed-case
1414
- // schema and an unquoted identifier would not match it. And
1415
- // reported rather than swallowed: a silently failing drop
1416
- // leaks one schema per run into the shared catalog database,
1417
- // which is the accumulation this hook exists to prevent.
1418
- await conn
1419
- .runSQL(`DROP SCHEMA cleanup."${schema}" CASCADE;`)
1420
- .catch((error: unknown) =>
1421
- console.warn(
1422
- `Failed to drop test schema "${schema}":`,
1423
- error,
1424
- ),
1425
- );
1426
- }
1427
- } catch (error) {
1428
- console.warn("DuckLake schema cleanup failed:", error);
1429
- } finally {
1430
- await conn.close().catch(() => undefined);
1431
- }
1432
- });
1433
-
1434
- const duckLakeConfig = async (
1435
- name: string,
1436
- metadataSchema?: string,
1437
- ) => {
1438
- const dataDir = path.join(
1439
- testEnvironmentPath,
1440
- `${name}_data`,
1441
- );
1442
- await fs.mkdir(dataDir, { recursive: true });
1443
- return {
1444
- catalog: {
1445
- postgresConnection: pgCatalog(),
1446
- ...(metadataSchema ? { metadataSchema } : {}),
1447
- },
1448
- storage: { bucketUrl: `${dataDir}/` },
1449
- } as components["schemas"]["DucklakeConnection"];
1450
- };
1451
-
1452
- // Bootstrap a catalog the way a BUILD does — a read-WRITE attach.
1453
- // The read-only serve attach cannot create one ("creating a new
1454
- // DuckLake is explicitly disabled"), so anything exercising the
1455
- // serve path needs this first. Also the natural place to assert
1456
- // that the build path carries METADATA_SCHEMA.
1457
- const bootstrapCatalog = async (
1458
- dbName: string,
1459
- cfg: components["schemas"]["DucklakeConnection"],
1460
- ): Promise<DuckDBConnection> => {
1461
- const workDir = await fs.mkdtemp(
1462
- path.join(os.tmpdir(), `dl-boot-${dbName}-`),
1463
- );
1464
- const conn = new DuckDBConnection(
1465
- `bootstrap_${dbName}`,
1466
- ":memory:",
1467
- workDir,
1468
- );
1469
- createdConnections.push(conn);
1470
- await attachDuckLakeReadWrite(conn, dbName, cfg);
1471
- return conn;
1472
- };
1473
-
1474
- const connect = async (defs: ApiConnection[]) => {
1475
- const { malloyConnections } =
1476
- await createEnvironmentConnections(
1477
- defs,
1478
- testEnvironmentPath,
1479
- );
1480
- const out = new Map<string, DuckDBConnection>();
1481
- for (const def of defs) {
1482
- const c = malloyConnections.get(
1483
- def.name!,
1484
- ) as DuckDBConnection;
1485
- createdConnections.push(c);
1486
- out.set(def.name!, c);
1487
- }
1488
- return out;
1489
- };
1490
-
1491
- const firstValue = (rows: Record<string, unknown>[]) =>
1492
- Number(Object.values(rows[0])[0]);
1493
-
1494
- it(
1495
- "puts DuckLake metadata in the configured schema, creating it",
1496
- async () => {
1497
- if (!hasPostgresCredentials()) {
1498
- console.log("Skipping: PostgreSQL not configured");
1499
- return;
1500
- }
1501
- const schema = uniqueSchema("lands");
1502
- const cfg = await duckLakeConfig("dl_lands", schema);
1503
- const c = await bootstrapCatalog("dl_lands", cfg);
1504
- await c.runSQL(
1505
- `CREATE OR REPLACE TABLE dl_lands.t AS SELECT 1 AS x`,
1506
- );
1507
-
1508
- // Read the catalog database directly: the ducklake_*
1509
- // bookkeeping tables must be in `schema`, which DuckLake
1510
- // created on attach (this test never created the schema).
1511
- await c.runSQL(
1512
- `ATTACH '${pgConnString()}' AS insp (TYPE postgres, READ_ONLY);`,
1513
- );
1514
- const found = await c.runSQL(
1515
- `SELECT count(*) AS n FROM insp.${schema}.ducklake_metadata;`,
1516
- );
1517
- expect(firstValue(found.rows)).toBeGreaterThan(0);
1518
- },
1519
- { timeout: 60000 },
1520
- );
1521
-
1522
- it(
1523
- "isolates two catalogs sharing one catalog database",
1524
- async () => {
1525
- if (!hasPostgresCredentials()) {
1526
- console.log("Skipping: PostgreSQL not configured");
1527
- return;
1528
- }
1529
- const a = await bootstrapCatalog(
1530
- "dl_a",
1531
- await duckLakeConfig("dl_a", uniqueSchema("a")),
1532
- );
1533
- const b = await bootstrapCatalog(
1534
- "dl_b",
1535
- await duckLakeConfig("dl_b", uniqueSchema("b")),
1536
- );
1537
-
1538
- await a.runSQL(
1539
- `CREATE OR REPLACE TABLE dl_a.only_in_a AS SELECT 1`,
1540
- );
1541
- await b.runSQL(
1542
- `CREATE OR REPLACE TABLE dl_b.only_in_b AS SELECT 2`,
1543
- );
1544
-
1545
- const tablesOf = async (
1546
- conn: DuckDBConnection,
1547
- db: string,
1548
- ) => {
1549
- const r = await conn.runSQL(
1550
- `SELECT table_name FROM duckdb_tables() WHERE database_name = '${db}';`,
1551
- );
1552
- return r.rows.map((row) => Object.values(row)[0]);
1553
- };
1554
-
1555
- // Each catalog sees its own table and NOT the other's —
1556
- // the whole point of a per-catalog metadata schema. Both
1557
- // live in ONE catalog database.
1558
- expect(await tablesOf(a, "dl_a")).toEqual(["only_in_a"]);
1559
- expect(await tablesOf(b, "dl_b")).toEqual(["only_in_b"]);
1560
- },
1561
- { timeout: 60000 },
1562
- );
1563
-
1564
- it(
1565
- "serves read-only from a catalog in a non-default schema",
1566
- async () => {
1567
- if (!hasPostgresCredentials()) {
1568
- console.log("Skipping: PostgreSQL not configured");
1569
- return;
1570
- }
1571
- // The read-only serve attach must pass METADATA_SCHEMA too:
1572
- // without it the attach looks in the catalog's default
1573
- // schema, finds nothing, and fails outright, because a
1574
- // read-only attach may not create a catalog.
1575
- const schema = uniqueSchema("serve");
1576
- const cfg = await duckLakeConfig("dl_serve", schema);
1577
- const boot = await bootstrapCatalog("dl_serve", cfg);
1578
- await boot.runSQL(
1579
- `CREATE OR REPLACE TABLE dl_serve.t AS SELECT 7 AS x`,
1580
- );
1581
-
1582
- const conns = await connect([
1583
- {
1584
- name: "dl_serve",
1585
- type: "ducklake",
1586
- ducklakeConnection: cfg,
1587
- } as ApiConnection,
1588
- ]);
1589
- const served = conns.get("dl_serve")!;
1590
- const r = await served.runSQL(`SELECT x FROM dl_serve.t;`);
1591
- expect(firstValue(r.rows)).toBe(7);
1592
- },
1593
- { timeout: 60000 },
1594
- );
1595
-
1596
- it(
1597
- "keeps the catalog-format preflight working for a non-default schema",
1598
- async () => {
1599
- if (!hasPostgresCredentials()) {
1600
- console.log("Skipping: PostgreSQL not configured");
1601
- return;
1602
- }
1603
- // The preflight reads `ducklake_metadata`, which lives in
1604
- // the configured schema. Read unqualified it would MISS,
1605
- // and the preflight fails SOFT (logs and returns) — so the
1606
- // range check would silently stop protecting exactly the
1607
- // catalogs setting this option. Poisoning the recorded
1608
- // version proves the read resolves: an unqualified read
1609
- // finds nothing and so raises nothing.
1610
- const schema = uniqueSchema("preflight");
1611
- const cfg = await duckLakeConfig("dl_pf", schema);
1612
- const boot = await bootstrapCatalog("dl_pf", cfg);
1613
- await boot.runSQL(
1614
- `ATTACH '${pgConnString()}' AS poison (TYPE postgres);`,
1615
- );
1616
- await boot.runSQL(
1617
- `UPDATE poison.${schema}.ducklake_metadata ` +
1618
- `SET value = '9999.0' WHERE key = 'version';`,
1619
- );
1620
-
1621
- // A fresh attach of the same catalog must now refuse with
1622
- // the clean typed range error rather than proceeding.
1623
- let err: unknown;
1624
- try {
1625
- await bootstrapCatalog("dl_pf_again", cfg);
1626
- } catch (e) {
1627
- err = e;
1628
- }
1629
- // The TYPE matters: converting a deep DuckDB failure into
1630
- // this typed refusal is the preflight's whole job, so a
1631
- // message match alone could not tell the two apart.
1632
- expect(err).toBeInstanceOf(UnsupportedCatalogFormatError);
1633
- // And it must be THIS catalog's poisoned format. Asserting
1634
- // the value keeps the test honest if some other schema in
1635
- // the shared catalog database holds a valid catalog: an
1636
- // unqualified read would report that one's format instead,
1637
- // which is exactly the bug being guarded against.
1638
- expect((err as Error).message).toContain("9999.0");
1639
- },
1640
- { timeout: 60000 },
1641
- );
1642
-
1643
- it(
1644
- "round-trips a mixed-case metadataSchema through the preflight",
1645
- async () => {
1646
- if (!hasPostgresCredentials()) {
1647
- console.log("Skipping: PostgreSQL not configured");
1648
- return;
1649
- }
1650
- // The validator accepts mixed case but nothing exercised it, so
1651
- // this covers the create -> preflight -> attach round-trip for
1652
- // such a name. Honest scope: it does NOT discriminate on the
1653
- // preflight's identifier quoting — measured, unquoted resolves
1654
- // too, because DuckDB matches identifiers case-insensitively.
1655
- // Poisoning the version is what proves the preflight read THIS
1656
- // catalog's metadata rather than missing silently.
1657
- const schema = `DlMixedCase_${runId}`;
1658
- createdSchemas.push(schema);
1659
- const cfg = await duckLakeConfig("dl_mixed", schema);
1660
- const boot = await bootstrapCatalog("dl_mixed", cfg);
1661
- await boot.runSQL(
1662
- `ATTACH '${pgConnString()}' AS poison_mixed (TYPE postgres);`,
1663
- );
1664
- await boot.runSQL(
1665
- `UPDATE poison_mixed."${schema}".ducklake_metadata ` +
1666
- `SET value = '9999.0' WHERE key = 'version';`,
1667
- );
1668
-
1669
- let err: unknown;
1670
- try {
1671
- await bootstrapCatalog("dl_mixed_again", cfg);
1672
- } catch (e) {
1673
- err = e;
1674
- }
1675
- expect(err).toBeInstanceOf(UnsupportedCatalogFormatError);
1676
- expect((err as Error).message).toContain("9999.0");
1677
- },
1678
- { timeout: 60000 },
1679
- );
1680
-
1681
- it(
1682
- "attaches without a metadataSchema exactly as before",
1683
- async () => {
1684
- if (!hasPostgresCredentials()) {
1685
- console.log("Skipping: PostgreSQL not configured");
1686
- return;
1687
- }
1688
- // Back-compat: the option is a conditional append, so an
1689
- // absent value emits the pre-existing ATTACH verbatim and
1690
- // DuckLake uses the catalog connection's default schema.
1691
- const cfg = await duckLakeConfig("dl_default");
1692
- const c = await bootstrapCatalog("dl_default", cfg);
1693
- await c.runSQL(
1694
- `CREATE OR REPLACE TABLE dl_default.t AS SELECT 7 AS x`,
1695
- );
1696
- const r = await c.runSQL(`SELECT x FROM dl_default.t;`);
1697
- expect(firstValue(r.rows)).toBe(7);
1698
-
1699
- await c.runSQL(
1700
- `ATTACH '${pgConnString()}' AS insp2 (TYPE postgres, READ_ONLY);`,
1701
- );
1702
- const inPublic = await c.runSQL(
1703
- `SELECT count(*) AS n FROM insp2.public.ducklake_metadata;`,
1704
- );
1705
- expect(firstValue(inPublic.rows)).toBeGreaterThan(0);
1706
- },
1707
- { timeout: 60000 },
1708
- );
1709
- });
1710
- });
1711
-
1712
- it("should throw error if DuckDB connection name conflicts with attached database", async () => {
1713
- await expect(
1714
- createEnvironmentConnections(
1715
- [
1716
- {
1717
- name: "conflict_db",
1718
- type: "duckdb",
1719
- duckdbConnection: {
1720
- attachedDatabases: [
1721
- {
1722
- name: "conflict_db",
1723
- type: "postgres",
1724
- postgresConnection: {
1725
- connectionString:
1726
- "postgresql://localhost/test",
1727
- },
1728
- },
1729
- ],
1730
- },
1731
- },
1732
- ],
1733
- testEnvironmentPath,
1734
- ),
1735
- ).rejects.toThrow(/cannot conflict/);
1736
- });
1737
-
1738
- it("should throw error if connection name is 'duckdb'", async () => {
1739
- await expect(
1740
- createEnvironmentConnections(
1741
- [
1742
- {
1743
- name: "duckdb",
1744
- type: "duckdb",
1745
- duckdbConnection: { attachedDatabases: [] },
1746
- },
1747
- ],
1748
- testEnvironmentPath,
1749
- ),
1750
- ).rejects.toThrow(/'duckdb' is reserved/);
1751
- });
1752
-
1753
- it("should reject DuckDB connections with no attachments", async () => {
1754
- // Env-level DuckDB connections must declare at least one
1755
- // attached foreign database; the empty-array case is operator
1756
- // confusion (the per-package "duckdb" sandbox already covers
1757
- // the plain-in-memory use case).
1758
- await expect(
1759
- createEnvironmentConnections(
1760
- [
1761
- {
1762
- name: "empty_duckdb",
1763
- type: "duckdb",
1764
- duckdbConnection: { attachedDatabases: [] },
1765
- },
1766
- ],
1767
- testEnvironmentPath,
1768
- ),
1769
- ).rejects.toThrow(/has no attached databases/);
1770
- });
1771
-
1772
- it("should reject unsupported DuckDB connector fields", async () => {
1773
- await expect(
1774
- createEnvironmentConnections(
1775
- [
1776
- {
1777
- name: "duckdb_with_setup_sql",
1778
- type: "duckdb",
1779
- duckdbConnection: {
1780
- attachedDatabases: [],
1781
- setupSQL: "INSTALL httpfs",
1782
- },
1783
- } as unknown as ApiConnection,
1784
- ],
1785
- testEnvironmentPath,
1786
- ),
1787
- ).rejects.toThrow(/Unsupported DuckDB connection field/);
1788
- });
1789
-
1790
- it("should reject environment-authored DuckDB policy fields", async () => {
1791
- await expect(
1792
- createEnvironmentConnections(
1793
- [
1794
- {
1795
- name: "duckdb_with_policy",
1796
- type: "duckdb",
1797
- duckdbConnection: {
1798
- attachedDatabases: [],
1799
- securityPolicy: "sandboxed",
1800
- },
1801
- } as unknown as ApiConnection,
1802
- ],
1803
- testEnvironmentPath,
1804
- ),
1805
- ).rejects.toThrow(/Unsupported DuckDB connection field/);
1806
- });
1807
-
1808
- it("should preserve Snowflake private-key auth options", async () => {
1809
- const { malloyConnections, releaseConnections } =
1810
- await createEnvironmentConnections(
1811
- [
1812
- {
1813
- name: "sf_private_key",
1814
- type: "snowflake",
1815
- snowflakeConnection: {
1816
- account: "test-account",
1817
- username: "test-user",
1818
- privateKey:
1819
- "-----BEGIN PRIVATE KEY-----MIIB-----END PRIVATE KEY-----",
1820
- warehouse: "test-warehouse",
1821
- },
1822
- },
1823
- ],
1824
- testEnvironmentPath,
1825
- );
1826
-
1827
- try {
1828
- const connection = malloyConnections.get(
1829
- "sf_private_key",
1830
- ) as unknown as { connOptions: Record<string, unknown> };
1831
- expect(connection.connOptions.authenticator).toBe(
1832
- "SNOWFLAKE_JWT",
1833
- );
1834
- expect(connection.connOptions.privateKey).toContain(
1835
- "BEGIN PRIVATE KEY",
1836
- );
1837
- } finally {
1838
- await releaseConnections();
1839
- }
1840
- });
1841
-
1842
- it("should translate Trino Peaka credentials to core extraCredential", () => {
1843
- const assembled = assembleEnvironmentConnections(
1844
- [
1845
- {
1846
- name: "trino_peaka",
1847
- type: "trino",
1848
- trinoConnection: {
1849
- server: "https://example.com",
1850
- port: 443,
1851
- catalog: "catalog",
1852
- schema: "schema",
1853
- user: "user",
1854
- peakaKey: "peaka-secret",
1855
- },
1856
- },
1857
- ],
1858
- testEnvironmentPath,
1859
- );
1860
-
1861
- expect(
1862
- assembled.pojo.connections.trino_peaka.extraCredential,
1863
- ).toEqual({ peakaKey: "peaka-secret" });
1864
- expect(
1865
- assembled.pojo.connections.trino_peaka.extraConfig,
1866
- ).toBeUndefined();
1867
- });
1868
-
1869
- it("should validate environment-level BigQuery service account keys", () => {
1870
- expect(() =>
1871
- assembleEnvironmentConnections(
1872
- [
1873
- {
1874
- name: "bq_invalid",
1875
- type: "bigquery",
1876
- bigqueryConnection: {
1877
- defaultProjectId: "test-environment",
1878
- serviceAccountKeyJson: '{"invalid":"key"}',
1879
- },
1880
- },
1881
- ],
1882
- testEnvironmentPath,
1883
- ),
1884
- ).toThrow(/missing "type" field/);
1885
- });
1886
-
1887
- it("should preserve PGSSLMODE for environment-level Postgres", () => {
1888
- const previousPgSslMode = process.env.PGSSLMODE;
1889
- process.env.PGSSLMODE = "require";
1890
- try {
1891
- const assembled = assembleEnvironmentConnections(
1892
- [
1893
- {
1894
- name: "pg_ssl",
1895
- type: "postgres",
1896
- postgresConnection: {
1897
- host: "localhost",
1898
- port: 5432,
1899
- userName: "user",
1900
- password: "pass",
1901
- databaseName: "db",
1902
- },
1903
- },
1904
- ],
1905
- testEnvironmentPath,
1906
- );
1907
-
1908
- expect(
1909
- assembled.pojo.connections.pg_ssl.connectionString,
1910
- ).toContain("sslmode=require");
1911
- } finally {
1912
- if (previousPgSslMode === undefined) {
1913
- delete process.env.PGSSLMODE;
1914
- } else {
1915
- process.env.PGSSLMODE = previousPgSslMode;
1916
- }
1917
- }
1918
- });
1919
-
1920
- it("should use environment-root-relative file paths for environment-level DuckDB", async () => {
1921
- const insideCsvPath = path.join(testEnvironmentPath, "inside.csv");
1922
- await fs.writeFile(insideCsvPath, "id\n1\n");
1923
-
1924
- const minimalGcsAttachment = {
1925
- name: "test_gcs",
1926
- type: "gcs" as const,
1927
- gcsConnection: {
1928
- keyId: "test-key-id",
1929
- secret: "test-secret",
1930
- },
1931
- };
1932
-
1933
- const { malloyConnections } = await createEnvironmentConnections(
1934
- [
1935
- {
1936
- name: "environment_scoped_duckdb",
1937
- type: "duckdb",
1938
- duckdbConnection: {
1939
- attachedDatabases: [minimalGcsAttachment],
1940
- },
1941
- },
1942
- ],
1943
- testEnvironmentPath,
1944
- );
1945
-
1946
- const connection = malloyConnections.get(
1947
- "environment_scoped_duckdb",
1948
- ) as DuckDBConnection;
1949
- createdConnections.push(connection);
1950
-
1951
- const assembled = assembleEnvironmentConnections(
1952
- [
1953
- {
1954
- name: "environment_scoped_duckdb",
1955
- type: "duckdb",
1956
- duckdbConnection: {
1957
- attachedDatabases: [minimalGcsAttachment],
1958
- },
1959
- },
1960
- ],
1961
- testEnvironmentPath,
1962
- );
1963
- expect(
1964
- assembled.pojo.connections.environment_scoped_duckdb
1965
- .workingDirectory,
1966
- ).toBeUndefined();
1967
- expect(
1968
- assembled.pojo.connections.environment_scoped_duckdb
1969
- .securityPolicy,
1970
- ).toBeUndefined();
1971
-
1972
- await expect(
1973
- connection.runSQL("SELECT * FROM read_csv_auto('inside.csv')"),
1974
- ).resolves.toBeDefined();
1975
- });
1976
-
1977
- it("should keep external access available for federated DuckDB entries", () => {
1978
- const assembled = assembleEnvironmentConnections(
1979
- [
1980
- {
1981
- name: "federated_duckdb",
1982
- type: "duckdb",
1983
- duckdbConnection: {
1984
- attachedDatabases: [
1985
- {
1986
- name: "pg",
1987
- type: "postgres",
1988
- postgresConnection: {
1989
- host: "localhost",
1990
- port: 5432,
1991
- userName: "user",
1992
- password: "pass",
1993
- databaseName: "db",
1994
- },
1995
- },
1996
- ],
1997
- },
1998
- },
1999
- ],
2000
- testEnvironmentPath,
2001
- );
2002
-
2003
- const entry = assembled.pojo.connections.federated_duckdb;
2004
- expect(entry.securityPolicy).toBeUndefined();
2005
- expect(entry.enableExternalAccess).toBeUndefined();
2006
- expect(entry.allowedDirectories).toBeUndefined();
2007
- });
2008
-
2009
- it("should keep external access available for MotherDuck entries", () => {
2010
- const assembled = assembleEnvironmentConnections(
2011
- [
2012
- {
2013
- name: "md",
2014
- type: "motherduck",
2015
- motherduckConnection: {
2016
- accessToken: "token",
2017
- database: "db",
2018
- },
2019
- },
2020
- ],
2021
- testEnvironmentPath,
2022
- );
2023
-
2024
- const entry = assembled.pojo.connections.md;
2025
- expect(entry.securityPolicy).toBeUndefined();
2026
- expect(entry.enableExternalAccess).toBeUndefined();
2027
- expect(entry.allowedDirectories).toBeUndefined();
2028
- });
2029
-
2030
- it("should handle already attached database gracefully", async () => {
2031
- if (!hasPostgresCredentials()) {
2032
- console.log("Skipping: PostgreSQL credentials not configured");
2033
- return;
2034
- }
2035
-
2036
- const postgresAttachment: AttachedDatabase = {
2037
- name: "pg_duplicate",
2038
- type: "postgres",
2039
- postgresConnection: {
2040
- host: process.env.POSTGRES_TEST_HOST,
2041
- port: parseInt(process.env.POSTGRES_TEST_PORT || "5432"),
2042
- userName: process.env.POSTGRES_TEST_USER!,
2043
- password: process.env.POSTGRES_TEST_PASSWORD!,
2044
- databaseName: process.env.POSTGRES_TEST_DATABASE,
2045
- },
2046
- };
2047
-
2048
- const { malloyConnections } = await createEnvironmentConnections(
2049
- [
2050
- {
2051
- name: "duckdb_duplicate_test",
2052
- type: "duckdb",
2053
- duckdbConnection: {
2054
- attachedDatabases: [
2055
- postgresAttachment,
2056
- postgresAttachment,
2057
- ],
2058
- },
2059
- },
2060
- ],
2061
- testEnvironmentPath,
2062
- );
2063
-
2064
- const connection = malloyConnections.get(
2065
- "duckdb_duplicate_test",
2066
- ) as DuckDBConnection;
2067
- createdConnections.push(connection);
2068
- expect(connection).toBeDefined();
2069
- });
2070
- });
2071
- });
2072
-
2073
- describe("testConnectionConfig", () => {
2074
- it(
2075
- "should successfully test valid PostgreSQL connection",
2076
- async () => {
2077
- if (!hasPostgresCredentials()) {
2078
- console.log("Skipping: PostgreSQL credentials not configured");
2079
- return;
2080
- }
2081
-
2082
- const result = await testConnectionConfig({
2083
- name: "test_postgres",
2084
- type: "postgres",
2085
- postgresConnection: {
2086
- host: process.env.POSTGRES_TEST_HOST,
2087
- port: parseInt(process.env.POSTGRES_TEST_PORT || "5432"),
2088
- userName: process.env.POSTGRES_TEST_USER!,
2089
- password: process.env.POSTGRES_TEST_PASSWORD!,
2090
- databaseName: process.env.POSTGRES_TEST_DATABASE,
2091
- },
2092
- });
2093
-
2094
- expect(result.status).toBe("ok");
2095
- expect(result.errorMessage).toBe("");
2096
- },
2097
- { timeout: 30000 },
2098
- );
2099
-
2100
- it(
2101
- "should fail for invalid PostgreSQL credentials",
2102
- async () => {
2103
- const result = await testConnectionConfig({
2104
- name: "test_postgres_invalid",
2105
- type: "postgres",
2106
- postgresConnection: {
2107
- host: process.env.POSTGRES_TEST_HOST,
2108
- port: parseInt(process.env.POSTGRES_TEST_PORT || "5432"),
2109
- userName: "invalid_user",
2110
- password: "invalid_password",
2111
- databaseName: "nonexistent",
2112
- },
2113
- });
2114
-
2115
- expect(result.status).toBe("failed");
2116
- expect(result.errorMessage).toBeDefined();
2117
- },
2118
- { timeout: 30000 },
2119
- );
2120
-
2121
- it("should fail for missing connection name", async () => {
2122
- const result = await testConnectionConfig({
2123
- name: "",
2124
- type: "postgres",
2125
- postgresConnection: {},
2126
- });
2127
-
2128
- expect(result.status).toBe("failed");
2129
- expect(result.errorMessage).toContain("name is required");
2130
- });
2131
- });
2132
-
2133
- describe("SQL injection prevention", () => {
2134
- it("should properly escape special characters in credentials", async () => {
2135
- if (!hasPostgresCredentials()) {
2136
- console.log("Skipping: PostgreSQL credentials not configured");
2137
- return;
2138
- }
2139
-
2140
- const { malloyConnections } = await createEnvironmentConnections(
2141
- [
2142
- {
2143
- name: "duckdb_special_chars",
2144
- type: "duckdb",
2145
- duckdbConnection: {
2146
- attachedDatabases: [
2147
- {
2148
- name: "pg_special",
2149
- type: "postgres",
2150
- postgresConnection: {
2151
- host: process.env.POSTGRES_TEST_HOST,
2152
- port: parseInt(
2153
- process.env.POSTGRES_TEST_PORT || "5432",
2154
- ),
2155
- userName: process.env.POSTGRES_TEST_USER!,
2156
- password: process.env.POSTGRES_TEST_PASSWORD!,
2157
- databaseName: process.env.POSTGRES_TEST_DATABASE,
2158
- },
2159
- },
2160
- ],
2161
- },
2162
- },
2163
- ],
2164
- testEnvironmentPath,
2165
- );
2166
-
2167
- const connection = malloyConnections.get(
2168
- "duckdb_special_chars",
2169
- ) as DuckDBConnection;
2170
- createdConnections.push(connection);
2171
- expect(connection).toBeDefined();
2172
- });
2173
- });
2174
-
2175
- describe("connection attributes", () => {
2176
- it(
2177
- "should return correct attributes for DuckDB connection",
2178
- async () => {
2179
- if (!hasPostgresCredentials()) {
2180
- console.log("Skipping: PostgreSQL credentials not configured");
2181
- return;
2182
- }
2183
-
2184
- const { apiConnections } = await createEnvironmentConnections(
2185
- [
2186
- {
2187
- name: "duckdb_attrs",
2188
- type: "duckdb",
2189
- duckdbConnection: {
2190
- attachedDatabases: [
2191
- {
2192
- name: "pg_attrs",
2193
- type: "postgres",
2194
- postgresConnection: {
2195
- host: process.env.POSTGRES_TEST_HOST,
2196
- port: parseInt(
2197
- process.env.POSTGRES_TEST_PORT || "5432",
2198
- ),
2199
- userName: process.env.POSTGRES_TEST_USER!,
2200
- password: process.env.POSTGRES_TEST_PASSWORD!,
2201
- databaseName:
2202
- process.env.POSTGRES_TEST_DATABASE,
2203
- },
2204
- },
2205
- ],
2206
- },
2207
- },
2208
- ],
2209
- testEnvironmentPath,
2210
- );
2211
-
2212
- const connection = apiConnections[0];
2213
- expect(connection.attributes).toBeDefined();
2214
- expect(connection.attributes?.dialectName).toBe("duckdb");
2215
- expect(typeof connection.attributes?.canPersist).toBe("boolean");
2216
- expect(typeof connection.attributes?.canStream).toBe("boolean");
2217
- expect(typeof connection.attributes?.isPool).toBe("boolean");
2218
- },
2219
- { timeout: 30000 },
2220
- );
2221
- });
2222
- });
2223
-
2224
- describe("buildProxiedSslQuery", () => {
2225
- const ORIG_CA = process.env.NODE_EXTRA_CA_CERTS;
2226
- afterEach(() => {
2227
- if (ORIG_CA === undefined) delete process.env.NODE_EXTRA_CA_CERTS;
2228
- else process.env.NODE_EXTRA_CA_CERTS = ORIG_CA;
2229
- });
2230
-
2231
- it("defaults to no-verify when sslmode is unset (force-SSL target isn't rejected for plaintext)", () => {
2232
- expect(buildProxiedSslQuery("conn")).toBe("?sslmode=no-verify");
2233
- });
2234
-
2235
- it("returns no-verify for explicit no-verify", () => {
2236
- expect(buildProxiedSslQuery("conn", "no-verify")).toBe(
2237
- "?sslmode=no-verify",
2238
- );
2239
- });
2240
-
2241
- it("emits an explicit sslmode=disable, never an empty query", () => {
2242
- // Regression: an empty query lets pg fall back to the deployment's PGSSLMODE
2243
- // env, which would verify and fail the 127.0.0.1 hostname check.
2244
- expect(buildProxiedSslQuery("conn", "disable")).toBe("?sslmode=disable");
2245
- });
2246
-
2247
- it("builds a chain-verifying, hostname-skipped query for verify-ca", () => {
2248
- process.env.NODE_EXTRA_CA_CERTS = "/etc/ssl/certs/rds bundle.pem";
2249
- const q = buildProxiedSslQuery("conn", "verify-ca");
2250
- expect(q).toContain("uselibpqcompat=true");
2251
- expect(q).toContain("sslmode=verify-ca");
2252
- expect(q).toContain(
2253
- `sslrootcert=${encodeURIComponent("/etc/ssl/certs/rds bundle.pem")}`,
2254
- );
2255
- });
2256
-
2257
- it("throws for verify-ca when no CA bundle is available", () => {
2258
- delete process.env.NODE_EXTRA_CA_CERTS;
2259
- expect(() => buildProxiedSslQuery("conn", "verify-ca")).toThrow(
2260
- /no trusted CA bundle/i,
2261
- );
2262
- });
2263
-
2264
- it("throws for an unsupported sslmode", () => {
2265
- expect(() => buildProxiedSslQuery("conn", "require")).toThrow(
2266
- /unsupported sslmode/i,
2267
- );
2268
- });
2269
-
2270
- it("verify-ca connectionString parses to an ssl config (pg-connection-string version guard)", async () => {
2271
- // uselibpqcompat=true (pg-connection-string >= 2.7) is what turns verify-ca
2272
- // into "verify chain, skip hostname". If pg is ever downgraded below that
2273
- // floor, sslmode would be ignored — this guard fails loudly instead.
2274
- const caPath = path.join(os.tmpdir(), `proxy-ca-${process.pid}.pem`);
2275
- await fs.writeFile(
2276
- caPath,
2277
- "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n",
2278
- );
2279
- process.env.NODE_EXTRA_CA_CERTS = caPath;
2280
- try {
2281
- const q = buildProxiedSslQuery("conn", "verify-ca");
2282
- const { parse } = await import("pg-connection-string");
2283
- const parsed = parse(`postgresql://127.0.0.1:5432/db${q}`);
2284
- expect(parsed.ssl).toBeTruthy();
2285
- expect(typeof parsed.ssl).toBe("object");
2286
- } finally {
2287
- await fs.rm(caPath, { force: true });
2288
- }
2289
- });
2290
- });
2291
-
2292
- describe("resolveProxiedTls", () => {
2293
- it("routes verify-full to a raw ssl object with servername = the real host", () => {
2294
- // verify-full (chain + hostname) can't ride the connectionString through a
2295
- // tunnel — it needs ssl.servername set to the real DB host so pg checks the
2296
- // cert's SAN against it, not against the 127.0.0.1 socket address.
2297
- const { query, ssl } = resolveProxiedTls(
2298
- "conn",
2299
- "db.internal.example.com",
2300
- "verify-full",
2301
- );
2302
- expect(query).toBe("");
2303
- expect(ssl).toEqual({
2304
- servername: "db.internal.example.com",
2305
- rejectUnauthorized: true,
2306
- });
2307
- });
2308
-
2309
- it("throws for verify-full with no host (servername would be undefined)", () => {
2310
- expect(() => resolveProxiedTls("conn", undefined, "verify-full")).toThrow(
2311
- /verify-full.*no host/i,
2312
- );
2313
- });
2314
-
2315
- it("routes the connectionString-param modes through buildProxiedSslQuery with no ssl object", () => {
2316
- // verify-ca needs a CA bundle to build its query, so set one for the loop.
2317
- const prior = process.env.NODE_EXTRA_CA_CERTS;
2318
- process.env.NODE_EXTRA_CA_CERTS = "/etc/ssl/certs/bundle.pem";
2319
- try {
2320
- for (const mode of ["disable", "no-verify", "verify-ca", undefined]) {
2321
- const { query, ssl } = resolveProxiedTls("conn", "127.0.0.1", mode);
2322
- expect(query).toBe(buildProxiedSslQuery("conn", mode));
2323
- expect(ssl).toBeUndefined();
2324
- }
2325
- } finally {
2326
- if (prior === undefined) delete process.env.NODE_EXTRA_CA_CERTS;
2327
- else process.env.NODE_EXTRA_CA_CERTS = prior;
2328
- }
2329
- });
2330
-
2331
- it("verify-full: a bare connectionString does not clobber the explicit ssl object (pg merge guard)", async () => {
2332
- // Load-bearing invariant: pg's ConnectionParameters does
2333
- // Object.assign({}, config, parse(connectionString)). A verify-full
2334
- // connectionString carries no sslmode, so pg-connection-string adds no
2335
- // `ssl` key and our explicit { servername, rejectUnauthorized } survives
2336
- // the merge. If a future pg bump changes this — or we accidentally emit an
2337
- // sslmode into the query — this guard fails loudly instead of silently
2338
- // downgrading verify-full.
2339
- const { query, ssl } = resolveProxiedTls(
2340
- "conn",
2341
- "db.internal.example.com",
2342
- "verify-full",
2343
- );
2344
- const { default: ConnectionParameters } = await import(
2345
- "pg/lib/connection-parameters.js"
2346
- );
2347
- const params = new (ConnectionParameters as unknown as new (
2348
- c: unknown,
2349
- ) => {
2350
- ssl: unknown;
2351
- })({
2352
- connectionString: `postgresql://u:p@127.0.0.1:5432/db${query}`,
2353
- ssl,
2354
- });
2355
- expect(params.ssl).toEqual({
2356
- servername: "db.internal.example.com",
2357
- rejectUnauthorized: true,
2358
- });
2359
- });
2360
- });