@malloy-publisher/server 0.0.235 → 0.0.237
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +83 -27
- package/dist/app/assets/{EnvironmentPage-BsAnavYN.js → EnvironmentPage-DaP6CpgK.js} +1 -1
- package/dist/app/assets/{HomePage-CADE138j.js → HomePage-C9yNw3iu.js} +1 -1
- package/dist/app/assets/{LightMode-Cfh7KzN8.js → LightMode-Be4-G-dX.js} +1 -1
- package/dist/app/assets/{MainPage-CO3pRlnV.js → MainPage-J_rU2SgI.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-p9YjkRXZ.js → MaterializationsPage-DHgPI4MO.js} +1 -1
- package/dist/app/assets/ModelPage-BzXfjTu5.js +1 -0
- package/dist/app/assets/{PackagePage-e4kN75YR.js → PackagePage-ByoSTLJ4.js} +1 -1
- package/dist/app/assets/RouteError-CrSeKN2r.js +1 -0
- package/dist/app/assets/{ThemeEditorPage-CciagFTq.js → ThemeEditorPage-Btxnmhtk.js} +1 -1
- package/dist/app/assets/{WorkbookPage-DNWmkCXa.js → WorkbookPage-D6JLPnWO.js} +1 -1
- package/dist/app/assets/{core-Rj_4rRnA.es-BZyvITuO.js → core-BSRgt1mG.es-Bc_D2Lq4.js} +1 -1
- package/dist/app/assets/{index-CH2AcDzc.js → index-BRijWj6K.js} +15 -15
- package/dist/app/assets/{index-DQa463gC.js → index-BX4D94xw.js} +1 -1
- package/dist/app/assets/{index-VBbcc8s6.js → index-Cvy0xtb7.js} +1 -1
- package/dist/app/assets/{index-DQpV7MyA.js → index-DMAhIJNC.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/package_load_worker.mjs +59 -20
- package/dist/server.mjs +1342 -616
- package/package.json +8 -2
- package/.eslintrc.json +0 -44
- package/.prettierignore +0 -1
- package/.prettierrc +0 -4
- package/build.ts +0 -97
- package/dist/app/assets/ModelPage-C1OSTv-x.js +0 -1
- package/dist/app/assets/RouteError-CzbfOkng.js +0 -1
- package/eslint.config.mjs +0 -76
- package/k6-tests/README.md +0 -144
- package/k6-tests/bun.lock +0 -797
- package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
- package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
- package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
- package/k6-tests/load-test/load-test-crud.ts +0 -142
- package/k6-tests/load-test/load-test.ts +0 -784
- package/k6-tests/package.json +0 -27
- package/k6-tests/scripts/run-with-otel.sh +0 -50
- package/k6-tests/smoke-test/smoke-test.ts +0 -158
- package/k6-tests/tsconfig.json +0 -29
- package/k6-tests/utils/client_factory.ts +0 -128
- package/k6-tests/utils/common.ts +0 -489
- package/k6-tests/utils/logger.ts +0 -62
- package/k6-tests/utils/types.d.ts +0 -35
- package/malloy_mcp.dxt +0 -0
- package/publisher.config.example.bigquery.json +0 -33
- package/publisher.config.example.duckdb.json +0 -23
- package/publisher.config.json +0 -23
- package/scripts/bake-duckdb-extensions.js +0 -110
- package/src/config.spec.ts +0 -1535
- package/src/config.theme.spec.ts +0 -178
- package/src/config.ts +0 -1142
- package/src/constants.ts +0 -105
- package/src/controller/compile.controller.ts +0 -40
- package/src/controller/connection.controller.spec.ts +0 -849
- package/src/controller/connection.controller.ts +0 -901
- package/src/controller/database.controller.ts +0 -24
- package/src/controller/materialization.controller.spec.ts +0 -293
- package/src/controller/materialization.controller.ts +0 -319
- package/src/controller/model.controller.ts +0 -164
- package/src/controller/package.controller.spec.ts +0 -273
- package/src/controller/package.controller.ts +0 -306
- package/src/controller/query.controller.ts +0 -152
- package/src/controller/theme.controller.ts +0 -83
- package/src/controller/watch-mode.controller.ts +0 -243
- package/src/data_styles.ts +0 -71
- package/src/default-publisher.config.json +0 -23
- package/src/ducklake_version.spec.ts +0 -163
- package/src/ducklake_version.ts +0 -153
- package/src/errors.spec.ts +0 -119
- package/src/errors.ts +0 -237
- package/src/filter_deprecation.spec.ts +0 -64
- package/src/filter_deprecation.ts +0 -42
- package/src/health.spec.ts +0 -123
- package/src/health.ts +0 -233
- package/src/heap_check.spec.ts +0 -144
- package/src/heap_check.ts +0 -144
- package/src/instrumentation.ts +0 -214
- package/src/json_utils.spec.ts +0 -51
- package/src/json_utils.ts +0 -33
- package/src/logger.spec.ts +0 -210
- package/src/logger.ts +0 -251
- package/src/malloy_pin_prereqs.spec.ts +0 -25
- package/src/materialization_metrics.spec.ts +0 -169
- package/src/materialization_metrics.ts +0 -346
- package/src/mcp/error_messages.spec.ts +0 -35
- package/src/mcp/error_messages.ts +0 -183
- package/src/mcp/handler_utils.spec.ts +0 -108
- package/src/mcp/handler_utils.ts +0 -251
- package/src/mcp/mcp_constants.ts +0 -6
- package/src/mcp/query_envelope.spec.ts +0 -229
- package/src/mcp/query_envelope.ts +0 -240
- package/src/mcp/server.protocol.spec.ts +0 -250
- package/src/mcp/server.ts +0 -87
- package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
- package/src/mcp/skills/build_skills_bundle.ts +0 -176
- package/src/mcp/skills/skills_bundle.json +0 -1
- package/src/mcp/skills/skills_bundle.spec.ts +0 -171
- package/src/mcp/tool_response.spec.ts +0 -108
- package/src/mcp/tool_response.ts +0 -138
- package/src/mcp/tools/compile_tool.spec.ts +0 -315
- package/src/mcp/tools/compile_tool.ts +0 -208
- package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
- package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
- package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
- package/src/mcp/tools/docs_search_tool.ts +0 -128
- package/src/mcp/tools/embedding_index.spec.ts +0 -1236
- package/src/mcp/tools/embedding_index.ts +0 -808
- package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
- package/src/mcp/tools/execute_query_tool.ts +0 -316
- package/src/mcp/tools/get_context_eval.ts +0 -275
- package/src/mcp/tools/get_context_tool.spec.ts +0 -592
- package/src/mcp/tools/get_context_tool.ts +0 -585
- package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
- package/src/mcp/tools/reload_package_tool.ts +0 -132
- package/src/mcp_config.spec.ts +0 -919
- package/src/mcp_config.ts +0 -425
- package/src/oom_guards.integration.spec.ts +0 -269
- package/src/package_load/package_load_pool.spec.ts +0 -252
- package/src/package_load/package_load_pool.ts +0 -931
- package/src/package_load/package_load_worker.ts +0 -1034
- package/src/package_load/protocol.ts +0 -364
- package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
- package/src/package_load/rpc_wait_accountant.ts +0 -76
- package/src/package_load_metrics.spec.ts +0 -114
- package/src/package_load_metrics.ts +0 -127
- package/src/path_safety.spec.ts +0 -158
- package/src/path_safety.ts +0 -146
- package/src/pg_helpers.spec.ts +0 -223
- package/src/pg_helpers.ts +0 -52
- package/src/query_cap_metrics.spec.ts +0 -89
- package/src/query_cap_metrics.ts +0 -116
- package/src/query_concurrency.spec.ts +0 -247
- package/src/query_concurrency.ts +0 -237
- package/src/query_metadata_metrics.ts +0 -49
- package/src/query_param_utils.ts +0 -29
- package/src/query_timeout.spec.ts +0 -224
- package/src/query_timeout.ts +0 -179
- package/src/runtime/publisher.js +0 -328
- package/src/server-old.ts +0 -988
- package/src/server.ts +0 -2059
- package/src/service/annotations.spec.ts +0 -118
- package/src/service/annotations.ts +0 -91
- package/src/service/authorize.spec.ts +0 -154
- package/src/service/authorize.ts +0 -498
- package/src/service/authorize_integration.spec.ts +0 -2041
- package/src/service/build_plan.spec.ts +0 -685
- package/src/service/build_plan.ts +0 -797
- package/src/service/compile_authorize.spec.ts +0 -85
- package/src/service/compile_fragment_techniques.spec.ts +0 -156
- package/src/service/connection.spec.ts +0 -2360
- package/src/service/connection.ts +0 -2187
- package/src/service/connection_config.spec.ts +0 -734
- package/src/service/connection_config.ts +0 -899
- package/src/service/connection_federation.spec.ts +0 -184
- package/src/service/connection_fingerprint.spec.ts +0 -102
- package/src/service/connection_service.spec.ts +0 -711
- package/src/service/connection_service.ts +0 -291
- package/src/service/cron_evaluator.spec.ts +0 -79
- package/src/service/cron_evaluator.ts +0 -105
- package/src/service/db_utils.spec.ts +0 -1085
- package/src/service/db_utils.ts +0 -1502
- package/src/service/duckdb_instance_isolation.spec.ts +0 -137
- package/src/service/ducklake_lazy_attach.spec.ts +0 -110
- package/src/service/embedding_provider.spec.ts +0 -329
- package/src/service/embedding_provider.ts +0 -236
- package/src/service/environment.ts +0 -2191
- package/src/service/environment_admission.spec.ts +0 -344
- package/src/service/environment_compile.spec.ts +0 -197
- package/src/service/environment_store.spec.ts +0 -2274
- package/src/service/environment_store.ts +0 -2434
- package/src/service/environment_store_anchoring.spec.ts +0 -107
- package/src/service/environment_store_clone.spec.ts +0 -350
- package/src/service/explore_visibility.spec.ts +0 -434
- package/src/service/exports_probe.spec.ts +0 -107
- package/src/service/extension_fetch_policy.spec.ts +0 -256
- package/src/service/filter.spec.ts +0 -458
- package/src/service/filter.ts +0 -341
- package/src/service/filter_bypass.spec.ts +0 -418
- package/src/service/filter_integration.spec.ts +0 -1004
- package/src/service/freshness.spec.ts +0 -183
- package/src/service/freshness.ts +0 -112
- package/src/service/gcs_s3_utils.ts +0 -427
- package/src/service/given.ts +0 -105
- package/src/service/givens_integration.spec.ts +0 -219
- package/src/service/manifest_loader.spec.ts +0 -189
- package/src/service/manifest_loader.ts +0 -156
- package/src/service/materialization_build_session.spec.ts +0 -435
- package/src/service/materialization_build_session.ts +0 -681
- package/src/service/materialization_config_validation.spec.ts +0 -99
- package/src/service/materialization_config_validation.ts +0 -120
- package/src/service/materialization_eligibility.spec.ts +0 -158
- package/src/service/materialization_eligibility.ts +0 -305
- package/src/service/materialization_schedule_surface.spec.ts +0 -256
- package/src/service/materialization_scheduler.spec.ts +0 -289
- package/src/service/materialization_scheduler.ts +0 -304
- package/src/service/materialization_serve_transform.spec.ts +0 -1003
- package/src/service/materialization_serve_transform.ts +0 -779
- package/src/service/materialization_service.spec.ts +0 -2373
- package/src/service/materialization_service.ts +0 -2342
- package/src/service/materialization_test_fixtures.ts +0 -249
- package/src/service/model.spec.ts +0 -1157
- package/src/service/model.ts +0 -3370
- package/src/service/model_limits.spec.ts +0 -209
- package/src/service/model_limits.ts +0 -131
- package/src/service/model_storage_serve.spec.ts +0 -193
- package/src/service/model_storage_serve_joins.spec.ts +0 -193
- package/src/service/package.spec.ts +0 -612
- package/src/service/package.ts +0 -1799
- package/src/service/package_manifest.spec.ts +0 -230
- package/src/service/package_manifest.ts +0 -241
- package/src/service/package_memory_governor.spec.ts +0 -173
- package/src/service/package_memory_governor.ts +0 -233
- package/src/service/package_quote_bound_tables.spec.ts +0 -171
- package/src/service/package_race.spec.ts +0 -208
- package/src/service/package_reload_safety.spec.ts +0 -193
- package/src/service/package_rollback.spec.ts +0 -190
- package/src/service/package_worker_path.spec.ts +0 -645
- package/src/service/path_injection.spec.ts +0 -39
- package/src/service/persist_annotation_validation.spec.ts +0 -89
- package/src/service/persist_annotation_validation.ts +0 -59
- package/src/service/persistence_policy.spec.ts +0 -374
- package/src/service/proxy.spec.ts +0 -414
- package/src/service/proxy.ts +0 -248
- package/src/service/query_boundary.spec.ts +0 -470
- package/src/service/query_metadata.spec.ts +0 -408
- package/src/service/query_metadata.ts +0 -492
- package/src/service/query_metadata_identity.spec.ts +0 -149
- package/src/service/query_text.spec.ts +0 -79
- package/src/service/query_text.ts +0 -43
- package/src/service/quoting.spec.ts +0 -135
- package/src/service/quoting.ts +0 -78
- package/src/service/resolve_environment.ts +0 -15
- package/src/service/restricted_mode.spec.ts +0 -299
- package/src/service/source_extraction.ts +0 -225
- package/src/service/theme_store.ts +0 -199
- package/src/storage/DatabaseInterface.ts +0 -198
- package/src/storage/StorageManager.ts +0 -113
- package/src/storage/duckdb/ConnectionRepository.ts +0 -143
- package/src/storage/duckdb/DuckDBConnection.ts +0 -123
- package/src/storage/duckdb/DuckDBRepository.ts +0 -208
- package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
- package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
- package/src/storage/duckdb/MaterializationRepository.ts +0 -336
- package/src/storage/duckdb/PackageRepository.ts +0 -135
- package/src/storage/duckdb/schema.ts +0 -229
- package/src/stream_helpers.spec.ts +0 -280
- package/src/stream_helpers.ts +0 -162
- package/src/telemetry.ts +0 -20
- package/src/test_helpers/metrics_harness.ts +0 -166
- package/src/theme_key_parity.spec.ts +0 -57
- package/src/utils.ts +0 -29
- package/tests/fixtures/authorize-compile/model.malloy +0 -9
- package/tests/fixtures/authorize-compile/publisher.json +0 -4
- package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
- package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
- package/tests/fixtures/html-pages-test/data.csv +0 -3
- package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
- package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
- package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
- package/tests/fixtures/html-pages-test/public/data.json +0 -1
- package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
- package/tests/fixtures/html-pages-test/public/index.html +0 -9
- package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
- package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
- package/tests/fixtures/html-pages-test/public/slides.html +0 -12
- package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
- package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
- package/tests/fixtures/html-pages-test/publisher.json +0 -5
- package/tests/fixtures/html-pages-test/report.malloy +0 -1
- package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
- package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
- package/tests/fixtures/persist-multi-level/publisher.json +0 -5
- package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
- package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
- package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
- package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
- package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
- package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
- package/tests/fixtures/persist-test/data/orders.csv +0 -5
- package/tests/fixtures/persist-test/persist_test.malloy +0 -11
- package/tests/fixtures/persist-test/publisher.json +0 -5
- package/tests/fixtures/publisher.config.json +0 -15
- package/tests/fixtures/query-givens/data/orders.csv +0 -7
- package/tests/fixtures/query-givens/model.malloy +0 -34
- package/tests/fixtures/query-givens/publisher.json +0 -5
- package/tests/fixtures/xlsx/database.xlsx +0 -0
- package/tests/harness/mcp_test_setup.ts +0 -332
- package/tests/harness/rest_e2e.ts +0 -68
- package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
- package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
- package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
- package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
- package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
- package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
- package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
- package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
- package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
- package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
- package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
- package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
- package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
- package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
- package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
- package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
- package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
- package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
- package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
- package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
- package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
- package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
- package/tests/unit/controller/theme.controller.spec.ts +0 -141
- package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
- package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
- package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
- package/tests/unit/duckdb/repositories.test.ts +0 -208
- package/tests/unit/ducklake/ducklake.test.ts +0 -952
- package/tests/unit/service/theme_store.spec.ts +0 -139
- package/tsconfig.json +0 -32
|
@@ -1,2360 +0,0 @@
|
|
|
1
|
-
import { DuckDBConnection } from "@malloydata/db-duckdb";
|
|
2
|
-
import {
|
|
3
|
-
afterAll,
|
|
4
|
-
afterEach,
|
|
5
|
-
beforeEach,
|
|
6
|
-
describe,
|
|
7
|
-
expect,
|
|
8
|
-
it,
|
|
9
|
-
} from "bun:test";
|
|
10
|
-
import fs from "fs/promises";
|
|
11
|
-
import os from "os";
|
|
12
|
-
import path from "path";
|
|
13
|
-
import sinon from "sinon";
|
|
14
|
-
import { components } from "../api";
|
|
15
|
-
import {
|
|
16
|
-
attachDuckLakeReadWrite,
|
|
17
|
-
buildProxiedSslQuery,
|
|
18
|
-
createEnvironmentConnections,
|
|
19
|
-
resolveProxiedTls,
|
|
20
|
-
testConnectionConfig,
|
|
21
|
-
} from "./connection";
|
|
22
|
-
import { assembleEnvironmentConnections } from "./connection_config";
|
|
23
|
-
import { UnsupportedCatalogFormatError } from "../errors";
|
|
24
|
-
import { EnvironmentStore } from "./environment_store";
|
|
25
|
-
|
|
26
|
-
type ApiConnection = components["schemas"]["Connection"];
|
|
27
|
-
type AttachedDatabase = components["schemas"]["AttachedDatabase"];
|
|
28
|
-
|
|
29
|
-
const hasPostgresCredentials = () =>
|
|
30
|
-
!!(
|
|
31
|
-
process.env.POSTGRES_TEST_HOST &&
|
|
32
|
-
process.env.POSTGRES_TEST_USER &&
|
|
33
|
-
process.env.POSTGRES_TEST_PASSWORD
|
|
34
|
-
);
|
|
35
|
-
|
|
36
|
-
const hasBigQueryCredentials = () =>
|
|
37
|
-
!!(
|
|
38
|
-
process.env.GOOGLE_APPLICATION_CREDENTIALS &&
|
|
39
|
-
process.env.BIGQUERY_TEST_PROJECT_ID
|
|
40
|
-
);
|
|
41
|
-
|
|
42
|
-
const hasSnowflakeCredentials = () =>
|
|
43
|
-
!!(
|
|
44
|
-
process.env.SNOWFLAKE_TEST_ACCOUNT &&
|
|
45
|
-
process.env.SNOWFLAKE_TEST_USER &&
|
|
46
|
-
process.env.SNOWFLAKE_TEST_PASSWORD &&
|
|
47
|
-
process.env.SNOWFLAKE_TEST_WAREHOUSE
|
|
48
|
-
);
|
|
49
|
-
|
|
50
|
-
const hasS3Credentials = () =>
|
|
51
|
-
!!(
|
|
52
|
-
process.env.S3_TEST_ACCESS_KEY_ID && process.env.S3_TEST_SECRET_ACCESS_KEY
|
|
53
|
-
);
|
|
54
|
-
|
|
55
|
-
const hasGCSCredentials = () =>
|
|
56
|
-
!!(process.env.GCS_TEST_KEY_ID && process.env.GCS_TEST_SECRET);
|
|
57
|
-
|
|
58
|
-
const readBigQueryServiceAccountJson = async (): Promise<string> =>
|
|
59
|
-
fs.readFile(process.env.GOOGLE_APPLICATION_CREDENTIALS!, "utf-8");
|
|
60
|
-
|
|
61
|
-
// `BIGQUERY_PUBLIC_DATA_*` env vars are populated by the
|
|
62
|
-
// `Setup BigQuery public-data credentials` step in
|
|
63
|
-
// `.github/workflows/connection-integration-tests.yml`. Kept separate
|
|
64
|
-
// from the existing `BIGQUERY_TEST_*` vars (which point at the
|
|
65
|
-
// org-scoped BQ_PRESTO_TRINO_KEY service account) so the two SAs can
|
|
66
|
-
// coexist without one overwriting the other.
|
|
67
|
-
const hasPublicDataBigQueryCredentials = () =>
|
|
68
|
-
!!(
|
|
69
|
-
process.env.BIGQUERY_PUBLIC_DATA_CREDENTIALS &&
|
|
70
|
-
process.env.BIGQUERY_PUBLIC_DATA_PROJECT_ID
|
|
71
|
-
);
|
|
72
|
-
|
|
73
|
-
const readPublicDataBigQueryServiceAccountJson = async (): Promise<string> =>
|
|
74
|
-
fs.readFile(process.env.BIGQUERY_PUBLIC_DATA_CREDENTIALS!, "utf-8");
|
|
75
|
-
|
|
76
|
-
describe("connection integration tests", () => {
|
|
77
|
-
const testEnvironmentPath = path.join(
|
|
78
|
-
process.cwd(),
|
|
79
|
-
"test-environment-connections",
|
|
80
|
-
);
|
|
81
|
-
let createdConnections: DuckDBConnection[] = [];
|
|
82
|
-
|
|
83
|
-
beforeEach(async () => {
|
|
84
|
-
await fs.mkdir(testEnvironmentPath, { recursive: true });
|
|
85
|
-
});
|
|
86
|
-
|
|
87
|
-
afterEach(async () => {
|
|
88
|
-
sinon.restore();
|
|
89
|
-
|
|
90
|
-
for (const conn of createdConnections) {
|
|
91
|
-
try {
|
|
92
|
-
await conn.close();
|
|
93
|
-
} catch (error) {
|
|
94
|
-
console.warn("Error closing connection:", error);
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
createdConnections = [];
|
|
98
|
-
|
|
99
|
-
const maxRetries = 5;
|
|
100
|
-
const delay = 100;
|
|
101
|
-
let lastError: unknown;
|
|
102
|
-
|
|
103
|
-
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
104
|
-
try {
|
|
105
|
-
await fs.rm(testEnvironmentPath, { recursive: true, force: true });
|
|
106
|
-
return;
|
|
107
|
-
} catch (error) {
|
|
108
|
-
lastError = error;
|
|
109
|
-
const errnoError = error as NodeJS.ErrnoException;
|
|
110
|
-
if (errnoError.code !== "EBUSY") throw error;
|
|
111
|
-
if (attempt < maxRetries - 1) {
|
|
112
|
-
await new Promise((resolve) =>
|
|
113
|
-
setTimeout(resolve, delay * Math.pow(2, attempt)),
|
|
114
|
-
);
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
if ((lastError as NodeJS.ErrnoException).code !== "EBUSY") {
|
|
120
|
-
throw lastError;
|
|
121
|
-
}
|
|
122
|
-
});
|
|
123
|
-
|
|
124
|
-
describe("createEnvironmentConnections", () => {
|
|
125
|
-
describe("DuckDB with PostgreSQL attachment", () => {
|
|
126
|
-
it(
|
|
127
|
-
"should create DuckDB connection with attached PostgreSQL database",
|
|
128
|
-
async () => {
|
|
129
|
-
if (!hasPostgresCredentials()) {
|
|
130
|
-
console.log(
|
|
131
|
-
"Skipping: PostgreSQL credentials not configured",
|
|
132
|
-
);
|
|
133
|
-
return;
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
const postgresAttachment: AttachedDatabase = {
|
|
137
|
-
name: "pg_test",
|
|
138
|
-
type: "postgres",
|
|
139
|
-
postgresConnection: {
|
|
140
|
-
host: process.env.POSTGRES_TEST_HOST,
|
|
141
|
-
port: parseInt(process.env.POSTGRES_TEST_PORT || "5432"),
|
|
142
|
-
userName: process.env.POSTGRES_TEST_USER!,
|
|
143
|
-
password: process.env.POSTGRES_TEST_PASSWORD!,
|
|
144
|
-
databaseName: process.env.POSTGRES_TEST_DATABASE,
|
|
145
|
-
},
|
|
146
|
-
};
|
|
147
|
-
|
|
148
|
-
const duckdbConnection: ApiConnection = {
|
|
149
|
-
name: "duckdb_with_postgres",
|
|
150
|
-
type: "duckdb",
|
|
151
|
-
duckdbConnection: { attachedDatabases: [postgresAttachment] },
|
|
152
|
-
};
|
|
153
|
-
|
|
154
|
-
const { malloyConnections, apiConnections } =
|
|
155
|
-
await createEnvironmentConnections(
|
|
156
|
-
[duckdbConnection],
|
|
157
|
-
testEnvironmentPath,
|
|
158
|
-
);
|
|
159
|
-
|
|
160
|
-
expect(malloyConnections.size).toBe(1);
|
|
161
|
-
expect(apiConnections.length).toBe(1);
|
|
162
|
-
|
|
163
|
-
const connection = malloyConnections.get(
|
|
164
|
-
"duckdb_with_postgres",
|
|
165
|
-
) as DuckDBConnection;
|
|
166
|
-
expect(connection).toBeDefined();
|
|
167
|
-
createdConnections.push(connection);
|
|
168
|
-
|
|
169
|
-
const databases = await connection.runSQL("SHOW DATABASES");
|
|
170
|
-
const dbNames = databases.rows.map(
|
|
171
|
-
(row) => Object.values(row)[0],
|
|
172
|
-
);
|
|
173
|
-
expect(dbNames).toContain("pg_test");
|
|
174
|
-
|
|
175
|
-
const result = await connection.runSQL(
|
|
176
|
-
"SELECT 1 as test_value FROM pg_test.information_schema.tables LIMIT 1",
|
|
177
|
-
);
|
|
178
|
-
expect(result.rows).toBeDefined();
|
|
179
|
-
},
|
|
180
|
-
{ timeout: 30000 },
|
|
181
|
-
);
|
|
182
|
-
|
|
183
|
-
it(
|
|
184
|
-
"should list schemas from attached PostgreSQL database",
|
|
185
|
-
async () => {
|
|
186
|
-
if (!hasPostgresCredentials()) {
|
|
187
|
-
console.log(
|
|
188
|
-
"Skipping: PostgreSQL credentials not configured",
|
|
189
|
-
);
|
|
190
|
-
return;
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
const duckdbConnection: ApiConnection = {
|
|
194
|
-
name: "duckdb_pg_schemas",
|
|
195
|
-
type: "duckdb",
|
|
196
|
-
duckdbConnection: {
|
|
197
|
-
attachedDatabases: [
|
|
198
|
-
{
|
|
199
|
-
name: "pg_schemas",
|
|
200
|
-
type: "postgres",
|
|
201
|
-
postgresConnection: {
|
|
202
|
-
host: process.env.POSTGRES_TEST_HOST,
|
|
203
|
-
port: parseInt(
|
|
204
|
-
process.env.POSTGRES_TEST_PORT || "5432",
|
|
205
|
-
),
|
|
206
|
-
userName: process.env.POSTGRES_TEST_USER!,
|
|
207
|
-
password: process.env.POSTGRES_TEST_PASSWORD!,
|
|
208
|
-
databaseName: process.env.POSTGRES_TEST_DATABASE,
|
|
209
|
-
},
|
|
210
|
-
},
|
|
211
|
-
],
|
|
212
|
-
},
|
|
213
|
-
};
|
|
214
|
-
|
|
215
|
-
const { malloyConnections } = await createEnvironmentConnections(
|
|
216
|
-
[duckdbConnection],
|
|
217
|
-
testEnvironmentPath,
|
|
218
|
-
);
|
|
219
|
-
|
|
220
|
-
const connection = malloyConnections.get(
|
|
221
|
-
"duckdb_pg_schemas",
|
|
222
|
-
) as DuckDBConnection;
|
|
223
|
-
createdConnections.push(connection);
|
|
224
|
-
|
|
225
|
-
// listSchemas equivalent
|
|
226
|
-
const result = await connection.runSQL(
|
|
227
|
-
"SELECT schema_name FROM pg_schemas.information_schema.schemata ORDER BY schema_name",
|
|
228
|
-
);
|
|
229
|
-
expect(result.rows.length).toBeGreaterThan(0);
|
|
230
|
-
const schemaNames = result.rows.map(
|
|
231
|
-
(row) => Object.values(row)[0] as string,
|
|
232
|
-
);
|
|
233
|
-
expect(schemaNames).toContain("public");
|
|
234
|
-
},
|
|
235
|
-
{ timeout: 30000 },
|
|
236
|
-
);
|
|
237
|
-
|
|
238
|
-
it(
|
|
239
|
-
"should list tables from attached PostgreSQL database",
|
|
240
|
-
async () => {
|
|
241
|
-
if (!hasPostgresCredentials()) {
|
|
242
|
-
console.log(
|
|
243
|
-
"Skipping: PostgreSQL credentials not configured",
|
|
244
|
-
);
|
|
245
|
-
return;
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
const duckdbConnection: ApiConnection = {
|
|
249
|
-
name: "duckdb_pg_tables",
|
|
250
|
-
type: "duckdb",
|
|
251
|
-
duckdbConnection: {
|
|
252
|
-
attachedDatabases: [
|
|
253
|
-
{
|
|
254
|
-
name: "pg_tables_db",
|
|
255
|
-
type: "postgres",
|
|
256
|
-
postgresConnection: {
|
|
257
|
-
host: process.env.POSTGRES_TEST_HOST,
|
|
258
|
-
port: parseInt(
|
|
259
|
-
process.env.POSTGRES_TEST_PORT || "5432",
|
|
260
|
-
),
|
|
261
|
-
userName: process.env.POSTGRES_TEST_USER!,
|
|
262
|
-
password: process.env.POSTGRES_TEST_PASSWORD!,
|
|
263
|
-
databaseName: process.env.POSTGRES_TEST_DATABASE,
|
|
264
|
-
},
|
|
265
|
-
},
|
|
266
|
-
],
|
|
267
|
-
},
|
|
268
|
-
};
|
|
269
|
-
|
|
270
|
-
const { malloyConnections } = await createEnvironmentConnections(
|
|
271
|
-
[duckdbConnection],
|
|
272
|
-
testEnvironmentPath,
|
|
273
|
-
);
|
|
274
|
-
|
|
275
|
-
const connection = malloyConnections.get(
|
|
276
|
-
"duckdb_pg_tables",
|
|
277
|
-
) as DuckDBConnection;
|
|
278
|
-
createdConnections.push(connection);
|
|
279
|
-
|
|
280
|
-
// listTables equivalent
|
|
281
|
-
const result = await connection.runSQL(`
|
|
282
|
-
SELECT table_schema, table_name, table_type
|
|
283
|
-
FROM pg_tables_db.information_schema.tables
|
|
284
|
-
WHERE table_schema NOT IN ('pg_catalog', 'information_schema')
|
|
285
|
-
ORDER BY table_schema, table_name
|
|
286
|
-
`);
|
|
287
|
-
expect(result.rows).toBeDefined();
|
|
288
|
-
},
|
|
289
|
-
{ timeout: 30000 },
|
|
290
|
-
);
|
|
291
|
-
|
|
292
|
-
it(
|
|
293
|
-
"should get table columns from attached PostgreSQL database",
|
|
294
|
-
async () => {
|
|
295
|
-
if (!hasPostgresCredentials()) {
|
|
296
|
-
console.log(
|
|
297
|
-
"Skipping: PostgreSQL credentials not configured",
|
|
298
|
-
);
|
|
299
|
-
return;
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
const duckdbConnection: ApiConnection = {
|
|
303
|
-
name: "duckdb_pg_cols",
|
|
304
|
-
type: "duckdb",
|
|
305
|
-
duckdbConnection: {
|
|
306
|
-
attachedDatabases: [
|
|
307
|
-
{
|
|
308
|
-
name: "pg_cols",
|
|
309
|
-
type: "postgres",
|
|
310
|
-
postgresConnection: {
|
|
311
|
-
host: process.env.POSTGRES_TEST_HOST,
|
|
312
|
-
port: parseInt(
|
|
313
|
-
process.env.POSTGRES_TEST_PORT || "5432",
|
|
314
|
-
),
|
|
315
|
-
userName: process.env.POSTGRES_TEST_USER!,
|
|
316
|
-
password: process.env.POSTGRES_TEST_PASSWORD!,
|
|
317
|
-
databaseName: process.env.POSTGRES_TEST_DATABASE,
|
|
318
|
-
},
|
|
319
|
-
},
|
|
320
|
-
],
|
|
321
|
-
},
|
|
322
|
-
};
|
|
323
|
-
|
|
324
|
-
const { malloyConnections } = await createEnvironmentConnections(
|
|
325
|
-
[duckdbConnection],
|
|
326
|
-
testEnvironmentPath,
|
|
327
|
-
);
|
|
328
|
-
|
|
329
|
-
const connection = malloyConnections.get(
|
|
330
|
-
"duckdb_pg_cols",
|
|
331
|
-
) as DuckDBConnection;
|
|
332
|
-
createdConnections.push(connection);
|
|
333
|
-
|
|
334
|
-
const result = await connection.runSQL(`
|
|
335
|
-
SELECT column_name, data_type, is_nullable
|
|
336
|
-
FROM pg_cols.information_schema.columns
|
|
337
|
-
WHERE table_schema = 'information_schema'
|
|
338
|
-
AND table_name = 'tables'
|
|
339
|
-
ORDER BY ordinal_position
|
|
340
|
-
`);
|
|
341
|
-
expect(result.rows.length).toBeGreaterThan(0);
|
|
342
|
-
const columnNames = result.rows.map(
|
|
343
|
-
(row) => (row as Record<string, unknown>)["column_name"],
|
|
344
|
-
);
|
|
345
|
-
expect(columnNames).toContain("table_name");
|
|
346
|
-
expect(columnNames).toContain("table_schema");
|
|
347
|
-
},
|
|
348
|
-
{ timeout: 30000 },
|
|
349
|
-
);
|
|
350
|
-
|
|
351
|
-
it(
|
|
352
|
-
"should handle PostgreSQL connection string format",
|
|
353
|
-
async () => {
|
|
354
|
-
if (!hasPostgresCredentials()) {
|
|
355
|
-
console.log(
|
|
356
|
-
"Skipping: PostgreSQL credentials not configured",
|
|
357
|
-
);
|
|
358
|
-
return;
|
|
359
|
-
}
|
|
360
|
-
|
|
361
|
-
const connectionString = `host=${process.env.POSTGRES_TEST_HOST} port=${process.env.POSTGRES_TEST_PORT || "5432"} dbname=${process.env.POSTGRES_TEST_DATABASE} user=${process.env.POSTGRES_TEST_USER} password=${process.env.POSTGRES_TEST_PASSWORD}`;
|
|
362
|
-
|
|
363
|
-
const duckdbConnection: ApiConnection = {
|
|
364
|
-
name: "duckdb_pg_string",
|
|
365
|
-
type: "duckdb",
|
|
366
|
-
duckdbConnection: {
|
|
367
|
-
attachedDatabases: [
|
|
368
|
-
{
|
|
369
|
-
name: "pg_conn_string",
|
|
370
|
-
type: "postgres",
|
|
371
|
-
postgresConnection: { connectionString },
|
|
372
|
-
},
|
|
373
|
-
],
|
|
374
|
-
},
|
|
375
|
-
};
|
|
376
|
-
|
|
377
|
-
const { malloyConnections } = await createEnvironmentConnections(
|
|
378
|
-
[duckdbConnection],
|
|
379
|
-
testEnvironmentPath,
|
|
380
|
-
);
|
|
381
|
-
|
|
382
|
-
const connection = malloyConnections.get(
|
|
383
|
-
"duckdb_pg_string",
|
|
384
|
-
) as DuckDBConnection;
|
|
385
|
-
createdConnections.push(connection);
|
|
386
|
-
|
|
387
|
-
const databases = await connection.runSQL("SHOW DATABASES");
|
|
388
|
-
const dbNames = databases.rows.map(
|
|
389
|
-
(row) => Object.values(row)[0],
|
|
390
|
-
);
|
|
391
|
-
expect(dbNames).toContain("pg_conn_string");
|
|
392
|
-
},
|
|
393
|
-
{ timeout: 30000 },
|
|
394
|
-
);
|
|
395
|
-
});
|
|
396
|
-
|
|
397
|
-
describe("DuckDB with BigQuery attachment", () => {
|
|
398
|
-
it(
|
|
399
|
-
"should create DuckDB connection with attached BigQuery database",
|
|
400
|
-
async () => {
|
|
401
|
-
if (!hasBigQueryCredentials()) {
|
|
402
|
-
console.log("Skipping: BigQuery credentials not configured");
|
|
403
|
-
return;
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
const serviceAccountJson =
|
|
407
|
-
await readBigQueryServiceAccountJson();
|
|
408
|
-
|
|
409
|
-
const duckdbConnection: ApiConnection = {
|
|
410
|
-
name: "duckdb_with_bigquery",
|
|
411
|
-
type: "duckdb",
|
|
412
|
-
duckdbConnection: {
|
|
413
|
-
attachedDatabases: [
|
|
414
|
-
{
|
|
415
|
-
name: "bq_test",
|
|
416
|
-
type: "bigquery",
|
|
417
|
-
bigqueryConnection: {
|
|
418
|
-
defaultProjectId:
|
|
419
|
-
process.env.BIGQUERY_TEST_PROJECT_ID!,
|
|
420
|
-
serviceAccountKeyJson: serviceAccountJson,
|
|
421
|
-
},
|
|
422
|
-
},
|
|
423
|
-
],
|
|
424
|
-
},
|
|
425
|
-
};
|
|
426
|
-
|
|
427
|
-
const { malloyConnections } = await createEnvironmentConnections(
|
|
428
|
-
[duckdbConnection],
|
|
429
|
-
testEnvironmentPath,
|
|
430
|
-
);
|
|
431
|
-
|
|
432
|
-
const connection = malloyConnections.get(
|
|
433
|
-
"duckdb_with_bigquery",
|
|
434
|
-
) as DuckDBConnection;
|
|
435
|
-
expect(connection).toBeDefined();
|
|
436
|
-
createdConnections.push(connection);
|
|
437
|
-
|
|
438
|
-
const databases = await connection.runSQL("SHOW DATABASES");
|
|
439
|
-
const dbNames = databases.rows.map(
|
|
440
|
-
(row) => Object.values(row)[0],
|
|
441
|
-
);
|
|
442
|
-
expect(dbNames).toContain("bq_test");
|
|
443
|
-
},
|
|
444
|
-
{ timeout: 60000 },
|
|
445
|
-
);
|
|
446
|
-
|
|
447
|
-
it(
|
|
448
|
-
"should list datasets (schemas) from attached BigQuery database",
|
|
449
|
-
async () => {
|
|
450
|
-
if (
|
|
451
|
-
!hasBigQueryCredentials() ||
|
|
452
|
-
!process.env.BIGQUERY_TEST_DATASET
|
|
453
|
-
) {
|
|
454
|
-
console.log(
|
|
455
|
-
"Skipping: GOOGLE_APPLICATION_CREDENTIALS, BIGQUERY_TEST_PROJECT_ID, or BIGQUERY_TEST_DATASET not configured",
|
|
456
|
-
);
|
|
457
|
-
return;
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
const serviceAccountJson =
|
|
461
|
-
await readBigQueryServiceAccountJson();
|
|
462
|
-
|
|
463
|
-
const duckdbConnection: ApiConnection = {
|
|
464
|
-
name: "duckdb_bq_schemas",
|
|
465
|
-
type: "duckdb",
|
|
466
|
-
duckdbConnection: {
|
|
467
|
-
attachedDatabases: [
|
|
468
|
-
{
|
|
469
|
-
name: "bq_schemas",
|
|
470
|
-
type: "bigquery",
|
|
471
|
-
bigqueryConnection: {
|
|
472
|
-
defaultProjectId:
|
|
473
|
-
process.env.BIGQUERY_TEST_PROJECT_ID!,
|
|
474
|
-
serviceAccountKeyJson: serviceAccountJson,
|
|
475
|
-
},
|
|
476
|
-
},
|
|
477
|
-
],
|
|
478
|
-
},
|
|
479
|
-
};
|
|
480
|
-
|
|
481
|
-
const { malloyConnections } = await createEnvironmentConnections(
|
|
482
|
-
[duckdbConnection],
|
|
483
|
-
testEnvironmentPath,
|
|
484
|
-
);
|
|
485
|
-
|
|
486
|
-
const connection = malloyConnections.get(
|
|
487
|
-
"duckdb_bq_schemas",
|
|
488
|
-
) as DuckDBConnection;
|
|
489
|
-
createdConnections.push(connection);
|
|
490
|
-
|
|
491
|
-
const result = await connection.runSQL(
|
|
492
|
-
"SELECT schema_name FROM bq_schemas.information_schema.schemata",
|
|
493
|
-
);
|
|
494
|
-
expect(result.rows.length).toBeGreaterThan(0);
|
|
495
|
-
const datasetNames = result.rows.map(
|
|
496
|
-
(row) => Object.values(row)[0] as string,
|
|
497
|
-
);
|
|
498
|
-
expect(datasetNames).toContain(
|
|
499
|
-
process.env.BIGQUERY_TEST_DATASET!,
|
|
500
|
-
);
|
|
501
|
-
},
|
|
502
|
-
{ timeout: 60000 },
|
|
503
|
-
);
|
|
504
|
-
|
|
505
|
-
it(
|
|
506
|
-
"should list tables from attached BigQuery dataset",
|
|
507
|
-
async () => {
|
|
508
|
-
if (
|
|
509
|
-
!hasBigQueryCredentials() ||
|
|
510
|
-
!process.env.BIGQUERY_TEST_DATASET
|
|
511
|
-
) {
|
|
512
|
-
console.log(
|
|
513
|
-
"Skipping: GOOGLE_APPLICATION_CREDENTIALS, BIGQUERY_TEST_PROJECT_ID, or BIGQUERY_TEST_DATASET not configured",
|
|
514
|
-
);
|
|
515
|
-
return;
|
|
516
|
-
}
|
|
517
|
-
|
|
518
|
-
const serviceAccountJson =
|
|
519
|
-
await readBigQueryServiceAccountJson();
|
|
520
|
-
|
|
521
|
-
const duckdbConnection: ApiConnection = {
|
|
522
|
-
name: "duckdb_bq_tables",
|
|
523
|
-
type: "duckdb",
|
|
524
|
-
duckdbConnection: {
|
|
525
|
-
attachedDatabases: [
|
|
526
|
-
{
|
|
527
|
-
name: "bq_tables",
|
|
528
|
-
type: "bigquery",
|
|
529
|
-
bigqueryConnection: {
|
|
530
|
-
defaultProjectId:
|
|
531
|
-
process.env.BIGQUERY_TEST_PROJECT_ID!,
|
|
532
|
-
serviceAccountKeyJson: serviceAccountJson,
|
|
533
|
-
},
|
|
534
|
-
},
|
|
535
|
-
],
|
|
536
|
-
},
|
|
537
|
-
};
|
|
538
|
-
|
|
539
|
-
const { malloyConnections } = await createEnvironmentConnections(
|
|
540
|
-
[duckdbConnection],
|
|
541
|
-
testEnvironmentPath,
|
|
542
|
-
);
|
|
543
|
-
|
|
544
|
-
const connection = malloyConnections.get(
|
|
545
|
-
"duckdb_bq_tables",
|
|
546
|
-
) as DuckDBConnection;
|
|
547
|
-
createdConnections.push(connection);
|
|
548
|
-
|
|
549
|
-
const result = await connection.runSQL(`
|
|
550
|
-
SELECT table_name, table_type
|
|
551
|
-
FROM bq_tables.${process.env.BIGQUERY_TEST_DATASET!}.INFORMATION_SCHEMA.TABLES
|
|
552
|
-
ORDER BY table_name
|
|
553
|
-
`);
|
|
554
|
-
expect(result.rows.length).toBeGreaterThan(0);
|
|
555
|
-
},
|
|
556
|
-
{ timeout: 60000 },
|
|
557
|
-
);
|
|
558
|
-
|
|
559
|
-
it(
|
|
560
|
-
"should get table columns from attached BigQuery table",
|
|
561
|
-
async () => {
|
|
562
|
-
if (
|
|
563
|
-
!hasBigQueryCredentials() ||
|
|
564
|
-
!process.env.BIGQUERY_TEST_TABLE
|
|
565
|
-
) {
|
|
566
|
-
console.log(
|
|
567
|
-
"Skipping: GOOGLE_APPLICATION_CREDENTIALS, BIGQUERY_TEST_PROJECT_ID, or BIGQUERY_TEST_TABLE not configured",
|
|
568
|
-
);
|
|
569
|
-
return;
|
|
570
|
-
}
|
|
571
|
-
|
|
572
|
-
const serviceAccountJson =
|
|
573
|
-
await readBigQueryServiceAccountJson();
|
|
574
|
-
const [dataset, table] =
|
|
575
|
-
process.env.BIGQUERY_TEST_TABLE!.split(".");
|
|
576
|
-
|
|
577
|
-
const duckdbConnection: ApiConnection = {
|
|
578
|
-
name: "duckdb_bq_cols",
|
|
579
|
-
type: "duckdb",
|
|
580
|
-
duckdbConnection: {
|
|
581
|
-
attachedDatabases: [
|
|
582
|
-
{
|
|
583
|
-
name: "bq_cols",
|
|
584
|
-
type: "bigquery",
|
|
585
|
-
bigqueryConnection: {
|
|
586
|
-
defaultProjectId:
|
|
587
|
-
process.env.BIGQUERY_TEST_PROJECT_ID!,
|
|
588
|
-
serviceAccountKeyJson: serviceAccountJson,
|
|
589
|
-
},
|
|
590
|
-
},
|
|
591
|
-
],
|
|
592
|
-
},
|
|
593
|
-
};
|
|
594
|
-
|
|
595
|
-
const { malloyConnections } = await createEnvironmentConnections(
|
|
596
|
-
[duckdbConnection],
|
|
597
|
-
testEnvironmentPath,
|
|
598
|
-
);
|
|
599
|
-
|
|
600
|
-
const connection = malloyConnections.get(
|
|
601
|
-
"duckdb_bq_cols",
|
|
602
|
-
) as DuckDBConnection;
|
|
603
|
-
createdConnections.push(connection);
|
|
604
|
-
|
|
605
|
-
// getTable equivalent — fetch column metadata
|
|
606
|
-
const result = await connection.runSQL(`
|
|
607
|
-
SELECT column_name, data_type, is_nullable
|
|
608
|
-
FROM bq_cols.${dataset}.INFORMATION_SCHEMA.COLUMNS
|
|
609
|
-
WHERE table_name = '${table}'
|
|
610
|
-
ORDER BY ordinal_position
|
|
611
|
-
`);
|
|
612
|
-
expect(result.rows.length).toBeGreaterThan(0);
|
|
613
|
-
},
|
|
614
|
-
{ timeout: 60000 },
|
|
615
|
-
);
|
|
616
|
-
|
|
617
|
-
it(
|
|
618
|
-
"should query data from attached BigQuery table",
|
|
619
|
-
async () => {
|
|
620
|
-
if (
|
|
621
|
-
!hasBigQueryCredentials() ||
|
|
622
|
-
!process.env.BIGQUERY_TEST_TABLE
|
|
623
|
-
) {
|
|
624
|
-
console.log(
|
|
625
|
-
"Skipping: GOOGLE_APPLICATION_CREDENTIALS, BIGQUERY_TEST_PROJECT_ID, or BIGQUERY_TEST_TABLE not configured",
|
|
626
|
-
);
|
|
627
|
-
return;
|
|
628
|
-
}
|
|
629
|
-
|
|
630
|
-
const serviceAccountJson =
|
|
631
|
-
await readBigQueryServiceAccountJson();
|
|
632
|
-
|
|
633
|
-
const duckdbConnection: ApiConnection = {
|
|
634
|
-
name: "duckdb_bq_query",
|
|
635
|
-
type: "duckdb",
|
|
636
|
-
duckdbConnection: {
|
|
637
|
-
attachedDatabases: [
|
|
638
|
-
{
|
|
639
|
-
name: "bq_query",
|
|
640
|
-
type: "bigquery",
|
|
641
|
-
bigqueryConnection: {
|
|
642
|
-
defaultProjectId:
|
|
643
|
-
process.env.BIGQUERY_TEST_PROJECT_ID!,
|
|
644
|
-
serviceAccountKeyJson: serviceAccountJson,
|
|
645
|
-
},
|
|
646
|
-
},
|
|
647
|
-
],
|
|
648
|
-
},
|
|
649
|
-
};
|
|
650
|
-
|
|
651
|
-
const { malloyConnections } = await createEnvironmentConnections(
|
|
652
|
-
[duckdbConnection],
|
|
653
|
-
testEnvironmentPath,
|
|
654
|
-
);
|
|
655
|
-
|
|
656
|
-
const connection = malloyConnections.get(
|
|
657
|
-
"duckdb_bq_query",
|
|
658
|
-
) as DuckDBConnection;
|
|
659
|
-
createdConnections.push(connection);
|
|
660
|
-
|
|
661
|
-
const result = await connection.runSQL(
|
|
662
|
-
`SELECT * FROM bq_query.${process.env.BIGQUERY_TEST_TABLE!} LIMIT 1`,
|
|
663
|
-
);
|
|
664
|
-
expect(result.rows).toBeDefined();
|
|
665
|
-
expect(result.rows.length).toBeGreaterThan(0);
|
|
666
|
-
},
|
|
667
|
-
{ timeout: 60000 },
|
|
668
|
-
);
|
|
669
|
-
|
|
670
|
-
it(
|
|
671
|
-
"should validate BigQuery service account key format",
|
|
672
|
-
async () => {
|
|
673
|
-
await expect(
|
|
674
|
-
createEnvironmentConnections(
|
|
675
|
-
[
|
|
676
|
-
{
|
|
677
|
-
name: "duckdb_bq_invalid",
|
|
678
|
-
type: "duckdb",
|
|
679
|
-
duckdbConnection: {
|
|
680
|
-
attachedDatabases: [
|
|
681
|
-
{
|
|
682
|
-
name: "bq_invalid",
|
|
683
|
-
type: "bigquery",
|
|
684
|
-
bigqueryConnection: {
|
|
685
|
-
defaultProjectId: "test-environment",
|
|
686
|
-
serviceAccountKeyJson: JSON.stringify({
|
|
687
|
-
invalid: "key",
|
|
688
|
-
}),
|
|
689
|
-
},
|
|
690
|
-
},
|
|
691
|
-
],
|
|
692
|
-
},
|
|
693
|
-
},
|
|
694
|
-
],
|
|
695
|
-
testEnvironmentPath,
|
|
696
|
-
),
|
|
697
|
-
).rejects.toThrow(/Invalid service account key/);
|
|
698
|
-
},
|
|
699
|
-
{ timeout: 30000 },
|
|
700
|
-
);
|
|
701
|
-
});
|
|
702
|
-
|
|
703
|
-
describe("BigQuery direct connection (public-data)", () => {
|
|
704
|
-
// Single end-to-end check that a real `bigquery`-typed
|
|
705
|
-
// connection authenticated by the BIGQUERY_PUBLIC_DATA_SA
|
|
706
|
-
// service account can actually round-trip a query against
|
|
707
|
-
// bigquery-public-data. Skips locally if the creds aren't
|
|
708
|
-
// set; runs in CI under .github/workflows/connection-integration-tests.yml.
|
|
709
|
-
//
|
|
710
|
-
// Picked `samples.shakespeare` because (a) it's tiny (~6 MB
|
|
711
|
-
// / ~165k rows) so the BigQuery byte-scanned cost is well
|
|
712
|
-
// under the 1 TB/month free tier even across many PR runs,
|
|
713
|
-
// (b) it's a long-stable public table so the assertion stays
|
|
714
|
-
// valid across years of reruns. If this assertion ever fails
|
|
715
|
-
// the cause is almost certainly auth, not data drift.
|
|
716
|
-
it(
|
|
717
|
-
"should query bigquery-public-data via real BigQuery connection",
|
|
718
|
-
async () => {
|
|
719
|
-
if (!hasPublicDataBigQueryCredentials()) {
|
|
720
|
-
console.log(
|
|
721
|
-
"Skipping: BIGQUERY_PUBLIC_DATA_CREDENTIALS or BIGQUERY_PUBLIC_DATA_PROJECT_ID not configured",
|
|
722
|
-
);
|
|
723
|
-
return;
|
|
724
|
-
}
|
|
725
|
-
|
|
726
|
-
const serviceAccountJson =
|
|
727
|
-
await readPublicDataBigQueryServiceAccountJson();
|
|
728
|
-
|
|
729
|
-
const bqConnection: ApiConnection = {
|
|
730
|
-
name: "bq_public_data",
|
|
731
|
-
type: "bigquery",
|
|
732
|
-
bigqueryConnection: {
|
|
733
|
-
// Billing/auth project (the SA's own project_id);
|
|
734
|
-
// the queried tables live in bigquery-public-data
|
|
735
|
-
// and are referenced explicitly in the SQL below.
|
|
736
|
-
defaultProjectId:
|
|
737
|
-
process.env.BIGQUERY_PUBLIC_DATA_PROJECT_ID!,
|
|
738
|
-
serviceAccountKeyJson: serviceAccountJson,
|
|
739
|
-
},
|
|
740
|
-
};
|
|
741
|
-
|
|
742
|
-
const { malloyConnections } = await createEnvironmentConnections(
|
|
743
|
-
[bqConnection],
|
|
744
|
-
testEnvironmentPath,
|
|
745
|
-
);
|
|
746
|
-
|
|
747
|
-
const connection = malloyConnections.get("bq_public_data");
|
|
748
|
-
expect(connection).toBeDefined();
|
|
749
|
-
|
|
750
|
-
try {
|
|
751
|
-
const result = await connection!.runSQL(
|
|
752
|
-
"SELECT COUNT(*) AS row_count FROM `bigquery-public-data.samples.shakespeare`",
|
|
753
|
-
);
|
|
754
|
-
expect(result.rows.length).toBe(1);
|
|
755
|
-
// Shakespeare has ~165k rows; bound on both sides so
|
|
756
|
-
// we catch both "auth succeeded but query returned
|
|
757
|
-
// nothing" and "got a confusingly large value" failure
|
|
758
|
-
// modes, while staying tolerant of any minor row-count
|
|
759
|
-
// jitter Google might introduce.
|
|
760
|
-
const row = result.rows[0] as Record<string, unknown>;
|
|
761
|
-
const rowCount = Number(row.row_count);
|
|
762
|
-
expect(rowCount).toBeGreaterThan(100_000);
|
|
763
|
-
expect(rowCount).toBeLessThan(200_000);
|
|
764
|
-
} finally {
|
|
765
|
-
// BigQuery driver holds an HTTP/2 client + auth refresh
|
|
766
|
-
// state; close it explicitly so we don't leak across
|
|
767
|
-
// the rest of the test run. (createdConnections is
|
|
768
|
-
// typed for DuckDBConnection, so we can't use the
|
|
769
|
-
// shared cleanup array here.)
|
|
770
|
-
await connection?.close();
|
|
771
|
-
}
|
|
772
|
-
},
|
|
773
|
-
{ timeout: 60000 },
|
|
774
|
-
);
|
|
775
|
-
});
|
|
776
|
-
|
|
777
|
-
describe("BigQuery package end-to-end (bq-hackernews)", () => {
|
|
778
|
-
// Step 2 of Sagar's bun-setup-fixes ask: not just "does the
|
|
779
|
-
// BigQuery driver round-trip a SQL query" (the previous
|
|
780
|
-
// describe block covers that), but "does the publisher's
|
|
781
|
-
// package-loading path successfully use the BigQuery
|
|
782
|
-
// connection on behalf of a Malloy package."
|
|
783
|
-
//
|
|
784
|
-
// Mechanism: stand up a real EnvironmentStore against a
|
|
785
|
-
// temp publisher.config.json that declares both the BQ
|
|
786
|
-
// connection (with the BIGQUERY_PUBLIC_DATA_SA injected via
|
|
787
|
-
// serviceAccountKeyJson) AND the bq-hackernews package
|
|
788
|
-
// (loaded from credibledata/malloy-samples on GitHub).
|
|
789
|
-
// EnvironmentStore.initialize then: clones the malloy-samples
|
|
790
|
-
// repo, extracts the bigquery-hackernews subdirectory, parses
|
|
791
|
-
// the package's Malloy models, and — crucially — introspects
|
|
792
|
-
// their BigQuery table schemas during model compilation. A
|
|
793
|
-
// successful Package.listModels() call therefore proves the
|
|
794
|
-
// entire package-uses-connection path, not just the bare
|
|
795
|
-
// driver auth.
|
|
796
|
-
//
|
|
797
|
-
// Cost: ~50 MB git clone (one time per test run since
|
|
798
|
-
// publisher_data lives under testEnvironmentPath which the
|
|
799
|
-
// afterEach cleans up) + ~6 MB BQ schema scan. Both well
|
|
800
|
-
// under any meaningful free-tier limit. Test budget: 3 min
|
|
801
|
-
// (clone is ~30-60s on GH runners; compile is ~10s).
|
|
802
|
-
it(
|
|
803
|
-
"should load bq-hackernews package and compile its models via real BQ",
|
|
804
|
-
async () => {
|
|
805
|
-
if (!hasPublicDataBigQueryCredentials()) {
|
|
806
|
-
console.log(
|
|
807
|
-
"Skipping: BIGQUERY_PUBLIC_DATA_CREDENTIALS or BIGQUERY_PUBLIC_DATA_PROJECT_ID not configured",
|
|
808
|
-
);
|
|
809
|
-
return;
|
|
810
|
-
}
|
|
811
|
-
|
|
812
|
-
const serviceAccountJson =
|
|
813
|
-
await readPublicDataBigQueryServiceAccountJson();
|
|
814
|
-
|
|
815
|
-
// Each test gets its own serverRoot so publisher_data
|
|
816
|
-
// doesn't leak across tests; afterEach removes
|
|
817
|
-
// testEnvironmentPath which carries the whole tree
|
|
818
|
-
// away.
|
|
819
|
-
const tempServerRoot = path.join(
|
|
820
|
-
testEnvironmentPath,
|
|
821
|
-
"bq-hackernews-pkg-test",
|
|
822
|
-
);
|
|
823
|
-
await fs.mkdir(tempServerRoot, { recursive: true });
|
|
824
|
-
|
|
825
|
-
const config = {
|
|
826
|
-
frozenConfig: false,
|
|
827
|
-
environments: [
|
|
828
|
-
{
|
|
829
|
-
name: "malloy-samples",
|
|
830
|
-
packages: [
|
|
831
|
-
{
|
|
832
|
-
name: "bigquery-hackernews",
|
|
833
|
-
location:
|
|
834
|
-
"https://github.com/credibledata/malloy-samples/tree/main/bigquery-hackernews",
|
|
835
|
-
},
|
|
836
|
-
],
|
|
837
|
-
connections: [
|
|
838
|
-
{
|
|
839
|
-
name: "bigquery",
|
|
840
|
-
type: "bigquery",
|
|
841
|
-
bigqueryConnection: {
|
|
842
|
-
defaultProjectId:
|
|
843
|
-
process.env
|
|
844
|
-
.BIGQUERY_PUBLIC_DATA_PROJECT_ID!,
|
|
845
|
-
serviceAccountKeyJson: serviceAccountJson,
|
|
846
|
-
},
|
|
847
|
-
},
|
|
848
|
-
],
|
|
849
|
-
},
|
|
850
|
-
],
|
|
851
|
-
};
|
|
852
|
-
await fs.writeFile(
|
|
853
|
-
path.join(tempServerRoot, "publisher.config.json"),
|
|
854
|
-
JSON.stringify(config),
|
|
855
|
-
);
|
|
856
|
-
|
|
857
|
-
// Force the load-from-config init path. Without this,
|
|
858
|
-
// EnvironmentStore.initialize() takes the load-from-DB
|
|
859
|
-
// branch and only falls back to config when the DB is
|
|
860
|
-
// empty (which it is here, by construction, but relying
|
|
861
|
-
// on that is brittle if the test pattern is reused).
|
|
862
|
-
//
|
|
863
|
-
// SAFETY PRECONDITION: this flag is the SAME one users
|
|
864
|
-
// opt into via `--init` / `start:init` to wipe persisted
|
|
865
|
-
// storage and re-initialize from config (see CLAUDE.md
|
|
866
|
-
// and `environment_store.ts:158`). It is destructive on
|
|
867
|
-
// a real serverRoot. We only set it here because
|
|
868
|
-
// `tempServerRoot` is a freshly-created empty directory
|
|
869
|
-
// (path.join(testEnvironmentPath, "bq-hackernews-pkg-test")
|
|
870
|
-
// mkdir'd ~20 lines above), so there is nothing to wipe.
|
|
871
|
-
// DO NOT copy this pattern into a test that points
|
|
872
|
-
// EnvironmentStore at a non-empty or shared serverRoot
|
|
873
|
-
// — it will delete state.
|
|
874
|
-
const previousInitializeStorage = process.env.INITIALIZE_STORAGE;
|
|
875
|
-
process.env.INITIALIZE_STORAGE = "true";
|
|
876
|
-
|
|
877
|
-
try {
|
|
878
|
-
const envStore = new EnvironmentStore(tempServerRoot);
|
|
879
|
-
await envStore.finishedInitialization;
|
|
880
|
-
|
|
881
|
-
// initialize() swallows top-level errors and just calls
|
|
882
|
-
// markNotReady(), so finishedInitialization always resolves
|
|
883
|
-
// regardless of success; operationalState is what says it
|
|
884
|
-
// succeeded. It does NOT imply every configured environment
|
|
885
|
-
// and package loaded: a load failure is skipped and still
|
|
886
|
-
// reports "serving", which is why loadErrors exists. Assert
|
|
887
|
-
// on both, and on the package list below.
|
|
888
|
-
const status = await envStore.getStatus();
|
|
889
|
-
expect(status.operationalState).toBe("serving");
|
|
890
|
-
expect(status.loadErrors).toBeUndefined();
|
|
891
|
-
|
|
892
|
-
const env = await envStore.getEnvironment("malloy-samples");
|
|
893
|
-
const apiPackages = await env.listPackages();
|
|
894
|
-
expect(apiPackages.map((p) => p.name)).toContain(
|
|
895
|
-
"bigquery-hackernews",
|
|
896
|
-
);
|
|
897
|
-
|
|
898
|
-
const apiConnections = env.listApiConnections();
|
|
899
|
-
expect(apiConnections.map((c) => c.name)).toContain(
|
|
900
|
-
"bigquery",
|
|
901
|
-
);
|
|
902
|
-
|
|
903
|
-
// The actual integration assertion. Package.listModels()
|
|
904
|
-
// collects compile errors per-model and returns ALL
|
|
905
|
-
// models in the result (with `error` populated on
|
|
906
|
-
// failures) — so models.length>0 alone would pass even
|
|
907
|
-
// if every BQ schema introspection failed. Filter for
|
|
908
|
-
// models that compiled cleanly: at least one is the
|
|
909
|
-
// real proof that BQ-on-behalf-of-a-package works.
|
|
910
|
-
const pkg = await env.getPackage("bigquery-hackernews");
|
|
911
|
-
const models = await pkg.listModels();
|
|
912
|
-
const okModels = models.filter((m) => !m.error);
|
|
913
|
-
if (okModels.length === 0) {
|
|
914
|
-
console.error(
|
|
915
|
-
"All bq-hackernews model compilations failed:",
|
|
916
|
-
models.map((m) => `${m.path}: ${m.error}`),
|
|
917
|
-
);
|
|
918
|
-
}
|
|
919
|
-
expect(okModels.length).toBeGreaterThan(0);
|
|
920
|
-
} finally {
|
|
921
|
-
if (previousInitializeStorage === undefined) {
|
|
922
|
-
delete process.env.INITIALIZE_STORAGE;
|
|
923
|
-
} else {
|
|
924
|
-
process.env.INITIALIZE_STORAGE = previousInitializeStorage;
|
|
925
|
-
}
|
|
926
|
-
}
|
|
927
|
-
},
|
|
928
|
-
{ timeout: 180000 },
|
|
929
|
-
);
|
|
930
|
-
});
|
|
931
|
-
|
|
932
|
-
describe("DuckDB with Snowflake attachment", () => {
|
|
933
|
-
it(
|
|
934
|
-
"should create DuckDB connection with attached Snowflake database",
|
|
935
|
-
async () => {
|
|
936
|
-
if (!hasSnowflakeCredentials()) {
|
|
937
|
-
console.log("Skipping: Snowflake credentials not configured");
|
|
938
|
-
return;
|
|
939
|
-
}
|
|
940
|
-
|
|
941
|
-
const duckdbConnection: ApiConnection = {
|
|
942
|
-
name: "duckdb_with_snowflake",
|
|
943
|
-
type: "duckdb",
|
|
944
|
-
duckdbConnection: {
|
|
945
|
-
attachedDatabases: [
|
|
946
|
-
{
|
|
947
|
-
name: "sf_test",
|
|
948
|
-
type: "snowflake",
|
|
949
|
-
snowflakeConnection: {
|
|
950
|
-
account: process.env.SNOWFLAKE_TEST_ACCOUNT!,
|
|
951
|
-
username: process.env.SNOWFLAKE_TEST_USER!,
|
|
952
|
-
password: process.env.SNOWFLAKE_TEST_PASSWORD!,
|
|
953
|
-
warehouse: process.env.SNOWFLAKE_TEST_WAREHOUSE!,
|
|
954
|
-
database: process.env.SNOWFLAKE_TEST_DATABASE,
|
|
955
|
-
schema: process.env.SNOWFLAKE_TEST_SCHEMA,
|
|
956
|
-
},
|
|
957
|
-
},
|
|
958
|
-
],
|
|
959
|
-
},
|
|
960
|
-
};
|
|
961
|
-
|
|
962
|
-
const { malloyConnections } = await createEnvironmentConnections(
|
|
963
|
-
[duckdbConnection],
|
|
964
|
-
testEnvironmentPath,
|
|
965
|
-
);
|
|
966
|
-
|
|
967
|
-
const connection = malloyConnections.get(
|
|
968
|
-
"duckdb_with_snowflake",
|
|
969
|
-
) as DuckDBConnection;
|
|
970
|
-
expect(connection).toBeDefined();
|
|
971
|
-
createdConnections.push(connection);
|
|
972
|
-
|
|
973
|
-
const databases = await connection.runSQL("SHOW DATABASES");
|
|
974
|
-
const dbNames = databases.rows.map(
|
|
975
|
-
(row) => Object.values(row)[0],
|
|
976
|
-
);
|
|
977
|
-
expect(dbNames).toContain("sf_test");
|
|
978
|
-
|
|
979
|
-
const result = await connection.runSQL(
|
|
980
|
-
"SELECT * FROM snowflake_query('SELECT 1 as test', 'sf_test_secret')",
|
|
981
|
-
);
|
|
982
|
-
expect(result.rows).toBeDefined();
|
|
983
|
-
},
|
|
984
|
-
{ timeout: 30000 },
|
|
985
|
-
);
|
|
986
|
-
|
|
987
|
-
it("should validate required Snowflake fields", async () => {
|
|
988
|
-
await expect(
|
|
989
|
-
createEnvironmentConnections(
|
|
990
|
-
[
|
|
991
|
-
{
|
|
992
|
-
name: "duckdb_sf_incomplete",
|
|
993
|
-
type: "duckdb",
|
|
994
|
-
duckdbConnection: {
|
|
995
|
-
attachedDatabases: [
|
|
996
|
-
{
|
|
997
|
-
name: "sf_incomplete",
|
|
998
|
-
type: "snowflake",
|
|
999
|
-
snowflakeConnection: {
|
|
1000
|
-
account: "test-account",
|
|
1001
|
-
username: "test-user",
|
|
1002
|
-
},
|
|
1003
|
-
},
|
|
1004
|
-
],
|
|
1005
|
-
},
|
|
1006
|
-
},
|
|
1007
|
-
],
|
|
1008
|
-
testEnvironmentPath,
|
|
1009
|
-
),
|
|
1010
|
-
).rejects.toThrow(/required/);
|
|
1011
|
-
});
|
|
1012
|
-
});
|
|
1013
|
-
|
|
1014
|
-
describe("DuckDB with S3 attachment", () => {
|
|
1015
|
-
it(
|
|
1016
|
-
"should create DuckDB connection with S3 configuration",
|
|
1017
|
-
async () => {
|
|
1018
|
-
if (!hasS3Credentials()) {
|
|
1019
|
-
console.log("Skipping: S3 credentials not configured");
|
|
1020
|
-
return;
|
|
1021
|
-
}
|
|
1022
|
-
|
|
1023
|
-
const duckdbConnection: ApiConnection = {
|
|
1024
|
-
name: "duckdb_with_s3",
|
|
1025
|
-
type: "duckdb",
|
|
1026
|
-
duckdbConnection: {
|
|
1027
|
-
attachedDatabases: [
|
|
1028
|
-
{
|
|
1029
|
-
name: "s3_test",
|
|
1030
|
-
type: "s3",
|
|
1031
|
-
s3Connection: {
|
|
1032
|
-
accessKeyId: process.env.S3_TEST_ACCESS_KEY_ID!,
|
|
1033
|
-
secretAccessKey:
|
|
1034
|
-
process.env.S3_TEST_SECRET_ACCESS_KEY!,
|
|
1035
|
-
region: process.env.S3_TEST_REGION || "us-east-1",
|
|
1036
|
-
},
|
|
1037
|
-
},
|
|
1038
|
-
],
|
|
1039
|
-
},
|
|
1040
|
-
};
|
|
1041
|
-
|
|
1042
|
-
const { malloyConnections } = await createEnvironmentConnections(
|
|
1043
|
-
[duckdbConnection],
|
|
1044
|
-
testEnvironmentPath,
|
|
1045
|
-
);
|
|
1046
|
-
|
|
1047
|
-
const connection = malloyConnections.get(
|
|
1048
|
-
"duckdb_with_s3",
|
|
1049
|
-
) as DuckDBConnection;
|
|
1050
|
-
expect(connection).toBeDefined();
|
|
1051
|
-
createdConnections.push(connection);
|
|
1052
|
-
|
|
1053
|
-
const secrets = await connection.runSQL(
|
|
1054
|
-
"SELECT name FROM duckdb_secrets()",
|
|
1055
|
-
);
|
|
1056
|
-
const secretNames = secrets.rows.map(
|
|
1057
|
-
(row) => Object.values(row)[0],
|
|
1058
|
-
);
|
|
1059
|
-
expect(
|
|
1060
|
-
secretNames.some((name) => String(name).includes("s3")),
|
|
1061
|
-
).toBe(true);
|
|
1062
|
-
},
|
|
1063
|
-
{ timeout: 30000 },
|
|
1064
|
-
);
|
|
1065
|
-
|
|
1066
|
-
it(
|
|
1067
|
-
"should handle S3 with custom endpoint",
|
|
1068
|
-
async () => {
|
|
1069
|
-
if (!hasS3Credentials()) {
|
|
1070
|
-
console.log("Skipping: S3 credentials not configured");
|
|
1071
|
-
return;
|
|
1072
|
-
}
|
|
1073
|
-
|
|
1074
|
-
const duckdbConnection: ApiConnection = {
|
|
1075
|
-
name: "duckdb_s3_custom",
|
|
1076
|
-
type: "duckdb",
|
|
1077
|
-
duckdbConnection: {
|
|
1078
|
-
attachedDatabases: [
|
|
1079
|
-
{
|
|
1080
|
-
name: "s3_custom",
|
|
1081
|
-
type: "s3",
|
|
1082
|
-
s3Connection: {
|
|
1083
|
-
accessKeyId: process.env.S3_TEST_ACCESS_KEY_ID!,
|
|
1084
|
-
secretAccessKey:
|
|
1085
|
-
process.env.S3_TEST_SECRET_ACCESS_KEY!,
|
|
1086
|
-
region: "us-east-1",
|
|
1087
|
-
endpoint: "https://s3.custom-endpoint.com",
|
|
1088
|
-
},
|
|
1089
|
-
},
|
|
1090
|
-
],
|
|
1091
|
-
},
|
|
1092
|
-
};
|
|
1093
|
-
|
|
1094
|
-
const { malloyConnections } = await createEnvironmentConnections(
|
|
1095
|
-
[duckdbConnection],
|
|
1096
|
-
testEnvironmentPath,
|
|
1097
|
-
);
|
|
1098
|
-
|
|
1099
|
-
const connection = malloyConnections.get(
|
|
1100
|
-
"duckdb_s3_custom",
|
|
1101
|
-
) as DuckDBConnection;
|
|
1102
|
-
createdConnections.push(connection);
|
|
1103
|
-
expect(connection).toBeDefined();
|
|
1104
|
-
},
|
|
1105
|
-
{ timeout: 30000 },
|
|
1106
|
-
);
|
|
1107
|
-
});
|
|
1108
|
-
|
|
1109
|
-
describe("DuckDB with GCS attachment", () => {
|
|
1110
|
-
it(
|
|
1111
|
-
"should create DuckDB connection with GCS configuration",
|
|
1112
|
-
async () => {
|
|
1113
|
-
if (!hasGCSCredentials()) {
|
|
1114
|
-
console.log("Skipping: GCS credentials not configured");
|
|
1115
|
-
return;
|
|
1116
|
-
}
|
|
1117
|
-
|
|
1118
|
-
const duckdbConnection: ApiConnection = {
|
|
1119
|
-
name: "duckdb_with_gcs",
|
|
1120
|
-
type: "duckdb",
|
|
1121
|
-
duckdbConnection: {
|
|
1122
|
-
attachedDatabases: [
|
|
1123
|
-
{
|
|
1124
|
-
name: "gcs_test",
|
|
1125
|
-
type: "gcs",
|
|
1126
|
-
gcsConnection: {
|
|
1127
|
-
keyId: process.env.GCS_TEST_KEY_ID!,
|
|
1128
|
-
secret: process.env.GCS_TEST_SECRET!,
|
|
1129
|
-
},
|
|
1130
|
-
},
|
|
1131
|
-
],
|
|
1132
|
-
},
|
|
1133
|
-
};
|
|
1134
|
-
|
|
1135
|
-
const { malloyConnections } = await createEnvironmentConnections(
|
|
1136
|
-
[duckdbConnection],
|
|
1137
|
-
testEnvironmentPath,
|
|
1138
|
-
);
|
|
1139
|
-
|
|
1140
|
-
const connection = malloyConnections.get(
|
|
1141
|
-
"duckdb_with_gcs",
|
|
1142
|
-
) as DuckDBConnection;
|
|
1143
|
-
expect(connection).toBeDefined();
|
|
1144
|
-
createdConnections.push(connection);
|
|
1145
|
-
|
|
1146
|
-
const secrets = await connection.runSQL(
|
|
1147
|
-
"SELECT name FROM duckdb_secrets()",
|
|
1148
|
-
);
|
|
1149
|
-
const secretNames = secrets.rows.map(
|
|
1150
|
-
(row) => Object.values(row)[0],
|
|
1151
|
-
);
|
|
1152
|
-
expect(
|
|
1153
|
-
secretNames.some((name) => String(name).includes("gcs")),
|
|
1154
|
-
).toBe(true);
|
|
1155
|
-
},
|
|
1156
|
-
{ timeout: 30000 },
|
|
1157
|
-
);
|
|
1158
|
-
});
|
|
1159
|
-
|
|
1160
|
-
describe("DuckDB with multiple attachments", () => {
|
|
1161
|
-
it(
|
|
1162
|
-
"should attach multiple databases to single DuckDB connection",
|
|
1163
|
-
async () => {
|
|
1164
|
-
const attachments: AttachedDatabase[] = [];
|
|
1165
|
-
|
|
1166
|
-
if (hasPostgresCredentials()) {
|
|
1167
|
-
attachments.push({
|
|
1168
|
-
name: "pg_multi",
|
|
1169
|
-
type: "postgres",
|
|
1170
|
-
postgresConnection: {
|
|
1171
|
-
host: process.env.POSTGRES_TEST_HOST,
|
|
1172
|
-
port: parseInt(
|
|
1173
|
-
process.env.POSTGRES_TEST_PORT || "5432",
|
|
1174
|
-
),
|
|
1175
|
-
userName: process.env.POSTGRES_TEST_USER!,
|
|
1176
|
-
password: process.env.POSTGRES_TEST_PASSWORD!,
|
|
1177
|
-
databaseName: process.env.POSTGRES_TEST_DATABASE,
|
|
1178
|
-
},
|
|
1179
|
-
});
|
|
1180
|
-
}
|
|
1181
|
-
|
|
1182
|
-
if (hasBigQueryCredentials()) {
|
|
1183
|
-
const serviceAccountJson =
|
|
1184
|
-
await readBigQueryServiceAccountJson();
|
|
1185
|
-
attachments.push({
|
|
1186
|
-
name: "bq_multi",
|
|
1187
|
-
type: "bigquery",
|
|
1188
|
-
bigqueryConnection: {
|
|
1189
|
-
defaultProjectId: process.env.BIGQUERY_TEST_PROJECT_ID!,
|
|
1190
|
-
serviceAccountKeyJson: serviceAccountJson,
|
|
1191
|
-
},
|
|
1192
|
-
});
|
|
1193
|
-
}
|
|
1194
|
-
|
|
1195
|
-
if (hasS3Credentials()) {
|
|
1196
|
-
attachments.push({
|
|
1197
|
-
name: "s3_multi",
|
|
1198
|
-
type: "s3",
|
|
1199
|
-
s3Connection: {
|
|
1200
|
-
accessKeyId: process.env.S3_TEST_ACCESS_KEY_ID!,
|
|
1201
|
-
secretAccessKey: process.env.S3_TEST_SECRET_ACCESS_KEY!,
|
|
1202
|
-
region: process.env.S3_TEST_REGION || "us-east-1",
|
|
1203
|
-
},
|
|
1204
|
-
});
|
|
1205
|
-
}
|
|
1206
|
-
|
|
1207
|
-
if (attachments.length === 0) {
|
|
1208
|
-
console.log("Skipping: No database credentials configured");
|
|
1209
|
-
return;
|
|
1210
|
-
}
|
|
1211
|
-
|
|
1212
|
-
const { malloyConnections } = await createEnvironmentConnections(
|
|
1213
|
-
[
|
|
1214
|
-
{
|
|
1215
|
-
name: "duckdb_multi",
|
|
1216
|
-
type: "duckdb",
|
|
1217
|
-
duckdbConnection: { attachedDatabases: attachments },
|
|
1218
|
-
},
|
|
1219
|
-
],
|
|
1220
|
-
testEnvironmentPath,
|
|
1221
|
-
);
|
|
1222
|
-
|
|
1223
|
-
const connection = malloyConnections.get(
|
|
1224
|
-
"duckdb_multi",
|
|
1225
|
-
) as DuckDBConnection;
|
|
1226
|
-
expect(connection).toBeDefined();
|
|
1227
|
-
createdConnections.push(connection);
|
|
1228
|
-
|
|
1229
|
-
const databases = await connection.runSQL("SHOW DATABASES");
|
|
1230
|
-
const dbNames = databases.rows.map(
|
|
1231
|
-
(row) => Object.values(row)[0],
|
|
1232
|
-
);
|
|
1233
|
-
attachments.forEach((attachment) => {
|
|
1234
|
-
expect(dbNames).toContain(attachment.name!);
|
|
1235
|
-
});
|
|
1236
|
-
},
|
|
1237
|
-
{ timeout: 60000 },
|
|
1238
|
-
);
|
|
1239
|
-
});
|
|
1240
|
-
|
|
1241
|
-
describe("error handling", () => {
|
|
1242
|
-
describe("DuckLake connection type", () => {
|
|
1243
|
-
it(
|
|
1244
|
-
"should create DuckLake connection",
|
|
1245
|
-
async () => {
|
|
1246
|
-
if (!hasPostgresCredentials() || !hasS3Credentials()) {
|
|
1247
|
-
console.log(
|
|
1248
|
-
"Skipping: PostgreSQL and S3 credentials not configured",
|
|
1249
|
-
);
|
|
1250
|
-
return;
|
|
1251
|
-
}
|
|
1252
|
-
|
|
1253
|
-
const { malloyConnections } =
|
|
1254
|
-
await createEnvironmentConnections(
|
|
1255
|
-
[
|
|
1256
|
-
{
|
|
1257
|
-
name: "ducklake_test",
|
|
1258
|
-
type: "ducklake",
|
|
1259
|
-
ducklakeConnection: {
|
|
1260
|
-
catalog: {
|
|
1261
|
-
postgresConnection: {
|
|
1262
|
-
host: process.env.POSTGRES_TEST_HOST,
|
|
1263
|
-
port: parseInt(
|
|
1264
|
-
process.env.POSTGRES_TEST_PORT ||
|
|
1265
|
-
"5432",
|
|
1266
|
-
),
|
|
1267
|
-
userName:
|
|
1268
|
-
process.env.POSTGRES_TEST_USER!,
|
|
1269
|
-
password:
|
|
1270
|
-
process.env.POSTGRES_TEST_PASSWORD!,
|
|
1271
|
-
databaseName:
|
|
1272
|
-
process.env.POSTGRES_TEST_DATABASE,
|
|
1273
|
-
},
|
|
1274
|
-
},
|
|
1275
|
-
storage: {
|
|
1276
|
-
bucketUrl:
|
|
1277
|
-
process.env.S3_TEST_BUCKET_URL ||
|
|
1278
|
-
"s3://test-bucket",
|
|
1279
|
-
s3Connection: {
|
|
1280
|
-
accessKeyId:
|
|
1281
|
-
process.env.S3_TEST_ACCESS_KEY_ID!,
|
|
1282
|
-
secretAccessKey:
|
|
1283
|
-
process.env
|
|
1284
|
-
.S3_TEST_SECRET_ACCESS_KEY!,
|
|
1285
|
-
},
|
|
1286
|
-
},
|
|
1287
|
-
},
|
|
1288
|
-
},
|
|
1289
|
-
],
|
|
1290
|
-
testEnvironmentPath,
|
|
1291
|
-
);
|
|
1292
|
-
|
|
1293
|
-
const connection = malloyConnections.get(
|
|
1294
|
-
"ducklake_test",
|
|
1295
|
-
) as DuckDBConnection;
|
|
1296
|
-
createdConnections.push(connection);
|
|
1297
|
-
expect(connection).toBeDefined();
|
|
1298
|
-
|
|
1299
|
-
// Verify DuckLake database is attached
|
|
1300
|
-
const databases = await connection.runSQL("SHOW DATABASES");
|
|
1301
|
-
const dbNames = databases.rows.map(
|
|
1302
|
-
(row) => Object.values(row)[0],
|
|
1303
|
-
);
|
|
1304
|
-
expect(dbNames).toContain("ducklake_test");
|
|
1305
|
-
},
|
|
1306
|
-
{ timeout: 30000 },
|
|
1307
|
-
);
|
|
1308
|
-
|
|
1309
|
-
it("should throw error if DuckLake catalog connection is missing", async () => {
|
|
1310
|
-
await expect(
|
|
1311
|
-
createEnvironmentConnections(
|
|
1312
|
-
[
|
|
1313
|
-
{
|
|
1314
|
-
name: "ducklake_no_catalog",
|
|
1315
|
-
type: "ducklake",
|
|
1316
|
-
ducklakeConnection: {
|
|
1317
|
-
storage: {
|
|
1318
|
-
bucketUrl: "s3://test-bucket",
|
|
1319
|
-
s3Connection: {
|
|
1320
|
-
accessKeyId: "test",
|
|
1321
|
-
secretAccessKey: "test",
|
|
1322
|
-
},
|
|
1323
|
-
},
|
|
1324
|
-
},
|
|
1325
|
-
} as ApiConnection,
|
|
1326
|
-
],
|
|
1327
|
-
testEnvironmentPath,
|
|
1328
|
-
),
|
|
1329
|
-
).rejects.toThrow(
|
|
1330
|
-
/PostgreSQL connection configuration is required/,
|
|
1331
|
-
);
|
|
1332
|
-
});
|
|
1333
|
-
|
|
1334
|
-
it("should throw error if DuckLake connection config is missing", async () => {
|
|
1335
|
-
await expect(
|
|
1336
|
-
createEnvironmentConnections(
|
|
1337
|
-
[
|
|
1338
|
-
{
|
|
1339
|
-
name: "ducklake_missing_config",
|
|
1340
|
-
type: "ducklake",
|
|
1341
|
-
},
|
|
1342
|
-
],
|
|
1343
|
-
testEnvironmentPath,
|
|
1344
|
-
),
|
|
1345
|
-
).rejects.toThrow(
|
|
1346
|
-
/DuckLake connection configuration is missing/,
|
|
1347
|
-
);
|
|
1348
|
-
});
|
|
1349
|
-
|
|
1350
|
-
// ── catalog.metadataSchema ──────────────────────────────────────
|
|
1351
|
-
//
|
|
1352
|
-
// Several independent DuckLake catalogs may share ONE catalog
|
|
1353
|
-
// database when each attaches its own metadata schema. These use a
|
|
1354
|
-
// LOCAL directory as DATA_PATH rather than S3/GCS, so they need only
|
|
1355
|
-
// the Postgres service and are not skipped for want of object-store
|
|
1356
|
-
// credentials.
|
|
1357
|
-
describe("catalog.metadataSchema", () => {
|
|
1358
|
-
const pgCatalog = () => ({
|
|
1359
|
-
host: process.env.POSTGRES_TEST_HOST,
|
|
1360
|
-
port: parseInt(process.env.POSTGRES_TEST_PORT || "5432"),
|
|
1361
|
-
userName: process.env.POSTGRES_TEST_USER!,
|
|
1362
|
-
password: process.env.POSTGRES_TEST_PASSWORD!,
|
|
1363
|
-
databaseName: process.env.POSTGRES_TEST_DATABASE,
|
|
1364
|
-
});
|
|
1365
|
-
|
|
1366
|
-
const pgConnString = () => {
|
|
1367
|
-
const pg = pgCatalog();
|
|
1368
|
-
return (
|
|
1369
|
-
`host=${pg.host} port=${pg.port} user=${pg.userName} ` +
|
|
1370
|
-
`password=${pg.password}` +
|
|
1371
|
-
(pg.databaseName ? ` dbname=${pg.databaseName}` : "")
|
|
1372
|
-
);
|
|
1373
|
-
};
|
|
1374
|
-
|
|
1375
|
-
// A unique schema per test AND per run: the catalog database is
|
|
1376
|
-
// shared across the specs in this file, and (locally) across runs,
|
|
1377
|
-
// so a fixed name would let one test observe another's metadata.
|
|
1378
|
-
const runId = Math.random().toString(36).slice(2, 8);
|
|
1379
|
-
let schemaSeq = 0;
|
|
1380
|
-
const createdSchemas: string[] = [];
|
|
1381
|
-
const uniqueSchema = (label: string) => {
|
|
1382
|
-
const schema = `dl_meta_${label}_${runId}_${++schemaSeq}`;
|
|
1383
|
-
createdSchemas.push(schema);
|
|
1384
|
-
return schema;
|
|
1385
|
-
};
|
|
1386
|
-
|
|
1387
|
-
// Leave the catalog database as we found it. These schemas are
|
|
1388
|
-
// created by DuckLake inside a database shared with the other
|
|
1389
|
-
// specs in this file, and enough of them makes a sibling test's
|
|
1390
|
-
// schema enumeration miss `public`.
|
|
1391
|
-
afterAll(async () => {
|
|
1392
|
-
if (
|
|
1393
|
-
!hasPostgresCredentials() ||
|
|
1394
|
-
createdSchemas.length === 0
|
|
1395
|
-
) {
|
|
1396
|
-
return;
|
|
1397
|
-
}
|
|
1398
|
-
const workDir = await fs.mkdtemp(
|
|
1399
|
-
path.join(os.tmpdir(), "dl-cleanup-"),
|
|
1400
|
-
);
|
|
1401
|
-
const conn = new DuckDBConnection(
|
|
1402
|
-
"dl_cleanup",
|
|
1403
|
-
":memory:",
|
|
1404
|
-
workDir,
|
|
1405
|
-
);
|
|
1406
|
-
try {
|
|
1407
|
-
await conn.runSQL(`INSTALL postgres`);
|
|
1408
|
-
await conn.runSQL(`LOAD postgres`);
|
|
1409
|
-
await conn.runSQL(
|
|
1410
|
-
`ATTACH '${pgConnString()}' AS cleanup (TYPE postgres);`,
|
|
1411
|
-
);
|
|
1412
|
-
for (const schema of createdSchemas) {
|
|
1413
|
-
// Quoted, because at least one test registers a mixed-case
|
|
1414
|
-
// schema and an unquoted identifier would not match it. And
|
|
1415
|
-
// reported rather than swallowed: a silently failing drop
|
|
1416
|
-
// leaks one schema per run into the shared catalog database,
|
|
1417
|
-
// which is the accumulation this hook exists to prevent.
|
|
1418
|
-
await conn
|
|
1419
|
-
.runSQL(`DROP SCHEMA cleanup."${schema}" CASCADE;`)
|
|
1420
|
-
.catch((error: unknown) =>
|
|
1421
|
-
console.warn(
|
|
1422
|
-
`Failed to drop test schema "${schema}":`,
|
|
1423
|
-
error,
|
|
1424
|
-
),
|
|
1425
|
-
);
|
|
1426
|
-
}
|
|
1427
|
-
} catch (error) {
|
|
1428
|
-
console.warn("DuckLake schema cleanup failed:", error);
|
|
1429
|
-
} finally {
|
|
1430
|
-
await conn.close().catch(() => undefined);
|
|
1431
|
-
}
|
|
1432
|
-
});
|
|
1433
|
-
|
|
1434
|
-
const duckLakeConfig = async (
|
|
1435
|
-
name: string,
|
|
1436
|
-
metadataSchema?: string,
|
|
1437
|
-
) => {
|
|
1438
|
-
const dataDir = path.join(
|
|
1439
|
-
testEnvironmentPath,
|
|
1440
|
-
`${name}_data`,
|
|
1441
|
-
);
|
|
1442
|
-
await fs.mkdir(dataDir, { recursive: true });
|
|
1443
|
-
return {
|
|
1444
|
-
catalog: {
|
|
1445
|
-
postgresConnection: pgCatalog(),
|
|
1446
|
-
...(metadataSchema ? { metadataSchema } : {}),
|
|
1447
|
-
},
|
|
1448
|
-
storage: { bucketUrl: `${dataDir}/` },
|
|
1449
|
-
} as components["schemas"]["DucklakeConnection"];
|
|
1450
|
-
};
|
|
1451
|
-
|
|
1452
|
-
// Bootstrap a catalog the way a BUILD does — a read-WRITE attach.
|
|
1453
|
-
// The read-only serve attach cannot create one ("creating a new
|
|
1454
|
-
// DuckLake is explicitly disabled"), so anything exercising the
|
|
1455
|
-
// serve path needs this first. Also the natural place to assert
|
|
1456
|
-
// that the build path carries METADATA_SCHEMA.
|
|
1457
|
-
const bootstrapCatalog = async (
|
|
1458
|
-
dbName: string,
|
|
1459
|
-
cfg: components["schemas"]["DucklakeConnection"],
|
|
1460
|
-
): Promise<DuckDBConnection> => {
|
|
1461
|
-
const workDir = await fs.mkdtemp(
|
|
1462
|
-
path.join(os.tmpdir(), `dl-boot-${dbName}-`),
|
|
1463
|
-
);
|
|
1464
|
-
const conn = new DuckDBConnection(
|
|
1465
|
-
`bootstrap_${dbName}`,
|
|
1466
|
-
":memory:",
|
|
1467
|
-
workDir,
|
|
1468
|
-
);
|
|
1469
|
-
createdConnections.push(conn);
|
|
1470
|
-
await attachDuckLakeReadWrite(conn, dbName, cfg);
|
|
1471
|
-
return conn;
|
|
1472
|
-
};
|
|
1473
|
-
|
|
1474
|
-
const connect = async (defs: ApiConnection[]) => {
|
|
1475
|
-
const { malloyConnections } =
|
|
1476
|
-
await createEnvironmentConnections(
|
|
1477
|
-
defs,
|
|
1478
|
-
testEnvironmentPath,
|
|
1479
|
-
);
|
|
1480
|
-
const out = new Map<string, DuckDBConnection>();
|
|
1481
|
-
for (const def of defs) {
|
|
1482
|
-
const c = malloyConnections.get(
|
|
1483
|
-
def.name!,
|
|
1484
|
-
) as DuckDBConnection;
|
|
1485
|
-
createdConnections.push(c);
|
|
1486
|
-
out.set(def.name!, c);
|
|
1487
|
-
}
|
|
1488
|
-
return out;
|
|
1489
|
-
};
|
|
1490
|
-
|
|
1491
|
-
const firstValue = (rows: Record<string, unknown>[]) =>
|
|
1492
|
-
Number(Object.values(rows[0])[0]);
|
|
1493
|
-
|
|
1494
|
-
it(
|
|
1495
|
-
"puts DuckLake metadata in the configured schema, creating it",
|
|
1496
|
-
async () => {
|
|
1497
|
-
if (!hasPostgresCredentials()) {
|
|
1498
|
-
console.log("Skipping: PostgreSQL not configured");
|
|
1499
|
-
return;
|
|
1500
|
-
}
|
|
1501
|
-
const schema = uniqueSchema("lands");
|
|
1502
|
-
const cfg = await duckLakeConfig("dl_lands", schema);
|
|
1503
|
-
const c = await bootstrapCatalog("dl_lands", cfg);
|
|
1504
|
-
await c.runSQL(
|
|
1505
|
-
`CREATE OR REPLACE TABLE dl_lands.t AS SELECT 1 AS x`,
|
|
1506
|
-
);
|
|
1507
|
-
|
|
1508
|
-
// Read the catalog database directly: the ducklake_*
|
|
1509
|
-
// bookkeeping tables must be in `schema`, which DuckLake
|
|
1510
|
-
// created on attach (this test never created the schema).
|
|
1511
|
-
await c.runSQL(
|
|
1512
|
-
`ATTACH '${pgConnString()}' AS insp (TYPE postgres, READ_ONLY);`,
|
|
1513
|
-
);
|
|
1514
|
-
const found = await c.runSQL(
|
|
1515
|
-
`SELECT count(*) AS n FROM insp.${schema}.ducklake_metadata;`,
|
|
1516
|
-
);
|
|
1517
|
-
expect(firstValue(found.rows)).toBeGreaterThan(0);
|
|
1518
|
-
},
|
|
1519
|
-
{ timeout: 60000 },
|
|
1520
|
-
);
|
|
1521
|
-
|
|
1522
|
-
it(
|
|
1523
|
-
"isolates two catalogs sharing one catalog database",
|
|
1524
|
-
async () => {
|
|
1525
|
-
if (!hasPostgresCredentials()) {
|
|
1526
|
-
console.log("Skipping: PostgreSQL not configured");
|
|
1527
|
-
return;
|
|
1528
|
-
}
|
|
1529
|
-
const a = await bootstrapCatalog(
|
|
1530
|
-
"dl_a",
|
|
1531
|
-
await duckLakeConfig("dl_a", uniqueSchema("a")),
|
|
1532
|
-
);
|
|
1533
|
-
const b = await bootstrapCatalog(
|
|
1534
|
-
"dl_b",
|
|
1535
|
-
await duckLakeConfig("dl_b", uniqueSchema("b")),
|
|
1536
|
-
);
|
|
1537
|
-
|
|
1538
|
-
await a.runSQL(
|
|
1539
|
-
`CREATE OR REPLACE TABLE dl_a.only_in_a AS SELECT 1`,
|
|
1540
|
-
);
|
|
1541
|
-
await b.runSQL(
|
|
1542
|
-
`CREATE OR REPLACE TABLE dl_b.only_in_b AS SELECT 2`,
|
|
1543
|
-
);
|
|
1544
|
-
|
|
1545
|
-
const tablesOf = async (
|
|
1546
|
-
conn: DuckDBConnection,
|
|
1547
|
-
db: string,
|
|
1548
|
-
) => {
|
|
1549
|
-
const r = await conn.runSQL(
|
|
1550
|
-
`SELECT table_name FROM duckdb_tables() WHERE database_name = '${db}';`,
|
|
1551
|
-
);
|
|
1552
|
-
return r.rows.map((row) => Object.values(row)[0]);
|
|
1553
|
-
};
|
|
1554
|
-
|
|
1555
|
-
// Each catalog sees its own table and NOT the other's —
|
|
1556
|
-
// the whole point of a per-catalog metadata schema. Both
|
|
1557
|
-
// live in ONE catalog database.
|
|
1558
|
-
expect(await tablesOf(a, "dl_a")).toEqual(["only_in_a"]);
|
|
1559
|
-
expect(await tablesOf(b, "dl_b")).toEqual(["only_in_b"]);
|
|
1560
|
-
},
|
|
1561
|
-
{ timeout: 60000 },
|
|
1562
|
-
);
|
|
1563
|
-
|
|
1564
|
-
it(
|
|
1565
|
-
"serves read-only from a catalog in a non-default schema",
|
|
1566
|
-
async () => {
|
|
1567
|
-
if (!hasPostgresCredentials()) {
|
|
1568
|
-
console.log("Skipping: PostgreSQL not configured");
|
|
1569
|
-
return;
|
|
1570
|
-
}
|
|
1571
|
-
// The read-only serve attach must pass METADATA_SCHEMA too:
|
|
1572
|
-
// without it the attach looks in the catalog's default
|
|
1573
|
-
// schema, finds nothing, and fails outright, because a
|
|
1574
|
-
// read-only attach may not create a catalog.
|
|
1575
|
-
const schema = uniqueSchema("serve");
|
|
1576
|
-
const cfg = await duckLakeConfig("dl_serve", schema);
|
|
1577
|
-
const boot = await bootstrapCatalog("dl_serve", cfg);
|
|
1578
|
-
await boot.runSQL(
|
|
1579
|
-
`CREATE OR REPLACE TABLE dl_serve.t AS SELECT 7 AS x`,
|
|
1580
|
-
);
|
|
1581
|
-
|
|
1582
|
-
const conns = await connect([
|
|
1583
|
-
{
|
|
1584
|
-
name: "dl_serve",
|
|
1585
|
-
type: "ducklake",
|
|
1586
|
-
ducklakeConnection: cfg,
|
|
1587
|
-
} as ApiConnection,
|
|
1588
|
-
]);
|
|
1589
|
-
const served = conns.get("dl_serve")!;
|
|
1590
|
-
const r = await served.runSQL(`SELECT x FROM dl_serve.t;`);
|
|
1591
|
-
expect(firstValue(r.rows)).toBe(7);
|
|
1592
|
-
},
|
|
1593
|
-
{ timeout: 60000 },
|
|
1594
|
-
);
|
|
1595
|
-
|
|
1596
|
-
it(
|
|
1597
|
-
"keeps the catalog-format preflight working for a non-default schema",
|
|
1598
|
-
async () => {
|
|
1599
|
-
if (!hasPostgresCredentials()) {
|
|
1600
|
-
console.log("Skipping: PostgreSQL not configured");
|
|
1601
|
-
return;
|
|
1602
|
-
}
|
|
1603
|
-
// The preflight reads `ducklake_metadata`, which lives in
|
|
1604
|
-
// the configured schema. Read unqualified it would MISS,
|
|
1605
|
-
// and the preflight fails SOFT (logs and returns) — so the
|
|
1606
|
-
// range check would silently stop protecting exactly the
|
|
1607
|
-
// catalogs setting this option. Poisoning the recorded
|
|
1608
|
-
// version proves the read resolves: an unqualified read
|
|
1609
|
-
// finds nothing and so raises nothing.
|
|
1610
|
-
const schema = uniqueSchema("preflight");
|
|
1611
|
-
const cfg = await duckLakeConfig("dl_pf", schema);
|
|
1612
|
-
const boot = await bootstrapCatalog("dl_pf", cfg);
|
|
1613
|
-
await boot.runSQL(
|
|
1614
|
-
`ATTACH '${pgConnString()}' AS poison (TYPE postgres);`,
|
|
1615
|
-
);
|
|
1616
|
-
await boot.runSQL(
|
|
1617
|
-
`UPDATE poison.${schema}.ducklake_metadata ` +
|
|
1618
|
-
`SET value = '9999.0' WHERE key = 'version';`,
|
|
1619
|
-
);
|
|
1620
|
-
|
|
1621
|
-
// A fresh attach of the same catalog must now refuse with
|
|
1622
|
-
// the clean typed range error rather than proceeding.
|
|
1623
|
-
let err: unknown;
|
|
1624
|
-
try {
|
|
1625
|
-
await bootstrapCatalog("dl_pf_again", cfg);
|
|
1626
|
-
} catch (e) {
|
|
1627
|
-
err = e;
|
|
1628
|
-
}
|
|
1629
|
-
// The TYPE matters: converting a deep DuckDB failure into
|
|
1630
|
-
// this typed refusal is the preflight's whole job, so a
|
|
1631
|
-
// message match alone could not tell the two apart.
|
|
1632
|
-
expect(err).toBeInstanceOf(UnsupportedCatalogFormatError);
|
|
1633
|
-
// And it must be THIS catalog's poisoned format. Asserting
|
|
1634
|
-
// the value keeps the test honest if some other schema in
|
|
1635
|
-
// the shared catalog database holds a valid catalog: an
|
|
1636
|
-
// unqualified read would report that one's format instead,
|
|
1637
|
-
// which is exactly the bug being guarded against.
|
|
1638
|
-
expect((err as Error).message).toContain("9999.0");
|
|
1639
|
-
},
|
|
1640
|
-
{ timeout: 60000 },
|
|
1641
|
-
);
|
|
1642
|
-
|
|
1643
|
-
it(
|
|
1644
|
-
"round-trips a mixed-case metadataSchema through the preflight",
|
|
1645
|
-
async () => {
|
|
1646
|
-
if (!hasPostgresCredentials()) {
|
|
1647
|
-
console.log("Skipping: PostgreSQL not configured");
|
|
1648
|
-
return;
|
|
1649
|
-
}
|
|
1650
|
-
// The validator accepts mixed case but nothing exercised it, so
|
|
1651
|
-
// this covers the create -> preflight -> attach round-trip for
|
|
1652
|
-
// such a name. Honest scope: it does NOT discriminate on the
|
|
1653
|
-
// preflight's identifier quoting — measured, unquoted resolves
|
|
1654
|
-
// too, because DuckDB matches identifiers case-insensitively.
|
|
1655
|
-
// Poisoning the version is what proves the preflight read THIS
|
|
1656
|
-
// catalog's metadata rather than missing silently.
|
|
1657
|
-
const schema = `DlMixedCase_${runId}`;
|
|
1658
|
-
createdSchemas.push(schema);
|
|
1659
|
-
const cfg = await duckLakeConfig("dl_mixed", schema);
|
|
1660
|
-
const boot = await bootstrapCatalog("dl_mixed", cfg);
|
|
1661
|
-
await boot.runSQL(
|
|
1662
|
-
`ATTACH '${pgConnString()}' AS poison_mixed (TYPE postgres);`,
|
|
1663
|
-
);
|
|
1664
|
-
await boot.runSQL(
|
|
1665
|
-
`UPDATE poison_mixed."${schema}".ducklake_metadata ` +
|
|
1666
|
-
`SET value = '9999.0' WHERE key = 'version';`,
|
|
1667
|
-
);
|
|
1668
|
-
|
|
1669
|
-
let err: unknown;
|
|
1670
|
-
try {
|
|
1671
|
-
await bootstrapCatalog("dl_mixed_again", cfg);
|
|
1672
|
-
} catch (e) {
|
|
1673
|
-
err = e;
|
|
1674
|
-
}
|
|
1675
|
-
expect(err).toBeInstanceOf(UnsupportedCatalogFormatError);
|
|
1676
|
-
expect((err as Error).message).toContain("9999.0");
|
|
1677
|
-
},
|
|
1678
|
-
{ timeout: 60000 },
|
|
1679
|
-
);
|
|
1680
|
-
|
|
1681
|
-
it(
|
|
1682
|
-
"attaches without a metadataSchema exactly as before",
|
|
1683
|
-
async () => {
|
|
1684
|
-
if (!hasPostgresCredentials()) {
|
|
1685
|
-
console.log("Skipping: PostgreSQL not configured");
|
|
1686
|
-
return;
|
|
1687
|
-
}
|
|
1688
|
-
// Back-compat: the option is a conditional append, so an
|
|
1689
|
-
// absent value emits the pre-existing ATTACH verbatim and
|
|
1690
|
-
// DuckLake uses the catalog connection's default schema.
|
|
1691
|
-
const cfg = await duckLakeConfig("dl_default");
|
|
1692
|
-
const c = await bootstrapCatalog("dl_default", cfg);
|
|
1693
|
-
await c.runSQL(
|
|
1694
|
-
`CREATE OR REPLACE TABLE dl_default.t AS SELECT 7 AS x`,
|
|
1695
|
-
);
|
|
1696
|
-
const r = await c.runSQL(`SELECT x FROM dl_default.t;`);
|
|
1697
|
-
expect(firstValue(r.rows)).toBe(7);
|
|
1698
|
-
|
|
1699
|
-
await c.runSQL(
|
|
1700
|
-
`ATTACH '${pgConnString()}' AS insp2 (TYPE postgres, READ_ONLY);`,
|
|
1701
|
-
);
|
|
1702
|
-
const inPublic = await c.runSQL(
|
|
1703
|
-
`SELECT count(*) AS n FROM insp2.public.ducklake_metadata;`,
|
|
1704
|
-
);
|
|
1705
|
-
expect(firstValue(inPublic.rows)).toBeGreaterThan(0);
|
|
1706
|
-
},
|
|
1707
|
-
{ timeout: 60000 },
|
|
1708
|
-
);
|
|
1709
|
-
});
|
|
1710
|
-
});
|
|
1711
|
-
|
|
1712
|
-
it("should throw error if DuckDB connection name conflicts with attached database", async () => {
|
|
1713
|
-
await expect(
|
|
1714
|
-
createEnvironmentConnections(
|
|
1715
|
-
[
|
|
1716
|
-
{
|
|
1717
|
-
name: "conflict_db",
|
|
1718
|
-
type: "duckdb",
|
|
1719
|
-
duckdbConnection: {
|
|
1720
|
-
attachedDatabases: [
|
|
1721
|
-
{
|
|
1722
|
-
name: "conflict_db",
|
|
1723
|
-
type: "postgres",
|
|
1724
|
-
postgresConnection: {
|
|
1725
|
-
connectionString:
|
|
1726
|
-
"postgresql://localhost/test",
|
|
1727
|
-
},
|
|
1728
|
-
},
|
|
1729
|
-
],
|
|
1730
|
-
},
|
|
1731
|
-
},
|
|
1732
|
-
],
|
|
1733
|
-
testEnvironmentPath,
|
|
1734
|
-
),
|
|
1735
|
-
).rejects.toThrow(/cannot conflict/);
|
|
1736
|
-
});
|
|
1737
|
-
|
|
1738
|
-
it("should throw error if connection name is 'duckdb'", async () => {
|
|
1739
|
-
await expect(
|
|
1740
|
-
createEnvironmentConnections(
|
|
1741
|
-
[
|
|
1742
|
-
{
|
|
1743
|
-
name: "duckdb",
|
|
1744
|
-
type: "duckdb",
|
|
1745
|
-
duckdbConnection: { attachedDatabases: [] },
|
|
1746
|
-
},
|
|
1747
|
-
],
|
|
1748
|
-
testEnvironmentPath,
|
|
1749
|
-
),
|
|
1750
|
-
).rejects.toThrow(/'duckdb' is reserved/);
|
|
1751
|
-
});
|
|
1752
|
-
|
|
1753
|
-
it("should reject DuckDB connections with no attachments", async () => {
|
|
1754
|
-
// Env-level DuckDB connections must declare at least one
|
|
1755
|
-
// attached foreign database; the empty-array case is operator
|
|
1756
|
-
// confusion (the per-package "duckdb" sandbox already covers
|
|
1757
|
-
// the plain-in-memory use case).
|
|
1758
|
-
await expect(
|
|
1759
|
-
createEnvironmentConnections(
|
|
1760
|
-
[
|
|
1761
|
-
{
|
|
1762
|
-
name: "empty_duckdb",
|
|
1763
|
-
type: "duckdb",
|
|
1764
|
-
duckdbConnection: { attachedDatabases: [] },
|
|
1765
|
-
},
|
|
1766
|
-
],
|
|
1767
|
-
testEnvironmentPath,
|
|
1768
|
-
),
|
|
1769
|
-
).rejects.toThrow(/has no attached databases/);
|
|
1770
|
-
});
|
|
1771
|
-
|
|
1772
|
-
it("should reject unsupported DuckDB connector fields", async () => {
|
|
1773
|
-
await expect(
|
|
1774
|
-
createEnvironmentConnections(
|
|
1775
|
-
[
|
|
1776
|
-
{
|
|
1777
|
-
name: "duckdb_with_setup_sql",
|
|
1778
|
-
type: "duckdb",
|
|
1779
|
-
duckdbConnection: {
|
|
1780
|
-
attachedDatabases: [],
|
|
1781
|
-
setupSQL: "INSTALL httpfs",
|
|
1782
|
-
},
|
|
1783
|
-
} as unknown as ApiConnection,
|
|
1784
|
-
],
|
|
1785
|
-
testEnvironmentPath,
|
|
1786
|
-
),
|
|
1787
|
-
).rejects.toThrow(/Unsupported DuckDB connection field/);
|
|
1788
|
-
});
|
|
1789
|
-
|
|
1790
|
-
it("should reject environment-authored DuckDB policy fields", async () => {
|
|
1791
|
-
await expect(
|
|
1792
|
-
createEnvironmentConnections(
|
|
1793
|
-
[
|
|
1794
|
-
{
|
|
1795
|
-
name: "duckdb_with_policy",
|
|
1796
|
-
type: "duckdb",
|
|
1797
|
-
duckdbConnection: {
|
|
1798
|
-
attachedDatabases: [],
|
|
1799
|
-
securityPolicy: "sandboxed",
|
|
1800
|
-
},
|
|
1801
|
-
} as unknown as ApiConnection,
|
|
1802
|
-
],
|
|
1803
|
-
testEnvironmentPath,
|
|
1804
|
-
),
|
|
1805
|
-
).rejects.toThrow(/Unsupported DuckDB connection field/);
|
|
1806
|
-
});
|
|
1807
|
-
|
|
1808
|
-
it("should preserve Snowflake private-key auth options", async () => {
|
|
1809
|
-
const { malloyConnections, releaseConnections } =
|
|
1810
|
-
await createEnvironmentConnections(
|
|
1811
|
-
[
|
|
1812
|
-
{
|
|
1813
|
-
name: "sf_private_key",
|
|
1814
|
-
type: "snowflake",
|
|
1815
|
-
snowflakeConnection: {
|
|
1816
|
-
account: "test-account",
|
|
1817
|
-
username: "test-user",
|
|
1818
|
-
privateKey:
|
|
1819
|
-
"-----BEGIN PRIVATE KEY-----MIIB-----END PRIVATE KEY-----",
|
|
1820
|
-
warehouse: "test-warehouse",
|
|
1821
|
-
},
|
|
1822
|
-
},
|
|
1823
|
-
],
|
|
1824
|
-
testEnvironmentPath,
|
|
1825
|
-
);
|
|
1826
|
-
|
|
1827
|
-
try {
|
|
1828
|
-
const connection = malloyConnections.get(
|
|
1829
|
-
"sf_private_key",
|
|
1830
|
-
) as unknown as { connOptions: Record<string, unknown> };
|
|
1831
|
-
expect(connection.connOptions.authenticator).toBe(
|
|
1832
|
-
"SNOWFLAKE_JWT",
|
|
1833
|
-
);
|
|
1834
|
-
expect(connection.connOptions.privateKey).toContain(
|
|
1835
|
-
"BEGIN PRIVATE KEY",
|
|
1836
|
-
);
|
|
1837
|
-
} finally {
|
|
1838
|
-
await releaseConnections();
|
|
1839
|
-
}
|
|
1840
|
-
});
|
|
1841
|
-
|
|
1842
|
-
it("should translate Trino Peaka credentials to core extraCredential", () => {
|
|
1843
|
-
const assembled = assembleEnvironmentConnections(
|
|
1844
|
-
[
|
|
1845
|
-
{
|
|
1846
|
-
name: "trino_peaka",
|
|
1847
|
-
type: "trino",
|
|
1848
|
-
trinoConnection: {
|
|
1849
|
-
server: "https://example.com",
|
|
1850
|
-
port: 443,
|
|
1851
|
-
catalog: "catalog",
|
|
1852
|
-
schema: "schema",
|
|
1853
|
-
user: "user",
|
|
1854
|
-
peakaKey: "peaka-secret",
|
|
1855
|
-
},
|
|
1856
|
-
},
|
|
1857
|
-
],
|
|
1858
|
-
testEnvironmentPath,
|
|
1859
|
-
);
|
|
1860
|
-
|
|
1861
|
-
expect(
|
|
1862
|
-
assembled.pojo.connections.trino_peaka.extraCredential,
|
|
1863
|
-
).toEqual({ peakaKey: "peaka-secret" });
|
|
1864
|
-
expect(
|
|
1865
|
-
assembled.pojo.connections.trino_peaka.extraConfig,
|
|
1866
|
-
).toBeUndefined();
|
|
1867
|
-
});
|
|
1868
|
-
|
|
1869
|
-
it("should validate environment-level BigQuery service account keys", () => {
|
|
1870
|
-
expect(() =>
|
|
1871
|
-
assembleEnvironmentConnections(
|
|
1872
|
-
[
|
|
1873
|
-
{
|
|
1874
|
-
name: "bq_invalid",
|
|
1875
|
-
type: "bigquery",
|
|
1876
|
-
bigqueryConnection: {
|
|
1877
|
-
defaultProjectId: "test-environment",
|
|
1878
|
-
serviceAccountKeyJson: '{"invalid":"key"}',
|
|
1879
|
-
},
|
|
1880
|
-
},
|
|
1881
|
-
],
|
|
1882
|
-
testEnvironmentPath,
|
|
1883
|
-
),
|
|
1884
|
-
).toThrow(/missing "type" field/);
|
|
1885
|
-
});
|
|
1886
|
-
|
|
1887
|
-
it("should preserve PGSSLMODE for environment-level Postgres", () => {
|
|
1888
|
-
const previousPgSslMode = process.env.PGSSLMODE;
|
|
1889
|
-
process.env.PGSSLMODE = "require";
|
|
1890
|
-
try {
|
|
1891
|
-
const assembled = assembleEnvironmentConnections(
|
|
1892
|
-
[
|
|
1893
|
-
{
|
|
1894
|
-
name: "pg_ssl",
|
|
1895
|
-
type: "postgres",
|
|
1896
|
-
postgresConnection: {
|
|
1897
|
-
host: "localhost",
|
|
1898
|
-
port: 5432,
|
|
1899
|
-
userName: "user",
|
|
1900
|
-
password: "pass",
|
|
1901
|
-
databaseName: "db",
|
|
1902
|
-
},
|
|
1903
|
-
},
|
|
1904
|
-
],
|
|
1905
|
-
testEnvironmentPath,
|
|
1906
|
-
);
|
|
1907
|
-
|
|
1908
|
-
expect(
|
|
1909
|
-
assembled.pojo.connections.pg_ssl.connectionString,
|
|
1910
|
-
).toContain("sslmode=require");
|
|
1911
|
-
} finally {
|
|
1912
|
-
if (previousPgSslMode === undefined) {
|
|
1913
|
-
delete process.env.PGSSLMODE;
|
|
1914
|
-
} else {
|
|
1915
|
-
process.env.PGSSLMODE = previousPgSslMode;
|
|
1916
|
-
}
|
|
1917
|
-
}
|
|
1918
|
-
});
|
|
1919
|
-
|
|
1920
|
-
it("should use environment-root-relative file paths for environment-level DuckDB", async () => {
|
|
1921
|
-
const insideCsvPath = path.join(testEnvironmentPath, "inside.csv");
|
|
1922
|
-
await fs.writeFile(insideCsvPath, "id\n1\n");
|
|
1923
|
-
|
|
1924
|
-
const minimalGcsAttachment = {
|
|
1925
|
-
name: "test_gcs",
|
|
1926
|
-
type: "gcs" as const,
|
|
1927
|
-
gcsConnection: {
|
|
1928
|
-
keyId: "test-key-id",
|
|
1929
|
-
secret: "test-secret",
|
|
1930
|
-
},
|
|
1931
|
-
};
|
|
1932
|
-
|
|
1933
|
-
const { malloyConnections } = await createEnvironmentConnections(
|
|
1934
|
-
[
|
|
1935
|
-
{
|
|
1936
|
-
name: "environment_scoped_duckdb",
|
|
1937
|
-
type: "duckdb",
|
|
1938
|
-
duckdbConnection: {
|
|
1939
|
-
attachedDatabases: [minimalGcsAttachment],
|
|
1940
|
-
},
|
|
1941
|
-
},
|
|
1942
|
-
],
|
|
1943
|
-
testEnvironmentPath,
|
|
1944
|
-
);
|
|
1945
|
-
|
|
1946
|
-
const connection = malloyConnections.get(
|
|
1947
|
-
"environment_scoped_duckdb",
|
|
1948
|
-
) as DuckDBConnection;
|
|
1949
|
-
createdConnections.push(connection);
|
|
1950
|
-
|
|
1951
|
-
const assembled = assembleEnvironmentConnections(
|
|
1952
|
-
[
|
|
1953
|
-
{
|
|
1954
|
-
name: "environment_scoped_duckdb",
|
|
1955
|
-
type: "duckdb",
|
|
1956
|
-
duckdbConnection: {
|
|
1957
|
-
attachedDatabases: [minimalGcsAttachment],
|
|
1958
|
-
},
|
|
1959
|
-
},
|
|
1960
|
-
],
|
|
1961
|
-
testEnvironmentPath,
|
|
1962
|
-
);
|
|
1963
|
-
expect(
|
|
1964
|
-
assembled.pojo.connections.environment_scoped_duckdb
|
|
1965
|
-
.workingDirectory,
|
|
1966
|
-
).toBeUndefined();
|
|
1967
|
-
expect(
|
|
1968
|
-
assembled.pojo.connections.environment_scoped_duckdb
|
|
1969
|
-
.securityPolicy,
|
|
1970
|
-
).toBeUndefined();
|
|
1971
|
-
|
|
1972
|
-
await expect(
|
|
1973
|
-
connection.runSQL("SELECT * FROM read_csv_auto('inside.csv')"),
|
|
1974
|
-
).resolves.toBeDefined();
|
|
1975
|
-
});
|
|
1976
|
-
|
|
1977
|
-
it("should keep external access available for federated DuckDB entries", () => {
|
|
1978
|
-
const assembled = assembleEnvironmentConnections(
|
|
1979
|
-
[
|
|
1980
|
-
{
|
|
1981
|
-
name: "federated_duckdb",
|
|
1982
|
-
type: "duckdb",
|
|
1983
|
-
duckdbConnection: {
|
|
1984
|
-
attachedDatabases: [
|
|
1985
|
-
{
|
|
1986
|
-
name: "pg",
|
|
1987
|
-
type: "postgres",
|
|
1988
|
-
postgresConnection: {
|
|
1989
|
-
host: "localhost",
|
|
1990
|
-
port: 5432,
|
|
1991
|
-
userName: "user",
|
|
1992
|
-
password: "pass",
|
|
1993
|
-
databaseName: "db",
|
|
1994
|
-
},
|
|
1995
|
-
},
|
|
1996
|
-
],
|
|
1997
|
-
},
|
|
1998
|
-
},
|
|
1999
|
-
],
|
|
2000
|
-
testEnvironmentPath,
|
|
2001
|
-
);
|
|
2002
|
-
|
|
2003
|
-
const entry = assembled.pojo.connections.federated_duckdb;
|
|
2004
|
-
expect(entry.securityPolicy).toBeUndefined();
|
|
2005
|
-
expect(entry.enableExternalAccess).toBeUndefined();
|
|
2006
|
-
expect(entry.allowedDirectories).toBeUndefined();
|
|
2007
|
-
});
|
|
2008
|
-
|
|
2009
|
-
it("should keep external access available for MotherDuck entries", () => {
|
|
2010
|
-
const assembled = assembleEnvironmentConnections(
|
|
2011
|
-
[
|
|
2012
|
-
{
|
|
2013
|
-
name: "md",
|
|
2014
|
-
type: "motherduck",
|
|
2015
|
-
motherduckConnection: {
|
|
2016
|
-
accessToken: "token",
|
|
2017
|
-
database: "db",
|
|
2018
|
-
},
|
|
2019
|
-
},
|
|
2020
|
-
],
|
|
2021
|
-
testEnvironmentPath,
|
|
2022
|
-
);
|
|
2023
|
-
|
|
2024
|
-
const entry = assembled.pojo.connections.md;
|
|
2025
|
-
expect(entry.securityPolicy).toBeUndefined();
|
|
2026
|
-
expect(entry.enableExternalAccess).toBeUndefined();
|
|
2027
|
-
expect(entry.allowedDirectories).toBeUndefined();
|
|
2028
|
-
});
|
|
2029
|
-
|
|
2030
|
-
it("should handle already attached database gracefully", async () => {
|
|
2031
|
-
if (!hasPostgresCredentials()) {
|
|
2032
|
-
console.log("Skipping: PostgreSQL credentials not configured");
|
|
2033
|
-
return;
|
|
2034
|
-
}
|
|
2035
|
-
|
|
2036
|
-
const postgresAttachment: AttachedDatabase = {
|
|
2037
|
-
name: "pg_duplicate",
|
|
2038
|
-
type: "postgres",
|
|
2039
|
-
postgresConnection: {
|
|
2040
|
-
host: process.env.POSTGRES_TEST_HOST,
|
|
2041
|
-
port: parseInt(process.env.POSTGRES_TEST_PORT || "5432"),
|
|
2042
|
-
userName: process.env.POSTGRES_TEST_USER!,
|
|
2043
|
-
password: process.env.POSTGRES_TEST_PASSWORD!,
|
|
2044
|
-
databaseName: process.env.POSTGRES_TEST_DATABASE,
|
|
2045
|
-
},
|
|
2046
|
-
};
|
|
2047
|
-
|
|
2048
|
-
const { malloyConnections } = await createEnvironmentConnections(
|
|
2049
|
-
[
|
|
2050
|
-
{
|
|
2051
|
-
name: "duckdb_duplicate_test",
|
|
2052
|
-
type: "duckdb",
|
|
2053
|
-
duckdbConnection: {
|
|
2054
|
-
attachedDatabases: [
|
|
2055
|
-
postgresAttachment,
|
|
2056
|
-
postgresAttachment,
|
|
2057
|
-
],
|
|
2058
|
-
},
|
|
2059
|
-
},
|
|
2060
|
-
],
|
|
2061
|
-
testEnvironmentPath,
|
|
2062
|
-
);
|
|
2063
|
-
|
|
2064
|
-
const connection = malloyConnections.get(
|
|
2065
|
-
"duckdb_duplicate_test",
|
|
2066
|
-
) as DuckDBConnection;
|
|
2067
|
-
createdConnections.push(connection);
|
|
2068
|
-
expect(connection).toBeDefined();
|
|
2069
|
-
});
|
|
2070
|
-
});
|
|
2071
|
-
});
|
|
2072
|
-
|
|
2073
|
-
describe("testConnectionConfig", () => {
|
|
2074
|
-
it(
|
|
2075
|
-
"should successfully test valid PostgreSQL connection",
|
|
2076
|
-
async () => {
|
|
2077
|
-
if (!hasPostgresCredentials()) {
|
|
2078
|
-
console.log("Skipping: PostgreSQL credentials not configured");
|
|
2079
|
-
return;
|
|
2080
|
-
}
|
|
2081
|
-
|
|
2082
|
-
const result = await testConnectionConfig({
|
|
2083
|
-
name: "test_postgres",
|
|
2084
|
-
type: "postgres",
|
|
2085
|
-
postgresConnection: {
|
|
2086
|
-
host: process.env.POSTGRES_TEST_HOST,
|
|
2087
|
-
port: parseInt(process.env.POSTGRES_TEST_PORT || "5432"),
|
|
2088
|
-
userName: process.env.POSTGRES_TEST_USER!,
|
|
2089
|
-
password: process.env.POSTGRES_TEST_PASSWORD!,
|
|
2090
|
-
databaseName: process.env.POSTGRES_TEST_DATABASE,
|
|
2091
|
-
},
|
|
2092
|
-
});
|
|
2093
|
-
|
|
2094
|
-
expect(result.status).toBe("ok");
|
|
2095
|
-
expect(result.errorMessage).toBe("");
|
|
2096
|
-
},
|
|
2097
|
-
{ timeout: 30000 },
|
|
2098
|
-
);
|
|
2099
|
-
|
|
2100
|
-
it(
|
|
2101
|
-
"should fail for invalid PostgreSQL credentials",
|
|
2102
|
-
async () => {
|
|
2103
|
-
const result = await testConnectionConfig({
|
|
2104
|
-
name: "test_postgres_invalid",
|
|
2105
|
-
type: "postgres",
|
|
2106
|
-
postgresConnection: {
|
|
2107
|
-
host: process.env.POSTGRES_TEST_HOST,
|
|
2108
|
-
port: parseInt(process.env.POSTGRES_TEST_PORT || "5432"),
|
|
2109
|
-
userName: "invalid_user",
|
|
2110
|
-
password: "invalid_password",
|
|
2111
|
-
databaseName: "nonexistent",
|
|
2112
|
-
},
|
|
2113
|
-
});
|
|
2114
|
-
|
|
2115
|
-
expect(result.status).toBe("failed");
|
|
2116
|
-
expect(result.errorMessage).toBeDefined();
|
|
2117
|
-
},
|
|
2118
|
-
{ timeout: 30000 },
|
|
2119
|
-
);
|
|
2120
|
-
|
|
2121
|
-
it("should fail for missing connection name", async () => {
|
|
2122
|
-
const result = await testConnectionConfig({
|
|
2123
|
-
name: "",
|
|
2124
|
-
type: "postgres",
|
|
2125
|
-
postgresConnection: {},
|
|
2126
|
-
});
|
|
2127
|
-
|
|
2128
|
-
expect(result.status).toBe("failed");
|
|
2129
|
-
expect(result.errorMessage).toContain("name is required");
|
|
2130
|
-
});
|
|
2131
|
-
});
|
|
2132
|
-
|
|
2133
|
-
describe("SQL injection prevention", () => {
|
|
2134
|
-
it("should properly escape special characters in credentials", async () => {
|
|
2135
|
-
if (!hasPostgresCredentials()) {
|
|
2136
|
-
console.log("Skipping: PostgreSQL credentials not configured");
|
|
2137
|
-
return;
|
|
2138
|
-
}
|
|
2139
|
-
|
|
2140
|
-
const { malloyConnections } = await createEnvironmentConnections(
|
|
2141
|
-
[
|
|
2142
|
-
{
|
|
2143
|
-
name: "duckdb_special_chars",
|
|
2144
|
-
type: "duckdb",
|
|
2145
|
-
duckdbConnection: {
|
|
2146
|
-
attachedDatabases: [
|
|
2147
|
-
{
|
|
2148
|
-
name: "pg_special",
|
|
2149
|
-
type: "postgres",
|
|
2150
|
-
postgresConnection: {
|
|
2151
|
-
host: process.env.POSTGRES_TEST_HOST,
|
|
2152
|
-
port: parseInt(
|
|
2153
|
-
process.env.POSTGRES_TEST_PORT || "5432",
|
|
2154
|
-
),
|
|
2155
|
-
userName: process.env.POSTGRES_TEST_USER!,
|
|
2156
|
-
password: process.env.POSTGRES_TEST_PASSWORD!,
|
|
2157
|
-
databaseName: process.env.POSTGRES_TEST_DATABASE,
|
|
2158
|
-
},
|
|
2159
|
-
},
|
|
2160
|
-
],
|
|
2161
|
-
},
|
|
2162
|
-
},
|
|
2163
|
-
],
|
|
2164
|
-
testEnvironmentPath,
|
|
2165
|
-
);
|
|
2166
|
-
|
|
2167
|
-
const connection = malloyConnections.get(
|
|
2168
|
-
"duckdb_special_chars",
|
|
2169
|
-
) as DuckDBConnection;
|
|
2170
|
-
createdConnections.push(connection);
|
|
2171
|
-
expect(connection).toBeDefined();
|
|
2172
|
-
});
|
|
2173
|
-
});
|
|
2174
|
-
|
|
2175
|
-
describe("connection attributes", () => {
|
|
2176
|
-
it(
|
|
2177
|
-
"should return correct attributes for DuckDB connection",
|
|
2178
|
-
async () => {
|
|
2179
|
-
if (!hasPostgresCredentials()) {
|
|
2180
|
-
console.log("Skipping: PostgreSQL credentials not configured");
|
|
2181
|
-
return;
|
|
2182
|
-
}
|
|
2183
|
-
|
|
2184
|
-
const { apiConnections } = await createEnvironmentConnections(
|
|
2185
|
-
[
|
|
2186
|
-
{
|
|
2187
|
-
name: "duckdb_attrs",
|
|
2188
|
-
type: "duckdb",
|
|
2189
|
-
duckdbConnection: {
|
|
2190
|
-
attachedDatabases: [
|
|
2191
|
-
{
|
|
2192
|
-
name: "pg_attrs",
|
|
2193
|
-
type: "postgres",
|
|
2194
|
-
postgresConnection: {
|
|
2195
|
-
host: process.env.POSTGRES_TEST_HOST,
|
|
2196
|
-
port: parseInt(
|
|
2197
|
-
process.env.POSTGRES_TEST_PORT || "5432",
|
|
2198
|
-
),
|
|
2199
|
-
userName: process.env.POSTGRES_TEST_USER!,
|
|
2200
|
-
password: process.env.POSTGRES_TEST_PASSWORD!,
|
|
2201
|
-
databaseName:
|
|
2202
|
-
process.env.POSTGRES_TEST_DATABASE,
|
|
2203
|
-
},
|
|
2204
|
-
},
|
|
2205
|
-
],
|
|
2206
|
-
},
|
|
2207
|
-
},
|
|
2208
|
-
],
|
|
2209
|
-
testEnvironmentPath,
|
|
2210
|
-
);
|
|
2211
|
-
|
|
2212
|
-
const connection = apiConnections[0];
|
|
2213
|
-
expect(connection.attributes).toBeDefined();
|
|
2214
|
-
expect(connection.attributes?.dialectName).toBe("duckdb");
|
|
2215
|
-
expect(typeof connection.attributes?.canPersist).toBe("boolean");
|
|
2216
|
-
expect(typeof connection.attributes?.canStream).toBe("boolean");
|
|
2217
|
-
expect(typeof connection.attributes?.isPool).toBe("boolean");
|
|
2218
|
-
},
|
|
2219
|
-
{ timeout: 30000 },
|
|
2220
|
-
);
|
|
2221
|
-
});
|
|
2222
|
-
});
|
|
2223
|
-
|
|
2224
|
-
describe("buildProxiedSslQuery", () => {
|
|
2225
|
-
const ORIG_CA = process.env.NODE_EXTRA_CA_CERTS;
|
|
2226
|
-
afterEach(() => {
|
|
2227
|
-
if (ORIG_CA === undefined) delete process.env.NODE_EXTRA_CA_CERTS;
|
|
2228
|
-
else process.env.NODE_EXTRA_CA_CERTS = ORIG_CA;
|
|
2229
|
-
});
|
|
2230
|
-
|
|
2231
|
-
it("defaults to no-verify when sslmode is unset (force-SSL target isn't rejected for plaintext)", () => {
|
|
2232
|
-
expect(buildProxiedSslQuery("conn")).toBe("?sslmode=no-verify");
|
|
2233
|
-
});
|
|
2234
|
-
|
|
2235
|
-
it("returns no-verify for explicit no-verify", () => {
|
|
2236
|
-
expect(buildProxiedSslQuery("conn", "no-verify")).toBe(
|
|
2237
|
-
"?sslmode=no-verify",
|
|
2238
|
-
);
|
|
2239
|
-
});
|
|
2240
|
-
|
|
2241
|
-
it("emits an explicit sslmode=disable, never an empty query", () => {
|
|
2242
|
-
// Regression: an empty query lets pg fall back to the deployment's PGSSLMODE
|
|
2243
|
-
// env, which would verify and fail the 127.0.0.1 hostname check.
|
|
2244
|
-
expect(buildProxiedSslQuery("conn", "disable")).toBe("?sslmode=disable");
|
|
2245
|
-
});
|
|
2246
|
-
|
|
2247
|
-
it("builds a chain-verifying, hostname-skipped query for verify-ca", () => {
|
|
2248
|
-
process.env.NODE_EXTRA_CA_CERTS = "/etc/ssl/certs/rds bundle.pem";
|
|
2249
|
-
const q = buildProxiedSslQuery("conn", "verify-ca");
|
|
2250
|
-
expect(q).toContain("uselibpqcompat=true");
|
|
2251
|
-
expect(q).toContain("sslmode=verify-ca");
|
|
2252
|
-
expect(q).toContain(
|
|
2253
|
-
`sslrootcert=${encodeURIComponent("/etc/ssl/certs/rds bundle.pem")}`,
|
|
2254
|
-
);
|
|
2255
|
-
});
|
|
2256
|
-
|
|
2257
|
-
it("throws for verify-ca when no CA bundle is available", () => {
|
|
2258
|
-
delete process.env.NODE_EXTRA_CA_CERTS;
|
|
2259
|
-
expect(() => buildProxiedSslQuery("conn", "verify-ca")).toThrow(
|
|
2260
|
-
/no trusted CA bundle/i,
|
|
2261
|
-
);
|
|
2262
|
-
});
|
|
2263
|
-
|
|
2264
|
-
it("throws for an unsupported sslmode", () => {
|
|
2265
|
-
expect(() => buildProxiedSslQuery("conn", "require")).toThrow(
|
|
2266
|
-
/unsupported sslmode/i,
|
|
2267
|
-
);
|
|
2268
|
-
});
|
|
2269
|
-
|
|
2270
|
-
it("verify-ca connectionString parses to an ssl config (pg-connection-string version guard)", async () => {
|
|
2271
|
-
// uselibpqcompat=true (pg-connection-string >= 2.7) is what turns verify-ca
|
|
2272
|
-
// into "verify chain, skip hostname". If pg is ever downgraded below that
|
|
2273
|
-
// floor, sslmode would be ignored — this guard fails loudly instead.
|
|
2274
|
-
const caPath = path.join(os.tmpdir(), `proxy-ca-${process.pid}.pem`);
|
|
2275
|
-
await fs.writeFile(
|
|
2276
|
-
caPath,
|
|
2277
|
-
"-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n",
|
|
2278
|
-
);
|
|
2279
|
-
process.env.NODE_EXTRA_CA_CERTS = caPath;
|
|
2280
|
-
try {
|
|
2281
|
-
const q = buildProxiedSslQuery("conn", "verify-ca");
|
|
2282
|
-
const { parse } = await import("pg-connection-string");
|
|
2283
|
-
const parsed = parse(`postgresql://127.0.0.1:5432/db${q}`);
|
|
2284
|
-
expect(parsed.ssl).toBeTruthy();
|
|
2285
|
-
expect(typeof parsed.ssl).toBe("object");
|
|
2286
|
-
} finally {
|
|
2287
|
-
await fs.rm(caPath, { force: true });
|
|
2288
|
-
}
|
|
2289
|
-
});
|
|
2290
|
-
});
|
|
2291
|
-
|
|
2292
|
-
describe("resolveProxiedTls", () => {
|
|
2293
|
-
it("routes verify-full to a raw ssl object with servername = the real host", () => {
|
|
2294
|
-
// verify-full (chain + hostname) can't ride the connectionString through a
|
|
2295
|
-
// tunnel — it needs ssl.servername set to the real DB host so pg checks the
|
|
2296
|
-
// cert's SAN against it, not against the 127.0.0.1 socket address.
|
|
2297
|
-
const { query, ssl } = resolveProxiedTls(
|
|
2298
|
-
"conn",
|
|
2299
|
-
"db.internal.example.com",
|
|
2300
|
-
"verify-full",
|
|
2301
|
-
);
|
|
2302
|
-
expect(query).toBe("");
|
|
2303
|
-
expect(ssl).toEqual({
|
|
2304
|
-
servername: "db.internal.example.com",
|
|
2305
|
-
rejectUnauthorized: true,
|
|
2306
|
-
});
|
|
2307
|
-
});
|
|
2308
|
-
|
|
2309
|
-
it("throws for verify-full with no host (servername would be undefined)", () => {
|
|
2310
|
-
expect(() => resolveProxiedTls("conn", undefined, "verify-full")).toThrow(
|
|
2311
|
-
/verify-full.*no host/i,
|
|
2312
|
-
);
|
|
2313
|
-
});
|
|
2314
|
-
|
|
2315
|
-
it("routes the connectionString-param modes through buildProxiedSslQuery with no ssl object", () => {
|
|
2316
|
-
// verify-ca needs a CA bundle to build its query, so set one for the loop.
|
|
2317
|
-
const prior = process.env.NODE_EXTRA_CA_CERTS;
|
|
2318
|
-
process.env.NODE_EXTRA_CA_CERTS = "/etc/ssl/certs/bundle.pem";
|
|
2319
|
-
try {
|
|
2320
|
-
for (const mode of ["disable", "no-verify", "verify-ca", undefined]) {
|
|
2321
|
-
const { query, ssl } = resolveProxiedTls("conn", "127.0.0.1", mode);
|
|
2322
|
-
expect(query).toBe(buildProxiedSslQuery("conn", mode));
|
|
2323
|
-
expect(ssl).toBeUndefined();
|
|
2324
|
-
}
|
|
2325
|
-
} finally {
|
|
2326
|
-
if (prior === undefined) delete process.env.NODE_EXTRA_CA_CERTS;
|
|
2327
|
-
else process.env.NODE_EXTRA_CA_CERTS = prior;
|
|
2328
|
-
}
|
|
2329
|
-
});
|
|
2330
|
-
|
|
2331
|
-
it("verify-full: a bare connectionString does not clobber the explicit ssl object (pg merge guard)", async () => {
|
|
2332
|
-
// Load-bearing invariant: pg's ConnectionParameters does
|
|
2333
|
-
// Object.assign({}, config, parse(connectionString)). A verify-full
|
|
2334
|
-
// connectionString carries no sslmode, so pg-connection-string adds no
|
|
2335
|
-
// `ssl` key and our explicit { servername, rejectUnauthorized } survives
|
|
2336
|
-
// the merge. If a future pg bump changes this — or we accidentally emit an
|
|
2337
|
-
// sslmode into the query — this guard fails loudly instead of silently
|
|
2338
|
-
// downgrading verify-full.
|
|
2339
|
-
const { query, ssl } = resolveProxiedTls(
|
|
2340
|
-
"conn",
|
|
2341
|
-
"db.internal.example.com",
|
|
2342
|
-
"verify-full",
|
|
2343
|
-
);
|
|
2344
|
-
const { default: ConnectionParameters } = await import(
|
|
2345
|
-
"pg/lib/connection-parameters.js"
|
|
2346
|
-
);
|
|
2347
|
-
const params = new (ConnectionParameters as unknown as new (
|
|
2348
|
-
c: unknown,
|
|
2349
|
-
) => {
|
|
2350
|
-
ssl: unknown;
|
|
2351
|
-
})({
|
|
2352
|
-
connectionString: `postgresql://u:p@127.0.0.1:5432/db${query}`,
|
|
2353
|
-
ssl,
|
|
2354
|
-
});
|
|
2355
|
-
expect(params.ssl).toEqual({
|
|
2356
|
-
servername: "db.internal.example.com",
|
|
2357
|
-
rejectUnauthorized: true,
|
|
2358
|
-
});
|
|
2359
|
-
});
|
|
2360
|
-
});
|