@malloy-publisher/server 0.0.235 → 0.0.237
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +83 -27
- package/dist/app/assets/{EnvironmentPage-BsAnavYN.js → EnvironmentPage-DaP6CpgK.js} +1 -1
- package/dist/app/assets/{HomePage-CADE138j.js → HomePage-C9yNw3iu.js} +1 -1
- package/dist/app/assets/{LightMode-Cfh7KzN8.js → LightMode-Be4-G-dX.js} +1 -1
- package/dist/app/assets/{MainPage-CO3pRlnV.js → MainPage-J_rU2SgI.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-p9YjkRXZ.js → MaterializationsPage-DHgPI4MO.js} +1 -1
- package/dist/app/assets/ModelPage-BzXfjTu5.js +1 -0
- package/dist/app/assets/{PackagePage-e4kN75YR.js → PackagePage-ByoSTLJ4.js} +1 -1
- package/dist/app/assets/RouteError-CrSeKN2r.js +1 -0
- package/dist/app/assets/{ThemeEditorPage-CciagFTq.js → ThemeEditorPage-Btxnmhtk.js} +1 -1
- package/dist/app/assets/{WorkbookPage-DNWmkCXa.js → WorkbookPage-D6JLPnWO.js} +1 -1
- package/dist/app/assets/{core-Rj_4rRnA.es-BZyvITuO.js → core-BSRgt1mG.es-Bc_D2Lq4.js} +1 -1
- package/dist/app/assets/{index-CH2AcDzc.js → index-BRijWj6K.js} +15 -15
- package/dist/app/assets/{index-DQa463gC.js → index-BX4D94xw.js} +1 -1
- package/dist/app/assets/{index-VBbcc8s6.js → index-Cvy0xtb7.js} +1 -1
- package/dist/app/assets/{index-DQpV7MyA.js → index-DMAhIJNC.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/package_load_worker.mjs +59 -20
- package/dist/server.mjs +1342 -616
- package/package.json +8 -2
- package/.eslintrc.json +0 -44
- package/.prettierignore +0 -1
- package/.prettierrc +0 -4
- package/build.ts +0 -97
- package/dist/app/assets/ModelPage-C1OSTv-x.js +0 -1
- package/dist/app/assets/RouteError-CzbfOkng.js +0 -1
- package/eslint.config.mjs +0 -76
- package/k6-tests/README.md +0 -144
- package/k6-tests/bun.lock +0 -797
- package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
- package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
- package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
- package/k6-tests/load-test/load-test-crud.ts +0 -142
- package/k6-tests/load-test/load-test.ts +0 -784
- package/k6-tests/package.json +0 -27
- package/k6-tests/scripts/run-with-otel.sh +0 -50
- package/k6-tests/smoke-test/smoke-test.ts +0 -158
- package/k6-tests/tsconfig.json +0 -29
- package/k6-tests/utils/client_factory.ts +0 -128
- package/k6-tests/utils/common.ts +0 -489
- package/k6-tests/utils/logger.ts +0 -62
- package/k6-tests/utils/types.d.ts +0 -35
- package/malloy_mcp.dxt +0 -0
- package/publisher.config.example.bigquery.json +0 -33
- package/publisher.config.example.duckdb.json +0 -23
- package/publisher.config.json +0 -23
- package/scripts/bake-duckdb-extensions.js +0 -110
- package/src/config.spec.ts +0 -1535
- package/src/config.theme.spec.ts +0 -178
- package/src/config.ts +0 -1142
- package/src/constants.ts +0 -105
- package/src/controller/compile.controller.ts +0 -40
- package/src/controller/connection.controller.spec.ts +0 -849
- package/src/controller/connection.controller.ts +0 -901
- package/src/controller/database.controller.ts +0 -24
- package/src/controller/materialization.controller.spec.ts +0 -293
- package/src/controller/materialization.controller.ts +0 -319
- package/src/controller/model.controller.ts +0 -164
- package/src/controller/package.controller.spec.ts +0 -273
- package/src/controller/package.controller.ts +0 -306
- package/src/controller/query.controller.ts +0 -152
- package/src/controller/theme.controller.ts +0 -83
- package/src/controller/watch-mode.controller.ts +0 -243
- package/src/data_styles.ts +0 -71
- package/src/default-publisher.config.json +0 -23
- package/src/ducklake_version.spec.ts +0 -163
- package/src/ducklake_version.ts +0 -153
- package/src/errors.spec.ts +0 -119
- package/src/errors.ts +0 -237
- package/src/filter_deprecation.spec.ts +0 -64
- package/src/filter_deprecation.ts +0 -42
- package/src/health.spec.ts +0 -123
- package/src/health.ts +0 -233
- package/src/heap_check.spec.ts +0 -144
- package/src/heap_check.ts +0 -144
- package/src/instrumentation.ts +0 -214
- package/src/json_utils.spec.ts +0 -51
- package/src/json_utils.ts +0 -33
- package/src/logger.spec.ts +0 -210
- package/src/logger.ts +0 -251
- package/src/malloy_pin_prereqs.spec.ts +0 -25
- package/src/materialization_metrics.spec.ts +0 -169
- package/src/materialization_metrics.ts +0 -346
- package/src/mcp/error_messages.spec.ts +0 -35
- package/src/mcp/error_messages.ts +0 -183
- package/src/mcp/handler_utils.spec.ts +0 -108
- package/src/mcp/handler_utils.ts +0 -251
- package/src/mcp/mcp_constants.ts +0 -6
- package/src/mcp/query_envelope.spec.ts +0 -229
- package/src/mcp/query_envelope.ts +0 -240
- package/src/mcp/server.protocol.spec.ts +0 -250
- package/src/mcp/server.ts +0 -87
- package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
- package/src/mcp/skills/build_skills_bundle.ts +0 -176
- package/src/mcp/skills/skills_bundle.json +0 -1
- package/src/mcp/skills/skills_bundle.spec.ts +0 -171
- package/src/mcp/tool_response.spec.ts +0 -108
- package/src/mcp/tool_response.ts +0 -138
- package/src/mcp/tools/compile_tool.spec.ts +0 -315
- package/src/mcp/tools/compile_tool.ts +0 -208
- package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
- package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
- package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
- package/src/mcp/tools/docs_search_tool.ts +0 -128
- package/src/mcp/tools/embedding_index.spec.ts +0 -1236
- package/src/mcp/tools/embedding_index.ts +0 -808
- package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
- package/src/mcp/tools/execute_query_tool.ts +0 -316
- package/src/mcp/tools/get_context_eval.ts +0 -275
- package/src/mcp/tools/get_context_tool.spec.ts +0 -592
- package/src/mcp/tools/get_context_tool.ts +0 -585
- package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
- package/src/mcp/tools/reload_package_tool.ts +0 -132
- package/src/mcp_config.spec.ts +0 -919
- package/src/mcp_config.ts +0 -425
- package/src/oom_guards.integration.spec.ts +0 -269
- package/src/package_load/package_load_pool.spec.ts +0 -252
- package/src/package_load/package_load_pool.ts +0 -931
- package/src/package_load/package_load_worker.ts +0 -1034
- package/src/package_load/protocol.ts +0 -364
- package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
- package/src/package_load/rpc_wait_accountant.ts +0 -76
- package/src/package_load_metrics.spec.ts +0 -114
- package/src/package_load_metrics.ts +0 -127
- package/src/path_safety.spec.ts +0 -158
- package/src/path_safety.ts +0 -146
- package/src/pg_helpers.spec.ts +0 -223
- package/src/pg_helpers.ts +0 -52
- package/src/query_cap_metrics.spec.ts +0 -89
- package/src/query_cap_metrics.ts +0 -116
- package/src/query_concurrency.spec.ts +0 -247
- package/src/query_concurrency.ts +0 -237
- package/src/query_metadata_metrics.ts +0 -49
- package/src/query_param_utils.ts +0 -29
- package/src/query_timeout.spec.ts +0 -224
- package/src/query_timeout.ts +0 -179
- package/src/runtime/publisher.js +0 -328
- package/src/server-old.ts +0 -988
- package/src/server.ts +0 -2059
- package/src/service/annotations.spec.ts +0 -118
- package/src/service/annotations.ts +0 -91
- package/src/service/authorize.spec.ts +0 -154
- package/src/service/authorize.ts +0 -498
- package/src/service/authorize_integration.spec.ts +0 -2041
- package/src/service/build_plan.spec.ts +0 -685
- package/src/service/build_plan.ts +0 -797
- package/src/service/compile_authorize.spec.ts +0 -85
- package/src/service/compile_fragment_techniques.spec.ts +0 -156
- package/src/service/connection.spec.ts +0 -2360
- package/src/service/connection.ts +0 -2187
- package/src/service/connection_config.spec.ts +0 -734
- package/src/service/connection_config.ts +0 -899
- package/src/service/connection_federation.spec.ts +0 -184
- package/src/service/connection_fingerprint.spec.ts +0 -102
- package/src/service/connection_service.spec.ts +0 -711
- package/src/service/connection_service.ts +0 -291
- package/src/service/cron_evaluator.spec.ts +0 -79
- package/src/service/cron_evaluator.ts +0 -105
- package/src/service/db_utils.spec.ts +0 -1085
- package/src/service/db_utils.ts +0 -1502
- package/src/service/duckdb_instance_isolation.spec.ts +0 -137
- package/src/service/ducklake_lazy_attach.spec.ts +0 -110
- package/src/service/embedding_provider.spec.ts +0 -329
- package/src/service/embedding_provider.ts +0 -236
- package/src/service/environment.ts +0 -2191
- package/src/service/environment_admission.spec.ts +0 -344
- package/src/service/environment_compile.spec.ts +0 -197
- package/src/service/environment_store.spec.ts +0 -2274
- package/src/service/environment_store.ts +0 -2434
- package/src/service/environment_store_anchoring.spec.ts +0 -107
- package/src/service/environment_store_clone.spec.ts +0 -350
- package/src/service/explore_visibility.spec.ts +0 -434
- package/src/service/exports_probe.spec.ts +0 -107
- package/src/service/extension_fetch_policy.spec.ts +0 -256
- package/src/service/filter.spec.ts +0 -458
- package/src/service/filter.ts +0 -341
- package/src/service/filter_bypass.spec.ts +0 -418
- package/src/service/filter_integration.spec.ts +0 -1004
- package/src/service/freshness.spec.ts +0 -183
- package/src/service/freshness.ts +0 -112
- package/src/service/gcs_s3_utils.ts +0 -427
- package/src/service/given.ts +0 -105
- package/src/service/givens_integration.spec.ts +0 -219
- package/src/service/manifest_loader.spec.ts +0 -189
- package/src/service/manifest_loader.ts +0 -156
- package/src/service/materialization_build_session.spec.ts +0 -435
- package/src/service/materialization_build_session.ts +0 -681
- package/src/service/materialization_config_validation.spec.ts +0 -99
- package/src/service/materialization_config_validation.ts +0 -120
- package/src/service/materialization_eligibility.spec.ts +0 -158
- package/src/service/materialization_eligibility.ts +0 -305
- package/src/service/materialization_schedule_surface.spec.ts +0 -256
- package/src/service/materialization_scheduler.spec.ts +0 -289
- package/src/service/materialization_scheduler.ts +0 -304
- package/src/service/materialization_serve_transform.spec.ts +0 -1003
- package/src/service/materialization_serve_transform.ts +0 -779
- package/src/service/materialization_service.spec.ts +0 -2373
- package/src/service/materialization_service.ts +0 -2342
- package/src/service/materialization_test_fixtures.ts +0 -249
- package/src/service/model.spec.ts +0 -1157
- package/src/service/model.ts +0 -3370
- package/src/service/model_limits.spec.ts +0 -209
- package/src/service/model_limits.ts +0 -131
- package/src/service/model_storage_serve.spec.ts +0 -193
- package/src/service/model_storage_serve_joins.spec.ts +0 -193
- package/src/service/package.spec.ts +0 -612
- package/src/service/package.ts +0 -1799
- package/src/service/package_manifest.spec.ts +0 -230
- package/src/service/package_manifest.ts +0 -241
- package/src/service/package_memory_governor.spec.ts +0 -173
- package/src/service/package_memory_governor.ts +0 -233
- package/src/service/package_quote_bound_tables.spec.ts +0 -171
- package/src/service/package_race.spec.ts +0 -208
- package/src/service/package_reload_safety.spec.ts +0 -193
- package/src/service/package_rollback.spec.ts +0 -190
- package/src/service/package_worker_path.spec.ts +0 -645
- package/src/service/path_injection.spec.ts +0 -39
- package/src/service/persist_annotation_validation.spec.ts +0 -89
- package/src/service/persist_annotation_validation.ts +0 -59
- package/src/service/persistence_policy.spec.ts +0 -374
- package/src/service/proxy.spec.ts +0 -414
- package/src/service/proxy.ts +0 -248
- package/src/service/query_boundary.spec.ts +0 -470
- package/src/service/query_metadata.spec.ts +0 -408
- package/src/service/query_metadata.ts +0 -492
- package/src/service/query_metadata_identity.spec.ts +0 -149
- package/src/service/query_text.spec.ts +0 -79
- package/src/service/query_text.ts +0 -43
- package/src/service/quoting.spec.ts +0 -135
- package/src/service/quoting.ts +0 -78
- package/src/service/resolve_environment.ts +0 -15
- package/src/service/restricted_mode.spec.ts +0 -299
- package/src/service/source_extraction.ts +0 -225
- package/src/service/theme_store.ts +0 -199
- package/src/storage/DatabaseInterface.ts +0 -198
- package/src/storage/StorageManager.ts +0 -113
- package/src/storage/duckdb/ConnectionRepository.ts +0 -143
- package/src/storage/duckdb/DuckDBConnection.ts +0 -123
- package/src/storage/duckdb/DuckDBRepository.ts +0 -208
- package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
- package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
- package/src/storage/duckdb/MaterializationRepository.ts +0 -336
- package/src/storage/duckdb/PackageRepository.ts +0 -135
- package/src/storage/duckdb/schema.ts +0 -229
- package/src/stream_helpers.spec.ts +0 -280
- package/src/stream_helpers.ts +0 -162
- package/src/telemetry.ts +0 -20
- package/src/test_helpers/metrics_harness.ts +0 -166
- package/src/theme_key_parity.spec.ts +0 -57
- package/src/utils.ts +0 -29
- package/tests/fixtures/authorize-compile/model.malloy +0 -9
- package/tests/fixtures/authorize-compile/publisher.json +0 -4
- package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
- package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
- package/tests/fixtures/html-pages-test/data.csv +0 -3
- package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
- package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
- package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
- package/tests/fixtures/html-pages-test/public/data.json +0 -1
- package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
- package/tests/fixtures/html-pages-test/public/index.html +0 -9
- package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
- package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
- package/tests/fixtures/html-pages-test/public/slides.html +0 -12
- package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
- package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
- package/tests/fixtures/html-pages-test/publisher.json +0 -5
- package/tests/fixtures/html-pages-test/report.malloy +0 -1
- package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
- package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
- package/tests/fixtures/persist-multi-level/publisher.json +0 -5
- package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
- package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
- package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
- package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
- package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
- package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
- package/tests/fixtures/persist-test/data/orders.csv +0 -5
- package/tests/fixtures/persist-test/persist_test.malloy +0 -11
- package/tests/fixtures/persist-test/publisher.json +0 -5
- package/tests/fixtures/publisher.config.json +0 -15
- package/tests/fixtures/query-givens/data/orders.csv +0 -7
- package/tests/fixtures/query-givens/model.malloy +0 -34
- package/tests/fixtures/query-givens/publisher.json +0 -5
- package/tests/fixtures/xlsx/database.xlsx +0 -0
- package/tests/harness/mcp_test_setup.ts +0 -332
- package/tests/harness/rest_e2e.ts +0 -68
- package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
- package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
- package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
- package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
- package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
- package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
- package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
- package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
- package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
- package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
- package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
- package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
- package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
- package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
- package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
- package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
- package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
- package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
- package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
- package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
- package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
- package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
- package/tests/unit/controller/theme.controller.spec.ts +0 -141
- package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
- package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
- package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
- package/tests/unit/duckdb/repositories.test.ts +0 -208
- package/tests/unit/ducklake/ducklake.test.ts +0 -952
- package/tests/unit/service/theme_store.spec.ts +0 -139
- package/tsconfig.json +0 -32
|
@@ -1,899 +0,0 @@
|
|
|
1
|
-
import { createPrivateKey } from "crypto";
|
|
2
|
-
import { existsSync } from "fs";
|
|
3
|
-
import path from "path";
|
|
4
|
-
import { components } from "../api";
|
|
5
|
-
import { logger } from "../logger";
|
|
6
|
-
import { parseHostKeys } from "./proxy";
|
|
7
|
-
import {
|
|
8
|
-
queryMetadataAdvisoryWarnings,
|
|
9
|
-
queryMetadataBudgetWarning,
|
|
10
|
-
queryMetadataViolations,
|
|
11
|
-
} from "./query_metadata";
|
|
12
|
-
|
|
13
|
-
type ApiConnection = components["schemas"]["Connection"];
|
|
14
|
-
type AttachedDatabase = components["schemas"]["AttachedDatabase"];
|
|
15
|
-
|
|
16
|
-
// TLS modes accepted on a proxied postgres connection. Canonical here (rather
|
|
17
|
-
// than in connection.ts, which imports this module) so both the config-load
|
|
18
|
-
// validator and the connect-time builder derive from one list. Mirrors the
|
|
19
|
-
// `sslmode` enum in api-doc.yaml.
|
|
20
|
-
export const PROXIED_SSLMODES = [
|
|
21
|
-
"disable",
|
|
22
|
-
"no-verify",
|
|
23
|
-
"verify-ca",
|
|
24
|
-
"verify-full",
|
|
25
|
-
] as const;
|
|
26
|
-
|
|
27
|
-
export type CoreConnectionEntry = {
|
|
28
|
-
is: string;
|
|
29
|
-
[key: string]: unknown;
|
|
30
|
-
};
|
|
31
|
-
|
|
32
|
-
export type CoreConnectionsPojo = {
|
|
33
|
-
connections: Record<string, CoreConnectionEntry>;
|
|
34
|
-
};
|
|
35
|
-
|
|
36
|
-
export type EnvironmentConnectionMetadata = {
|
|
37
|
-
apiConnection: ApiConnection;
|
|
38
|
-
attachedDatabases: AttachedDatabase[];
|
|
39
|
-
hasAzureAttachment: boolean;
|
|
40
|
-
hasSnowflakePrivateKey: boolean;
|
|
41
|
-
isDuckLake: boolean;
|
|
42
|
-
databasePath?: string;
|
|
43
|
-
workingDirectory: string;
|
|
44
|
-
proxy?: ApiConnection["proxy"];
|
|
45
|
-
};
|
|
46
|
-
|
|
47
|
-
export type AssembledEnvironmentConnections = {
|
|
48
|
-
pojo: CoreConnectionsPojo;
|
|
49
|
-
metadata: Map<string, EnvironmentConnectionMetadata>;
|
|
50
|
-
apiConnections: ApiConnection[];
|
|
51
|
-
};
|
|
52
|
-
|
|
53
|
-
const PUBLISHER_DUCKDB_API_FIELDS = new Set<string>(["attachedDatabases"]);
|
|
54
|
-
|
|
55
|
-
export function normalizeSnowflakePrivateKey(privateKey: string): string {
|
|
56
|
-
let privateKeyContent = privateKey.trim();
|
|
57
|
-
|
|
58
|
-
if (!privateKeyContent.includes("\n")) {
|
|
59
|
-
const keyPatterns = [
|
|
60
|
-
{
|
|
61
|
-
beginRegex: /-----BEGIN\s+ENCRYPTED\s+PRIVATE\s+KEY-----/i,
|
|
62
|
-
endRegex: /-----END\s+ENCRYPTED\s+PRIVATE\s+KEY-----/i,
|
|
63
|
-
beginMarker: "-----BEGIN ENCRYPTED PRIVATE KEY-----",
|
|
64
|
-
endMarker: "-----END ENCRYPTED PRIVATE KEY-----",
|
|
65
|
-
},
|
|
66
|
-
{
|
|
67
|
-
beginRegex: /-----BEGIN\s+PRIVATE\s+KEY-----/i,
|
|
68
|
-
endRegex: /-----END\s+PRIVATE\s+KEY-----/i,
|
|
69
|
-
beginMarker: "-----BEGIN PRIVATE KEY-----",
|
|
70
|
-
endMarker: "-----END PRIVATE KEY-----",
|
|
71
|
-
},
|
|
72
|
-
{
|
|
73
|
-
beginRegex: /-----BEGIN\s+RSA\s+PRIVATE\s+KEY-----/i,
|
|
74
|
-
endRegex: /-----END\s+RSA\s+PRIVATE\s+KEY-----/i,
|
|
75
|
-
beginMarker: "-----BEGIN RSA PRIVATE KEY-----",
|
|
76
|
-
endMarker: "-----END RSA PRIVATE KEY-----",
|
|
77
|
-
},
|
|
78
|
-
];
|
|
79
|
-
|
|
80
|
-
for (const pattern of keyPatterns) {
|
|
81
|
-
const beginMatch = privateKeyContent.match(pattern.beginRegex);
|
|
82
|
-
const endMatch = privateKeyContent.match(pattern.endRegex);
|
|
83
|
-
|
|
84
|
-
if (beginMatch && endMatch) {
|
|
85
|
-
const beginPos = beginMatch.index! + beginMatch[0].length;
|
|
86
|
-
const endPos = endMatch.index!;
|
|
87
|
-
const keyData = privateKeyContent
|
|
88
|
-
.substring(beginPos, endPos)
|
|
89
|
-
.replace(/\s+/g, "");
|
|
90
|
-
|
|
91
|
-
const lines: string[] = [];
|
|
92
|
-
for (let i = 0; i < keyData.length; i += 64) {
|
|
93
|
-
lines.push(keyData.slice(i, i + 64));
|
|
94
|
-
}
|
|
95
|
-
privateKeyContent = `${pattern.beginMarker}\n${lines.join("\n")}\n${pattern.endMarker}\n`;
|
|
96
|
-
break;
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
} else if (!privateKeyContent.endsWith("\n")) {
|
|
100
|
-
privateKeyContent += "\n";
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
// Snowflake's Node SDK requires PKCS#8 ("BEGIN PRIVATE KEY"). Convert
|
|
104
|
-
// PKCS#1 ("BEGIN RSA PRIVATE KEY") so users can paste either format.
|
|
105
|
-
if (/-----BEGIN\s+RSA\s+PRIVATE\s+KEY-----/i.test(privateKeyContent)) {
|
|
106
|
-
try {
|
|
107
|
-
privateKeyContent = createPrivateKey({
|
|
108
|
-
key: privateKeyContent,
|
|
109
|
-
format: "pem",
|
|
110
|
-
})
|
|
111
|
-
.export({ type: "pkcs8", format: "pem" })
|
|
112
|
-
.toString();
|
|
113
|
-
} catch (err) {
|
|
114
|
-
throw new Error(
|
|
115
|
-
`Failed to convert Snowflake RSA private key (PKCS#1) to PKCS#8: ${
|
|
116
|
-
err instanceof Error ? err.message : String(err)
|
|
117
|
-
}`,
|
|
118
|
-
);
|
|
119
|
-
}
|
|
120
|
-
if (!privateKeyContent.endsWith("\n")) {
|
|
121
|
-
privateKeyContent += "\n";
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
return privateKeyContent;
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
// NOTE: This narrows the environment-author API surface (it rejects securityPolicy,
|
|
129
|
-
// allowedDirectories, setupSQL, etc.). It is NOT a filesystem isolation
|
|
130
|
-
// boundary: attachedDatabases[].path is not normalized or constrained to stay
|
|
131
|
-
// under the environment root, and DuckDB's local-file access is unchanged.
|
|
132
|
-
// Adversarial filesystem isolation is an explicit non-goal here: DuckDB
|
|
133
|
-
// hardening knobs are not exposed and there is no adversarial DuckDB
|
|
134
|
-
// filesystem isolation. Future work owns any path-traversal/allowlist
|
|
135
|
-
// enforcement.
|
|
136
|
-
export function validateDuckdbApiSurface(connection: ApiConnection): void {
|
|
137
|
-
if (connection.type !== "duckdb" || !connection.duckdbConnection) return;
|
|
138
|
-
|
|
139
|
-
const unsupportedFields = Object.keys(connection.duckdbConnection).filter(
|
|
140
|
-
(field) =>
|
|
141
|
-
!PUBLISHER_DUCKDB_API_FIELDS.has(field) &&
|
|
142
|
-
(connection.duckdbConnection as Record<string, unknown>)[field] !==
|
|
143
|
-
undefined,
|
|
144
|
-
);
|
|
145
|
-
|
|
146
|
-
if (unsupportedFields.length > 0) {
|
|
147
|
-
throw new Error(
|
|
148
|
-
`Unsupported DuckDB connection field(s): ${unsupportedFields.join(
|
|
149
|
-
", ",
|
|
150
|
-
)}. Publisher only supports attachedDatabases for environment-authored DuckDB connections.`,
|
|
151
|
-
);
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
function cloneApiConnection(connection: ApiConnection): ApiConnection {
|
|
156
|
-
return { ...connection };
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
function getStaticConnectionAttributes(
|
|
160
|
-
type: ApiConnection["type"],
|
|
161
|
-
): components["schemas"]["ConnectionAttributes"] | undefined {
|
|
162
|
-
switch (type) {
|
|
163
|
-
case "postgres":
|
|
164
|
-
return {
|
|
165
|
-
dialectName: "postgres",
|
|
166
|
-
isPool: false,
|
|
167
|
-
canPersist: true,
|
|
168
|
-
canStream: true,
|
|
169
|
-
};
|
|
170
|
-
case "bigquery":
|
|
171
|
-
return {
|
|
172
|
-
dialectName: "standardsql",
|
|
173
|
-
isPool: false,
|
|
174
|
-
canPersist: true,
|
|
175
|
-
canStream: true,
|
|
176
|
-
};
|
|
177
|
-
case "snowflake":
|
|
178
|
-
return {
|
|
179
|
-
dialectName: "snowflake",
|
|
180
|
-
isPool: true,
|
|
181
|
-
canPersist: true,
|
|
182
|
-
canStream: true,
|
|
183
|
-
};
|
|
184
|
-
case "trino":
|
|
185
|
-
return {
|
|
186
|
-
dialectName: "trino",
|
|
187
|
-
isPool: false,
|
|
188
|
-
canPersist: true,
|
|
189
|
-
canStream: false,
|
|
190
|
-
};
|
|
191
|
-
case "databricks":
|
|
192
|
-
return {
|
|
193
|
-
dialectName: "databricks",
|
|
194
|
-
isPool: false,
|
|
195
|
-
canPersist: true,
|
|
196
|
-
canStream: false,
|
|
197
|
-
};
|
|
198
|
-
case "mysql":
|
|
199
|
-
return {
|
|
200
|
-
dialectName: "mysql",
|
|
201
|
-
isPool: false,
|
|
202
|
-
canPersist: true,
|
|
203
|
-
canStream: false,
|
|
204
|
-
};
|
|
205
|
-
case "duckdb":
|
|
206
|
-
case "motherduck":
|
|
207
|
-
case "ducklake":
|
|
208
|
-
return {
|
|
209
|
-
dialectName: "duckdb",
|
|
210
|
-
isPool: false,
|
|
211
|
-
canPersist: true,
|
|
212
|
-
canStream: true,
|
|
213
|
-
};
|
|
214
|
-
default:
|
|
215
|
-
return undefined;
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
type ServiceAccountKey = {
|
|
220
|
-
type?: string;
|
|
221
|
-
environment_id?: string;
|
|
222
|
-
private_key?: string;
|
|
223
|
-
client_email?: string;
|
|
224
|
-
[key: string]: unknown;
|
|
225
|
-
};
|
|
226
|
-
|
|
227
|
-
function parseServiceAccountKey(json?: string): ServiceAccountKey | undefined {
|
|
228
|
-
if (!json) return undefined;
|
|
229
|
-
const keyData = JSON.parse(json) as ServiceAccountKey;
|
|
230
|
-
const requiredFields = ["type", "project_id", "private_key", "client_email"];
|
|
231
|
-
for (const field of requiredFields) {
|
|
232
|
-
if (!keyData[field]) {
|
|
233
|
-
throw new Error(
|
|
234
|
-
`Invalid service account key: missing "${field}" field`,
|
|
235
|
-
);
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
if (keyData.type !== "service_account") {
|
|
239
|
-
throw new Error('Invalid service account key: incorrect "type" field');
|
|
240
|
-
}
|
|
241
|
-
return keyData;
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
function buildPostgresConnectionString(
|
|
245
|
-
config: components["schemas"]["PostgresConnection"],
|
|
246
|
-
): string | undefined {
|
|
247
|
-
if (config.connectionString || !process.env.PGSSLMODE) {
|
|
248
|
-
return config.connectionString;
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
const params = new URLSearchParams();
|
|
252
|
-
params.set("sslmode", process.env.PGSSLMODE);
|
|
253
|
-
const auth =
|
|
254
|
-
config.userName && config.password
|
|
255
|
-
? `${encodeURIComponent(config.userName)}:${encodeURIComponent(
|
|
256
|
-
config.password,
|
|
257
|
-
)}@`
|
|
258
|
-
: config.userName
|
|
259
|
-
? `${encodeURIComponent(config.userName)}@`
|
|
260
|
-
: "";
|
|
261
|
-
const host = config.host ?? "localhost";
|
|
262
|
-
const port = config.port ? `:${config.port}` : "";
|
|
263
|
-
const database = config.databaseName
|
|
264
|
-
? `/${encodeURIComponent(config.databaseName)}`
|
|
265
|
-
: "";
|
|
266
|
-
return `postgresql://${auth}${host}${port}${database}?${params.toString()}`;
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
function buildDuckdbEntry(
|
|
270
|
-
name: string,
|
|
271
|
-
environmentPath: string,
|
|
272
|
-
databaseFilename = `${name}.duckdb`,
|
|
273
|
-
): CoreConnectionEntry {
|
|
274
|
-
return {
|
|
275
|
-
is: "duckdb",
|
|
276
|
-
databasePath: path.join(environmentPath, databaseFilename),
|
|
277
|
-
};
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
/**
|
|
281
|
-
* Report a connection default that will not do what it says — a property name
|
|
282
|
-
* the contract rejects, one BigQuery would drop, a bag with no room for the
|
|
283
|
-
* server's own context.
|
|
284
|
-
*
|
|
285
|
-
* Warns rather than throws, unlike everything else in this file: query metadata
|
|
286
|
-
* is observability, and an environment that refuses to load because a tag has a
|
|
287
|
-
* hyphen in it would trade a missing label for an outage. The connection update
|
|
288
|
-
* API rejects the same bag outright (see validateAdminAuthoredConnection) —
|
|
289
|
-
* strict where a human is waiting, lenient where a config is being loaded.
|
|
290
|
-
*/
|
|
291
|
-
function warnOnConnectionQueryMetadata(connection: ApiConnection): void {
|
|
292
|
-
let declared = 0;
|
|
293
|
-
for (const field of ["queryMetadata", "queryMetadataEnforced"] as const) {
|
|
294
|
-
const metadata = connection[field];
|
|
295
|
-
if (!metadata) continue;
|
|
296
|
-
declared += Object.keys(metadata).length;
|
|
297
|
-
const problems = [
|
|
298
|
-
...queryMetadataViolations(metadata),
|
|
299
|
-
...queryMetadataAdvisoryWarnings(metadata),
|
|
300
|
-
];
|
|
301
|
-
for (const problem of problems) {
|
|
302
|
-
logger.warn("Connection query metadata will not apply as declared", {
|
|
303
|
-
connectionName: connection.name,
|
|
304
|
-
field,
|
|
305
|
-
problem,
|
|
306
|
-
});
|
|
307
|
-
}
|
|
308
|
-
}
|
|
309
|
-
// The budget is checked over BOTH maps, not each one: they merge into the
|
|
310
|
-
// same bag, so a connection declaring 6 defaults and 6 enforced is over it
|
|
311
|
-
// while neither map is. This is the boundary where the admin who created the
|
|
312
|
-
// squeeze is the one reading the warning.
|
|
313
|
-
const overBudget = queryMetadataBudgetWarning(declared);
|
|
314
|
-
if (overBudget) {
|
|
315
|
-
logger.warn("Connection query metadata will not apply as declared", {
|
|
316
|
-
connectionName: connection.name,
|
|
317
|
-
problem: overBudget,
|
|
318
|
-
});
|
|
319
|
-
}
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
function validateConnectionShape(connection: ApiConnection): void {
|
|
323
|
-
if (connection.proxy) {
|
|
324
|
-
// A connection proxy makes THIS server open an outbound SSH tunnel to a
|
|
325
|
-
// tenant-configured bastion. It's a normal connection capability,
|
|
326
|
-
// authorized by whoever configures the connection — deliberately NOT gated
|
|
327
|
-
// by an env flag, and kept separate from the `publisher` HTTP multi-hop
|
|
328
|
-
// type's PUBLISHER_ALLOW_PROXY_CONNECTIONS gate below (that flag is about
|
|
329
|
-
// publisher-to-publisher proxying, a distinct operator decision). Optional
|
|
330
|
-
// host-key pinning is fail-closed at connect time when configured (see
|
|
331
|
-
// openProxy); the proxy-specific fields are validated up front below so a
|
|
332
|
-
// permanent misconfig fails at config load, not by repeatedly dialing the
|
|
333
|
-
// tenant's bastion at query time.
|
|
334
|
-
if (connection.proxy.type !== "ssh") {
|
|
335
|
-
throw new Error(
|
|
336
|
-
`Connection '${connection.name}' has an unsupported proxy type '${connection.proxy.type}'. Only 'ssh' is supported.`,
|
|
337
|
-
);
|
|
338
|
-
}
|
|
339
|
-
if (connection.type !== "postgres") {
|
|
340
|
-
throw new Error(
|
|
341
|
-
`Connection proxy is not supported for type '${connection.type}' (only 'postgres' today).`,
|
|
342
|
-
);
|
|
343
|
-
}
|
|
344
|
-
if (!connection.proxy.ssh) {
|
|
345
|
-
throw new Error(
|
|
346
|
-
`Connection proxy on '${connection.name}' has type 'ssh' but no 'ssh' config object.`,
|
|
347
|
-
);
|
|
348
|
-
}
|
|
349
|
-
// The tunnel forwards to an explicit host:port; the connectionString form
|
|
350
|
-
// can't be rewritten to the local endpoint. Reject it outright when a
|
|
351
|
-
// proxy is set — normal postgres gives connectionString precedence over
|
|
352
|
-
// host/port, so a config carrying BOTH would silently tunnel to
|
|
353
|
-
// host/port and ignore the connectionString, connecting to a different
|
|
354
|
-
// database than the operator configured. Require discrete host/port.
|
|
355
|
-
if (connection.postgresConnection?.connectionString) {
|
|
356
|
-
throw new Error(
|
|
357
|
-
`Connection proxy on '${connection.name}' does not support the connectionString form; ` +
|
|
358
|
-
`provide discrete host and port instead (the tunnel forwards to an explicit endpoint).`,
|
|
359
|
-
);
|
|
360
|
-
}
|
|
361
|
-
if (
|
|
362
|
-
!connection.postgresConnection?.host ||
|
|
363
|
-
!connection.postgresConnection?.port
|
|
364
|
-
) {
|
|
365
|
-
throw new Error(
|
|
366
|
-
`Connection proxy on '${connection.name}' requires explicit host and port on the ` +
|
|
367
|
-
`postgres connection; the connectionString form is not supported with a proxy.`,
|
|
368
|
-
);
|
|
369
|
-
}
|
|
370
|
-
|
|
371
|
-
// hostKey is optional (omitted or empty string => connect unpinned), but a
|
|
372
|
-
// non-empty hostKey that parses to zero keys — only blank lines, whitespace,
|
|
373
|
-
// or `#` comments, e.g. a paste that grabbed just ssh-keyscan's
|
|
374
|
-
// `# host:port ...` header — is a misconfigured pin, not a licence to
|
|
375
|
-
// connect unverified. Reject it here so the operator gets a config error
|
|
376
|
-
// instead of a silently unpinned tunnel. (Truthiness, not trim(): "" is the
|
|
377
|
-
// unpinned signal; " " is a non-empty value that must yield a key.)
|
|
378
|
-
const hostKey = connection.proxy.ssh?.hostKey;
|
|
379
|
-
if (hostKey && parseHostKeys(hostKey).size === 0) {
|
|
380
|
-
throw new Error(
|
|
381
|
-
`Connection proxy on '${connection.name}' has a hostKey with no usable host-key line ` +
|
|
382
|
-
`(only blanks/comments). Provide an OpenSSH known_hosts line or base64 blob, or omit ` +
|
|
383
|
-
`hostKey to connect unpinned.`,
|
|
384
|
-
);
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
// Validate the proxied TLS mode up front. The tunnel is dialed lazily on
|
|
388
|
-
// first lookup and a failed build is retried on every subsequent query, so
|
|
389
|
-
// a permanent sslmode misconfig left to throw at connect time would re-dial
|
|
390
|
-
// the tenant's bastion indefinitely. Fail at config load instead.
|
|
391
|
-
// `!= null` (not truthiness) so a present-but-empty sslmode ("") is caught
|
|
392
|
-
// here as unsupported rather than slipping through to fail at tunnel-build;
|
|
393
|
-
// null/undefined mean unset (server applies the default).
|
|
394
|
-
const sslmode = connection.postgresConnection?.sslmode;
|
|
395
|
-
if (sslmode != null) {
|
|
396
|
-
if (!(PROXIED_SSLMODES as readonly string[]).includes(sslmode)) {
|
|
397
|
-
throw new Error(
|
|
398
|
-
`Connection proxy on '${connection.name}' has unsupported sslmode '${sslmode}' ` +
|
|
399
|
-
`(expected ${PROXIED_SSLMODES.join(" | ")}).`,
|
|
400
|
-
);
|
|
401
|
-
}
|
|
402
|
-
if (sslmode === "verify-ca") {
|
|
403
|
-
const caBundle = process.env.NODE_EXTRA_CA_CERTS;
|
|
404
|
-
if (!caBundle || !existsSync(caBundle)) {
|
|
405
|
-
throw new Error(
|
|
406
|
-
`Connection proxy on '${connection.name}' uses sslmode 'verify-ca' but no readable ` +
|
|
407
|
-
`CA bundle is available (NODE_EXTRA_CA_CERTS is unset or points at a missing file). ` +
|
|
408
|
-
`Add the CA bundle to the image or use sslmode 'no-verify'.`,
|
|
409
|
-
);
|
|
410
|
-
}
|
|
411
|
-
}
|
|
412
|
-
// No precondition for `verify-full`: unlike `verify-ca` (which passes an
|
|
413
|
-
// explicit `sslrootcert` path), it verifies against Node's ambient trust
|
|
414
|
-
// anchors (its bundled Mozilla CA roots + NODE_EXTRA_CA_CERTS), so
|
|
415
|
-
// requiring a bundle here would wrongly reject targets with a
|
|
416
|
-
// publicly-trusted CA. An untrusted CA surfaces as a clear pg error at
|
|
417
|
-
// connect time instead. This means verify-full's CA trust is broader
|
|
418
|
-
// than verify-ca's pinned bundle — intentional (it adds the hostname
|
|
419
|
-
// check without narrowing trust), so unlike libpq, verify-full here is
|
|
420
|
-
// not a strict superset of verify-ca.
|
|
421
|
-
}
|
|
422
|
-
|
|
423
|
-
// The proxied path builds a connectionString to the local tunnel endpoint;
|
|
424
|
-
// pg decodes the database path with decodeURI, which leaves URI-reserved
|
|
425
|
-
// characters percent-encoded — so a db name containing them would resolve
|
|
426
|
-
// to the wrong database. Reject it clearly rather than failing later with a
|
|
427
|
-
// confusing "database does not exist". (user/password use decodeURIComponent
|
|
428
|
-
// on parse and round-trip fine.)
|
|
429
|
-
const dbName = connection.postgresConnection?.databaseName;
|
|
430
|
-
if (dbName && decodeURI(encodeURIComponent(dbName)) !== dbName) {
|
|
431
|
-
throw new Error(
|
|
432
|
-
`Connection proxy on '${connection.name}' has a database name with characters that can't ` +
|
|
433
|
-
`be carried over a proxied connection (${JSON.stringify(dbName)}). Use a database name ` +
|
|
434
|
-
`without URI-reserved characters (; , / ? : @ & = + $ #).`,
|
|
435
|
-
);
|
|
436
|
-
}
|
|
437
|
-
}
|
|
438
|
-
|
|
439
|
-
// sslmode is only honored on the proxied path (the direct path builds TLS from
|
|
440
|
-
// the deployment PGSSLMODE). Reject it on a non-proxied connection so a tenant
|
|
441
|
-
// who sets it doesn't silently get a different TLS posture than they asked for.
|
|
442
|
-
if (!connection.proxy && connection.postgresConnection?.sslmode) {
|
|
443
|
-
throw new Error(
|
|
444
|
-
`Connection '${connection.name}' sets postgresConnection.sslmode but has no proxy; sslmode is ` +
|
|
445
|
-
`only supported for proxied connections (direct connections use the deployment PGSSLMODE).`,
|
|
446
|
-
);
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
switch (connection.type) {
|
|
450
|
-
case "postgres":
|
|
451
|
-
case "mysql":
|
|
452
|
-
case "bigquery":
|
|
453
|
-
break;
|
|
454
|
-
case "duckdb":
|
|
455
|
-
if (!connection.duckdbConnection) {
|
|
456
|
-
throw new Error("DuckDB connection configuration is missing.");
|
|
457
|
-
}
|
|
458
|
-
{
|
|
459
|
-
const attached =
|
|
460
|
-
connection.duckdbConnection.attachedDatabases ?? [];
|
|
461
|
-
if (attached.length === 0) {
|
|
462
|
-
throw new Error(
|
|
463
|
-
`DuckDB connection "${connection.name}" has no attached databases. Add at least one foreign database (BigQuery, Snowflake, Postgres, GCS, S3, Azure) to attachedDatabases, or remove this connection entirely — each package already gets a per-package DuckDB sandbox named "duckdb" automatically.`,
|
|
464
|
-
);
|
|
465
|
-
}
|
|
466
|
-
}
|
|
467
|
-
break;
|
|
468
|
-
case "motherduck":
|
|
469
|
-
if (!connection.motherduckConnection) {
|
|
470
|
-
throw new Error("MotherDuck connection configuration is missing.");
|
|
471
|
-
}
|
|
472
|
-
if (!connection.motherduckConnection.accessToken) {
|
|
473
|
-
throw new Error("MotherDuck access token is required.");
|
|
474
|
-
}
|
|
475
|
-
break;
|
|
476
|
-
case "ducklake":
|
|
477
|
-
// Every field the ATTACH demands (see attachDuckLakeWithMode), checked
|
|
478
|
-
// together here rather than split between this validator and the pojo
|
|
479
|
-
// assembler below. `bucketUrl` was the one nobody checked: a destination
|
|
480
|
-
// is only attached at its first BUILD, so a missing bucket surfaced
|
|
481
|
-
// hours after the config change that caused it.
|
|
482
|
-
if (!connection.ducklakeConnection) {
|
|
483
|
-
throw new Error("DuckLake connection configuration is missing.");
|
|
484
|
-
}
|
|
485
|
-
if (!connection.ducklakeConnection.catalog?.postgresConnection) {
|
|
486
|
-
throw new Error(
|
|
487
|
-
`PostgreSQL connection configuration is required for DuckLake catalog: ${connection.name}`,
|
|
488
|
-
);
|
|
489
|
-
}
|
|
490
|
-
if (!connection.ducklakeConnection.storage?.bucketUrl) {
|
|
491
|
-
throw new Error(
|
|
492
|
-
`Storage bucketUrl is required for DuckLake: ${connection.name}`,
|
|
493
|
-
);
|
|
494
|
-
}
|
|
495
|
-
// metadataSchema is optional, but when present it reaches the ATTACH as a
|
|
496
|
-
// quoted string literal AND the catalog-format preflight as a quoted
|
|
497
|
-
// identifier. Rather than escape one value for two grammars, restrict it
|
|
498
|
-
// to a plain identifier here — a deterministic config error, caught at
|
|
499
|
-
// load instead of at the connection's first attach.
|
|
500
|
-
//
|
|
501
|
-
// The typeof check is load-bearing, not defensive: the value arrives from
|
|
502
|
-
// untyped JSON, and RegExp.test() coerces its argument, so `true` and `null`
|
|
503
|
-
// both satisfy the pattern as "true"/"null" and would reach escapeSQL's
|
|
504
|
-
// String.replace as a non-string — a TypeError at the first attach, which is
|
|
505
|
-
// exactly the failure this check exists to turn into a config error.
|
|
506
|
-
if (
|
|
507
|
-
connection.ducklakeConnection.catalog.metadataSchema !== undefined
|
|
508
|
-
) {
|
|
509
|
-
const schema = connection.ducklakeConnection.catalog.metadataSchema;
|
|
510
|
-
if (
|
|
511
|
-
typeof schema !== "string" ||
|
|
512
|
-
!/^[A-Za-z_][A-Za-z0-9_]*$/.test(schema)
|
|
513
|
-
) {
|
|
514
|
-
throw new Error(
|
|
515
|
-
`DuckLake catalog metadataSchema must be a plain identifier ` +
|
|
516
|
-
`([A-Za-z_][A-Za-z0-9_]*), got '${schema}' for connection: ` +
|
|
517
|
-
`${connection.name}`,
|
|
518
|
-
);
|
|
519
|
-
}
|
|
520
|
-
}
|
|
521
|
-
break;
|
|
522
|
-
case "trino":
|
|
523
|
-
if (!connection.trinoConnection) {
|
|
524
|
-
throw new Error("Trino connection configuration is missing.");
|
|
525
|
-
}
|
|
526
|
-
break;
|
|
527
|
-
case "databricks": {
|
|
528
|
-
const databricks = connection.databricksConnection;
|
|
529
|
-
if (!databricks) {
|
|
530
|
-
throw new Error("Databricks connection configuration is missing.");
|
|
531
|
-
}
|
|
532
|
-
if (!databricks.host) {
|
|
533
|
-
throw new Error("Databricks host is required.");
|
|
534
|
-
}
|
|
535
|
-
if (!databricks.path) {
|
|
536
|
-
throw new Error("Databricks SQL warehouse HTTP path is required.");
|
|
537
|
-
}
|
|
538
|
-
const hasToken = !!databricks.token;
|
|
539
|
-
const hasOAuth =
|
|
540
|
-
!!databricks.oauthClientId && !!databricks.oauthClientSecret;
|
|
541
|
-
if (!hasToken && !hasOAuth) {
|
|
542
|
-
throw new Error(
|
|
543
|
-
"Databricks requires either a personal access token or OAuth M2M client ID and secret.",
|
|
544
|
-
);
|
|
545
|
-
}
|
|
546
|
-
const hasDefaultCatalog = !!databricks.defaultCatalog;
|
|
547
|
-
if (!hasDefaultCatalog) {
|
|
548
|
-
throw new Error("Databricks default catalog is required.");
|
|
549
|
-
}
|
|
550
|
-
break;
|
|
551
|
-
}
|
|
552
|
-
case "snowflake": {
|
|
553
|
-
const snowflakeConnection = connection.snowflakeConnection;
|
|
554
|
-
if (!snowflakeConnection) {
|
|
555
|
-
throw new Error("Snowflake connection configuration is missing.");
|
|
556
|
-
}
|
|
557
|
-
if (!snowflakeConnection.account) {
|
|
558
|
-
throw new Error("Snowflake account is required.");
|
|
559
|
-
}
|
|
560
|
-
if (!snowflakeConnection.username) {
|
|
561
|
-
throw new Error("Snowflake username is required.");
|
|
562
|
-
}
|
|
563
|
-
if (!snowflakeConnection.password && !snowflakeConnection.privateKey) {
|
|
564
|
-
throw new Error(
|
|
565
|
-
"Snowflake password or private key or private key path is required.",
|
|
566
|
-
);
|
|
567
|
-
}
|
|
568
|
-
if (!snowflakeConnection.warehouse) {
|
|
569
|
-
throw new Error("Snowflake warehouse is required.");
|
|
570
|
-
}
|
|
571
|
-
break;
|
|
572
|
-
}
|
|
573
|
-
case "publisher": {
|
|
574
|
-
// SSRF gate (default-deny / fail-closed). A `publisher` connection
|
|
575
|
-
// makes THIS server issue outbound HTTP to a tenant-controlled
|
|
576
|
-
// `connectionUri` (both the query path in db-publisher's
|
|
577
|
-
// PublisherConnection and the introspection path in db_utils). That is
|
|
578
|
-
// the intended behavior for local `--watch-env` authoring, but in a
|
|
579
|
-
// hosted multi-tenant deployment (e.g. Credible running this server) it
|
|
580
|
-
// is an SSRF surface. Require an explicit opt-in so the type is refused
|
|
581
|
-
// unless the operator deliberately enabled it. This is the single
|
|
582
|
-
// choke point — every connection passes validateConnectionShape before
|
|
583
|
-
// it can be assembled, queried, or introspected — so denying here shuts
|
|
584
|
-
// off all three at once.
|
|
585
|
-
if (process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS !== "true") {
|
|
586
|
-
throw new Error(
|
|
587
|
-
`Publisher proxy connection '${connection.name}' is disabled in this deployment. ` +
|
|
588
|
-
`'publisher' connections make the server issue outbound requests to a configured connectionUri, ` +
|
|
589
|
-
`which is only appropriate for local --watch-env authoring. ` +
|
|
590
|
-
`Fix: set the environment variable PUBLISHER_ALLOW_PROXY_CONNECTIONS=true to enable them.`,
|
|
591
|
-
);
|
|
592
|
-
}
|
|
593
|
-
const publisher = connection.publisherConnection;
|
|
594
|
-
if (!publisher?.connectionUri) {
|
|
595
|
-
throw new Error(
|
|
596
|
-
`Invalid publisher connection '${connection.name}': missing connectionUri. ` +
|
|
597
|
-
`Fix: { "name": "${connection.name}", "type": "publisher", ` +
|
|
598
|
-
`"publisherConnection": { "connectionUri": "https://…/connections/${connection.name}", "accessToken": "<jwt>" } }`,
|
|
599
|
-
);
|
|
600
|
-
}
|
|
601
|
-
// Reject a malformed connectionUri here, at config-load, rather than
|
|
602
|
-
// letting it fail deep in the request path — where the thrown error can
|
|
603
|
-
// echo the raw value back, leaking any credentials embedded in it
|
|
604
|
-
// (`redactUrlCredentials` returns the URI unchanged when it can't parse
|
|
605
|
-
// it). Never include the raw connectionUri in these messages; the
|
|
606
|
-
// scheme is safe to name.
|
|
607
|
-
let parsedUri: URL;
|
|
608
|
-
try {
|
|
609
|
-
parsedUri = new URL(publisher.connectionUri);
|
|
610
|
-
} catch {
|
|
611
|
-
throw new Error(
|
|
612
|
-
`Invalid publisher connection '${connection.name}': connectionUri is not a valid URL. ` +
|
|
613
|
-
`Fix: set connectionUri to an absolute https URL like "https://…/connections/${connection.name}".`,
|
|
614
|
-
);
|
|
615
|
-
}
|
|
616
|
-
if (
|
|
617
|
-
parsedUri.protocol !== "http:" &&
|
|
618
|
-
parsedUri.protocol !== "https:"
|
|
619
|
-
) {
|
|
620
|
-
throw new Error(
|
|
621
|
-
`Invalid publisher connection '${connection.name}': connectionUri must use http or https (got '${parsedUri.protocol}'). ` +
|
|
622
|
-
`Fix: set connectionUri to an absolute https URL like "https://…/connections/${connection.name}".`,
|
|
623
|
-
);
|
|
624
|
-
}
|
|
625
|
-
break;
|
|
626
|
-
}
|
|
627
|
-
}
|
|
628
|
-
}
|
|
629
|
-
|
|
630
|
-
export function assembleEnvironmentConnections(
|
|
631
|
-
connections: ApiConnection[] = [],
|
|
632
|
-
environmentPath = "",
|
|
633
|
-
): AssembledEnvironmentConnections {
|
|
634
|
-
const pojo: CoreConnectionsPojo = { connections: {} };
|
|
635
|
-
const metadata = new Map<string, EnvironmentConnectionMetadata>();
|
|
636
|
-
const apiConnections: ApiConnection[] = [];
|
|
637
|
-
const processedConnections = new Set<string>();
|
|
638
|
-
|
|
639
|
-
for (const connection of connections) {
|
|
640
|
-
if (!connection.name) {
|
|
641
|
-
throw new Error("Invalid connection configuration. No name.");
|
|
642
|
-
}
|
|
643
|
-
|
|
644
|
-
if (processedConnections.has(connection.name)) {
|
|
645
|
-
continue;
|
|
646
|
-
}
|
|
647
|
-
|
|
648
|
-
if (connection.name === "duckdb") {
|
|
649
|
-
throw new Error(
|
|
650
|
-
"Connection name 'duckdb' is reserved for per-package sandboxes. Choose a different name for environment-level DuckDB connections (e.g. 'shared_duckdb').",
|
|
651
|
-
);
|
|
652
|
-
}
|
|
653
|
-
|
|
654
|
-
processedConnections.add(connection.name);
|
|
655
|
-
validateDuckdbApiSurface(connection);
|
|
656
|
-
validateConnectionShape(connection);
|
|
657
|
-
warnOnConnectionQueryMetadata(connection);
|
|
658
|
-
|
|
659
|
-
const apiConnection = cloneApiConnection(connection);
|
|
660
|
-
apiConnection.attributes = getStaticConnectionAttributes(connection.type);
|
|
661
|
-
const attachedDatabases =
|
|
662
|
-
connection.duckdbConnection?.attachedDatabases ?? [];
|
|
663
|
-
const isDuckLake = connection.type === "ducklake";
|
|
664
|
-
const isDuckdb = connection.type === "duckdb";
|
|
665
|
-
const databasePath = isDuckLake
|
|
666
|
-
? path.join(environmentPath, `${connection.name}_ducklake.duckdb`)
|
|
667
|
-
: isDuckdb
|
|
668
|
-
? path.join(environmentPath, `${connection.name}.duckdb`)
|
|
669
|
-
: undefined;
|
|
670
|
-
|
|
671
|
-
metadata.set(connection.name, {
|
|
672
|
-
apiConnection,
|
|
673
|
-
attachedDatabases,
|
|
674
|
-
hasAzureAttachment: attachedDatabases.some(
|
|
675
|
-
(database) => database.type === "azure",
|
|
676
|
-
),
|
|
677
|
-
hasSnowflakePrivateKey:
|
|
678
|
-
connection.type === "snowflake" &&
|
|
679
|
-
!!connection.snowflakeConnection?.privateKey,
|
|
680
|
-
isDuckLake,
|
|
681
|
-
databasePath,
|
|
682
|
-
workingDirectory: environmentPath,
|
|
683
|
-
proxy: connection.proxy,
|
|
684
|
-
});
|
|
685
|
-
|
|
686
|
-
switch (connection.type) {
|
|
687
|
-
case "postgres": {
|
|
688
|
-
const postgresConnection = connection.postgresConnection;
|
|
689
|
-
pojo.connections[connection.name] = {
|
|
690
|
-
is: "postgres",
|
|
691
|
-
host: postgresConnection?.host,
|
|
692
|
-
port: postgresConnection?.port,
|
|
693
|
-
username: postgresConnection?.userName,
|
|
694
|
-
password: postgresConnection?.password,
|
|
695
|
-
databaseName: postgresConnection?.databaseName,
|
|
696
|
-
connectionString: postgresConnection
|
|
697
|
-
? buildPostgresConnectionString(postgresConnection)
|
|
698
|
-
: undefined,
|
|
699
|
-
};
|
|
700
|
-
break;
|
|
701
|
-
}
|
|
702
|
-
|
|
703
|
-
case "mysql": {
|
|
704
|
-
pojo.connections[connection.name] = {
|
|
705
|
-
is: "mysql",
|
|
706
|
-
host: connection.mysqlConnection?.host,
|
|
707
|
-
port: connection.mysqlConnection?.port,
|
|
708
|
-
user: connection.mysqlConnection?.user,
|
|
709
|
-
password: connection.mysqlConnection?.password,
|
|
710
|
-
database: connection.mysqlConnection?.database,
|
|
711
|
-
};
|
|
712
|
-
break;
|
|
713
|
-
}
|
|
714
|
-
|
|
715
|
-
case "bigquery": {
|
|
716
|
-
const serviceAccountKey = parseServiceAccountKey(
|
|
717
|
-
connection.bigqueryConnection?.serviceAccountKeyJson as
|
|
718
|
-
| string
|
|
719
|
-
| undefined,
|
|
720
|
-
);
|
|
721
|
-
pojo.connections[connection.name] = {
|
|
722
|
-
is: "bigquery",
|
|
723
|
-
projectId:
|
|
724
|
-
connection.bigqueryConnection?.defaultProjectId ??
|
|
725
|
-
serviceAccountKey?.environment_id,
|
|
726
|
-
serviceAccountKey,
|
|
727
|
-
location: connection.bigqueryConnection?.location,
|
|
728
|
-
maximumBytesBilled:
|
|
729
|
-
connection.bigqueryConnection?.maximumBytesBilled,
|
|
730
|
-
timeoutMs:
|
|
731
|
-
connection.bigqueryConnection?.queryTimeoutMilliseconds,
|
|
732
|
-
billingProjectId:
|
|
733
|
-
connection.bigqueryConnection?.billingProjectId,
|
|
734
|
-
};
|
|
735
|
-
break;
|
|
736
|
-
}
|
|
737
|
-
|
|
738
|
-
case "snowflake": {
|
|
739
|
-
pojo.connections[connection.name] = {
|
|
740
|
-
is: "snowflake",
|
|
741
|
-
account: connection.snowflakeConnection?.account,
|
|
742
|
-
username: connection.snowflakeConnection?.username,
|
|
743
|
-
password: connection.snowflakeConnection?.password,
|
|
744
|
-
privateKey: connection.snowflakeConnection?.privateKey
|
|
745
|
-
? normalizeSnowflakePrivateKey(
|
|
746
|
-
connection.snowflakeConnection.privateKey,
|
|
747
|
-
)
|
|
748
|
-
: undefined,
|
|
749
|
-
privateKeyPass: connection.snowflakeConnection?.privateKeyPass,
|
|
750
|
-
warehouse: connection.snowflakeConnection?.warehouse,
|
|
751
|
-
database: connection.snowflakeConnection?.database,
|
|
752
|
-
schema: connection.snowflakeConnection?.schema,
|
|
753
|
-
role: connection.snowflakeConnection?.role,
|
|
754
|
-
timeoutMs:
|
|
755
|
-
connection.snowflakeConnection?.responseTimeoutMilliseconds,
|
|
756
|
-
// Pool sizing is server-owned policy (matches the values
|
|
757
|
-
// main's deleted switch passed pre-MalloyConfig adoption).
|
|
758
|
-
// Not exposed through the public API.
|
|
759
|
-
poolMin: 1,
|
|
760
|
-
poolMax: 20,
|
|
761
|
-
};
|
|
762
|
-
break;
|
|
763
|
-
}
|
|
764
|
-
|
|
765
|
-
case "trino": {
|
|
766
|
-
pojo.connections[connection.name] = {
|
|
767
|
-
is: "trino",
|
|
768
|
-
...validateAndBuildTrinoCoreConfig(connection.trinoConnection),
|
|
769
|
-
};
|
|
770
|
-
break;
|
|
771
|
-
}
|
|
772
|
-
|
|
773
|
-
case "databricks": {
|
|
774
|
-
const databricks = connection.databricksConnection;
|
|
775
|
-
pojo.connections[connection.name] = {
|
|
776
|
-
is: "databricks",
|
|
777
|
-
host: databricks?.host,
|
|
778
|
-
path: databricks?.path,
|
|
779
|
-
token: databricks?.token,
|
|
780
|
-
oauthClientId: databricks?.oauthClientId,
|
|
781
|
-
oauthClientSecret: databricks?.oauthClientSecret,
|
|
782
|
-
defaultCatalog: databricks?.defaultCatalog,
|
|
783
|
-
defaultSchema: databricks?.defaultSchema,
|
|
784
|
-
setupSQL: databricks?.setupSQL,
|
|
785
|
-
};
|
|
786
|
-
break;
|
|
787
|
-
}
|
|
788
|
-
|
|
789
|
-
case "duckdb": {
|
|
790
|
-
if (
|
|
791
|
-
attachedDatabases.some(
|
|
792
|
-
(database) => database.name === connection.name,
|
|
793
|
-
)
|
|
794
|
-
) {
|
|
795
|
-
throw new Error(
|
|
796
|
-
`DuckDB attached database names cannot conflict with connection name ${connection.name}`,
|
|
797
|
-
);
|
|
798
|
-
}
|
|
799
|
-
pojo.connections[connection.name] = buildDuckdbEntry(
|
|
800
|
-
connection.name,
|
|
801
|
-
environmentPath,
|
|
802
|
-
`${connection.name}.duckdb`,
|
|
803
|
-
);
|
|
804
|
-
break;
|
|
805
|
-
}
|
|
806
|
-
|
|
807
|
-
case "motherduck": {
|
|
808
|
-
if (!connection.motherduckConnection?.accessToken) {
|
|
809
|
-
throw new Error("MotherDuck access token is required.");
|
|
810
|
-
}
|
|
811
|
-
|
|
812
|
-
pojo.connections[connection.name] = {
|
|
813
|
-
is: "duckdb",
|
|
814
|
-
databasePath: connection.motherduckConnection.database
|
|
815
|
-
? `md:${connection.motherduckConnection.database}?attach_mode=single`
|
|
816
|
-
: "md:",
|
|
817
|
-
motherDuckToken: connection.motherduckConnection.accessToken,
|
|
818
|
-
};
|
|
819
|
-
break;
|
|
820
|
-
}
|
|
821
|
-
|
|
822
|
-
case "ducklake": {
|
|
823
|
-
// Shape is validated up front by validateConnectionShape.
|
|
824
|
-
pojo.connections[connection.name] = buildDuckdbEntry(
|
|
825
|
-
connection.name,
|
|
826
|
-
environmentPath,
|
|
827
|
-
`${connection.name}_ducklake.duckdb`,
|
|
828
|
-
);
|
|
829
|
-
break;
|
|
830
|
-
}
|
|
831
|
-
|
|
832
|
-
case "publisher": {
|
|
833
|
-
// connectionUri presence is validated by validateConnectionShape
|
|
834
|
-
// above. The proxied dataplane owns auth and read-only enforcement;
|
|
835
|
-
// PublisherConnection itself does not reject writes. The real
|
|
836
|
-
// dialect is the remote connection's and is resolved at runtime by
|
|
837
|
-
// the live connection, so getStaticConnectionAttributes returns
|
|
838
|
-
// undefined for publisher (falls through to its default).
|
|
839
|
-
const publisher = connection.publisherConnection!;
|
|
840
|
-
pojo.connections[connection.name] = {
|
|
841
|
-
is: "publisher",
|
|
842
|
-
connectionUri: publisher.connectionUri,
|
|
843
|
-
accessToken: publisher.accessToken,
|
|
844
|
-
};
|
|
845
|
-
break;
|
|
846
|
-
}
|
|
847
|
-
|
|
848
|
-
default: {
|
|
849
|
-
throw new Error(`Unsupported connection type: ${connection.type}`);
|
|
850
|
-
}
|
|
851
|
-
}
|
|
852
|
-
|
|
853
|
-
apiConnections.push(apiConnection);
|
|
854
|
-
}
|
|
855
|
-
|
|
856
|
-
return { pojo, metadata, apiConnections };
|
|
857
|
-
}
|
|
858
|
-
|
|
859
|
-
function validateAndBuildTrinoCoreConfig(
|
|
860
|
-
trinoConfig: components["schemas"]["TrinoConnection"] | undefined,
|
|
861
|
-
): Record<string, unknown> {
|
|
862
|
-
if (!trinoConfig) {
|
|
863
|
-
return {};
|
|
864
|
-
}
|
|
865
|
-
|
|
866
|
-
const server =
|
|
867
|
-
trinoConfig.server && trinoConfig.port
|
|
868
|
-
? trinoConfig.server.includes(trinoConfig.port.toString())
|
|
869
|
-
? trinoConfig.server
|
|
870
|
-
: `${trinoConfig.server}:${trinoConfig.port}`
|
|
871
|
-
: trinoConfig.server;
|
|
872
|
-
|
|
873
|
-
const baseConfig: Record<string, unknown> = {
|
|
874
|
-
server,
|
|
875
|
-
port: trinoConfig.port,
|
|
876
|
-
catalog: trinoConfig.catalog,
|
|
877
|
-
schema: trinoConfig.schema,
|
|
878
|
-
user: trinoConfig.user,
|
|
879
|
-
};
|
|
880
|
-
|
|
881
|
-
if (trinoConfig.peakaKey) {
|
|
882
|
-
baseConfig.extraCredential = {
|
|
883
|
-
peakaKey: trinoConfig.peakaKey,
|
|
884
|
-
};
|
|
885
|
-
return baseConfig;
|
|
886
|
-
}
|
|
887
|
-
|
|
888
|
-
if (server?.startsWith("https://") && trinoConfig.password) {
|
|
889
|
-
baseConfig.password = trinoConfig.password;
|
|
890
|
-
}
|
|
891
|
-
|
|
892
|
-
if (server?.startsWith("http://") || server?.startsWith("https://")) {
|
|
893
|
-
return baseConfig;
|
|
894
|
-
}
|
|
895
|
-
|
|
896
|
-
throw new Error(
|
|
897
|
-
`Invalid Trino connection: expected "http://server:port" or "https://server:port".`,
|
|
898
|
-
);
|
|
899
|
-
}
|