@malloy-publisher/server 0.0.235 → 0.0.237
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +83 -27
- package/dist/app/assets/{EnvironmentPage-BsAnavYN.js → EnvironmentPage-DaP6CpgK.js} +1 -1
- package/dist/app/assets/{HomePage-CADE138j.js → HomePage-C9yNw3iu.js} +1 -1
- package/dist/app/assets/{LightMode-Cfh7KzN8.js → LightMode-Be4-G-dX.js} +1 -1
- package/dist/app/assets/{MainPage-CO3pRlnV.js → MainPage-J_rU2SgI.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-p9YjkRXZ.js → MaterializationsPage-DHgPI4MO.js} +1 -1
- package/dist/app/assets/ModelPage-BzXfjTu5.js +1 -0
- package/dist/app/assets/{PackagePage-e4kN75YR.js → PackagePage-ByoSTLJ4.js} +1 -1
- package/dist/app/assets/RouteError-CrSeKN2r.js +1 -0
- package/dist/app/assets/{ThemeEditorPage-CciagFTq.js → ThemeEditorPage-Btxnmhtk.js} +1 -1
- package/dist/app/assets/{WorkbookPage-DNWmkCXa.js → WorkbookPage-D6JLPnWO.js} +1 -1
- package/dist/app/assets/{core-Rj_4rRnA.es-BZyvITuO.js → core-BSRgt1mG.es-Bc_D2Lq4.js} +1 -1
- package/dist/app/assets/{index-CH2AcDzc.js → index-BRijWj6K.js} +15 -15
- package/dist/app/assets/{index-DQa463gC.js → index-BX4D94xw.js} +1 -1
- package/dist/app/assets/{index-VBbcc8s6.js → index-Cvy0xtb7.js} +1 -1
- package/dist/app/assets/{index-DQpV7MyA.js → index-DMAhIJNC.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/package_load_worker.mjs +59 -20
- package/dist/server.mjs +1342 -616
- package/package.json +8 -2
- package/.eslintrc.json +0 -44
- package/.prettierignore +0 -1
- package/.prettierrc +0 -4
- package/build.ts +0 -97
- package/dist/app/assets/ModelPage-C1OSTv-x.js +0 -1
- package/dist/app/assets/RouteError-CzbfOkng.js +0 -1
- package/eslint.config.mjs +0 -76
- package/k6-tests/README.md +0 -144
- package/k6-tests/bun.lock +0 -797
- package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
- package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
- package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
- package/k6-tests/load-test/load-test-crud.ts +0 -142
- package/k6-tests/load-test/load-test.ts +0 -784
- package/k6-tests/package.json +0 -27
- package/k6-tests/scripts/run-with-otel.sh +0 -50
- package/k6-tests/smoke-test/smoke-test.ts +0 -158
- package/k6-tests/tsconfig.json +0 -29
- package/k6-tests/utils/client_factory.ts +0 -128
- package/k6-tests/utils/common.ts +0 -489
- package/k6-tests/utils/logger.ts +0 -62
- package/k6-tests/utils/types.d.ts +0 -35
- package/malloy_mcp.dxt +0 -0
- package/publisher.config.example.bigquery.json +0 -33
- package/publisher.config.example.duckdb.json +0 -23
- package/publisher.config.json +0 -23
- package/scripts/bake-duckdb-extensions.js +0 -110
- package/src/config.spec.ts +0 -1535
- package/src/config.theme.spec.ts +0 -178
- package/src/config.ts +0 -1142
- package/src/constants.ts +0 -105
- package/src/controller/compile.controller.ts +0 -40
- package/src/controller/connection.controller.spec.ts +0 -849
- package/src/controller/connection.controller.ts +0 -901
- package/src/controller/database.controller.ts +0 -24
- package/src/controller/materialization.controller.spec.ts +0 -293
- package/src/controller/materialization.controller.ts +0 -319
- package/src/controller/model.controller.ts +0 -164
- package/src/controller/package.controller.spec.ts +0 -273
- package/src/controller/package.controller.ts +0 -306
- package/src/controller/query.controller.ts +0 -152
- package/src/controller/theme.controller.ts +0 -83
- package/src/controller/watch-mode.controller.ts +0 -243
- package/src/data_styles.ts +0 -71
- package/src/default-publisher.config.json +0 -23
- package/src/ducklake_version.spec.ts +0 -163
- package/src/ducklake_version.ts +0 -153
- package/src/errors.spec.ts +0 -119
- package/src/errors.ts +0 -237
- package/src/filter_deprecation.spec.ts +0 -64
- package/src/filter_deprecation.ts +0 -42
- package/src/health.spec.ts +0 -123
- package/src/health.ts +0 -233
- package/src/heap_check.spec.ts +0 -144
- package/src/heap_check.ts +0 -144
- package/src/instrumentation.ts +0 -214
- package/src/json_utils.spec.ts +0 -51
- package/src/json_utils.ts +0 -33
- package/src/logger.spec.ts +0 -210
- package/src/logger.ts +0 -251
- package/src/malloy_pin_prereqs.spec.ts +0 -25
- package/src/materialization_metrics.spec.ts +0 -169
- package/src/materialization_metrics.ts +0 -346
- package/src/mcp/error_messages.spec.ts +0 -35
- package/src/mcp/error_messages.ts +0 -183
- package/src/mcp/handler_utils.spec.ts +0 -108
- package/src/mcp/handler_utils.ts +0 -251
- package/src/mcp/mcp_constants.ts +0 -6
- package/src/mcp/query_envelope.spec.ts +0 -229
- package/src/mcp/query_envelope.ts +0 -240
- package/src/mcp/server.protocol.spec.ts +0 -250
- package/src/mcp/server.ts +0 -87
- package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
- package/src/mcp/skills/build_skills_bundle.ts +0 -176
- package/src/mcp/skills/skills_bundle.json +0 -1
- package/src/mcp/skills/skills_bundle.spec.ts +0 -171
- package/src/mcp/tool_response.spec.ts +0 -108
- package/src/mcp/tool_response.ts +0 -138
- package/src/mcp/tools/compile_tool.spec.ts +0 -315
- package/src/mcp/tools/compile_tool.ts +0 -208
- package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
- package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
- package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
- package/src/mcp/tools/docs_search_tool.ts +0 -128
- package/src/mcp/tools/embedding_index.spec.ts +0 -1236
- package/src/mcp/tools/embedding_index.ts +0 -808
- package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
- package/src/mcp/tools/execute_query_tool.ts +0 -316
- package/src/mcp/tools/get_context_eval.ts +0 -275
- package/src/mcp/tools/get_context_tool.spec.ts +0 -592
- package/src/mcp/tools/get_context_tool.ts +0 -585
- package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
- package/src/mcp/tools/reload_package_tool.ts +0 -132
- package/src/mcp_config.spec.ts +0 -919
- package/src/mcp_config.ts +0 -425
- package/src/oom_guards.integration.spec.ts +0 -269
- package/src/package_load/package_load_pool.spec.ts +0 -252
- package/src/package_load/package_load_pool.ts +0 -931
- package/src/package_load/package_load_worker.ts +0 -1034
- package/src/package_load/protocol.ts +0 -364
- package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
- package/src/package_load/rpc_wait_accountant.ts +0 -76
- package/src/package_load_metrics.spec.ts +0 -114
- package/src/package_load_metrics.ts +0 -127
- package/src/path_safety.spec.ts +0 -158
- package/src/path_safety.ts +0 -146
- package/src/pg_helpers.spec.ts +0 -223
- package/src/pg_helpers.ts +0 -52
- package/src/query_cap_metrics.spec.ts +0 -89
- package/src/query_cap_metrics.ts +0 -116
- package/src/query_concurrency.spec.ts +0 -247
- package/src/query_concurrency.ts +0 -237
- package/src/query_metadata_metrics.ts +0 -49
- package/src/query_param_utils.ts +0 -29
- package/src/query_timeout.spec.ts +0 -224
- package/src/query_timeout.ts +0 -179
- package/src/runtime/publisher.js +0 -328
- package/src/server-old.ts +0 -988
- package/src/server.ts +0 -2059
- package/src/service/annotations.spec.ts +0 -118
- package/src/service/annotations.ts +0 -91
- package/src/service/authorize.spec.ts +0 -154
- package/src/service/authorize.ts +0 -498
- package/src/service/authorize_integration.spec.ts +0 -2041
- package/src/service/build_plan.spec.ts +0 -685
- package/src/service/build_plan.ts +0 -797
- package/src/service/compile_authorize.spec.ts +0 -85
- package/src/service/compile_fragment_techniques.spec.ts +0 -156
- package/src/service/connection.spec.ts +0 -2360
- package/src/service/connection.ts +0 -2187
- package/src/service/connection_config.spec.ts +0 -734
- package/src/service/connection_config.ts +0 -899
- package/src/service/connection_federation.spec.ts +0 -184
- package/src/service/connection_fingerprint.spec.ts +0 -102
- package/src/service/connection_service.spec.ts +0 -711
- package/src/service/connection_service.ts +0 -291
- package/src/service/cron_evaluator.spec.ts +0 -79
- package/src/service/cron_evaluator.ts +0 -105
- package/src/service/db_utils.spec.ts +0 -1085
- package/src/service/db_utils.ts +0 -1502
- package/src/service/duckdb_instance_isolation.spec.ts +0 -137
- package/src/service/ducklake_lazy_attach.spec.ts +0 -110
- package/src/service/embedding_provider.spec.ts +0 -329
- package/src/service/embedding_provider.ts +0 -236
- package/src/service/environment.ts +0 -2191
- package/src/service/environment_admission.spec.ts +0 -344
- package/src/service/environment_compile.spec.ts +0 -197
- package/src/service/environment_store.spec.ts +0 -2274
- package/src/service/environment_store.ts +0 -2434
- package/src/service/environment_store_anchoring.spec.ts +0 -107
- package/src/service/environment_store_clone.spec.ts +0 -350
- package/src/service/explore_visibility.spec.ts +0 -434
- package/src/service/exports_probe.spec.ts +0 -107
- package/src/service/extension_fetch_policy.spec.ts +0 -256
- package/src/service/filter.spec.ts +0 -458
- package/src/service/filter.ts +0 -341
- package/src/service/filter_bypass.spec.ts +0 -418
- package/src/service/filter_integration.spec.ts +0 -1004
- package/src/service/freshness.spec.ts +0 -183
- package/src/service/freshness.ts +0 -112
- package/src/service/gcs_s3_utils.ts +0 -427
- package/src/service/given.ts +0 -105
- package/src/service/givens_integration.spec.ts +0 -219
- package/src/service/manifest_loader.spec.ts +0 -189
- package/src/service/manifest_loader.ts +0 -156
- package/src/service/materialization_build_session.spec.ts +0 -435
- package/src/service/materialization_build_session.ts +0 -681
- package/src/service/materialization_config_validation.spec.ts +0 -99
- package/src/service/materialization_config_validation.ts +0 -120
- package/src/service/materialization_eligibility.spec.ts +0 -158
- package/src/service/materialization_eligibility.ts +0 -305
- package/src/service/materialization_schedule_surface.spec.ts +0 -256
- package/src/service/materialization_scheduler.spec.ts +0 -289
- package/src/service/materialization_scheduler.ts +0 -304
- package/src/service/materialization_serve_transform.spec.ts +0 -1003
- package/src/service/materialization_serve_transform.ts +0 -779
- package/src/service/materialization_service.spec.ts +0 -2373
- package/src/service/materialization_service.ts +0 -2342
- package/src/service/materialization_test_fixtures.ts +0 -249
- package/src/service/model.spec.ts +0 -1157
- package/src/service/model.ts +0 -3370
- package/src/service/model_limits.spec.ts +0 -209
- package/src/service/model_limits.ts +0 -131
- package/src/service/model_storage_serve.spec.ts +0 -193
- package/src/service/model_storage_serve_joins.spec.ts +0 -193
- package/src/service/package.spec.ts +0 -612
- package/src/service/package.ts +0 -1799
- package/src/service/package_manifest.spec.ts +0 -230
- package/src/service/package_manifest.ts +0 -241
- package/src/service/package_memory_governor.spec.ts +0 -173
- package/src/service/package_memory_governor.ts +0 -233
- package/src/service/package_quote_bound_tables.spec.ts +0 -171
- package/src/service/package_race.spec.ts +0 -208
- package/src/service/package_reload_safety.spec.ts +0 -193
- package/src/service/package_rollback.spec.ts +0 -190
- package/src/service/package_worker_path.spec.ts +0 -645
- package/src/service/path_injection.spec.ts +0 -39
- package/src/service/persist_annotation_validation.spec.ts +0 -89
- package/src/service/persist_annotation_validation.ts +0 -59
- package/src/service/persistence_policy.spec.ts +0 -374
- package/src/service/proxy.spec.ts +0 -414
- package/src/service/proxy.ts +0 -248
- package/src/service/query_boundary.spec.ts +0 -470
- package/src/service/query_metadata.spec.ts +0 -408
- package/src/service/query_metadata.ts +0 -492
- package/src/service/query_metadata_identity.spec.ts +0 -149
- package/src/service/query_text.spec.ts +0 -79
- package/src/service/query_text.ts +0 -43
- package/src/service/quoting.spec.ts +0 -135
- package/src/service/quoting.ts +0 -78
- package/src/service/resolve_environment.ts +0 -15
- package/src/service/restricted_mode.spec.ts +0 -299
- package/src/service/source_extraction.ts +0 -225
- package/src/service/theme_store.ts +0 -199
- package/src/storage/DatabaseInterface.ts +0 -198
- package/src/storage/StorageManager.ts +0 -113
- package/src/storage/duckdb/ConnectionRepository.ts +0 -143
- package/src/storage/duckdb/DuckDBConnection.ts +0 -123
- package/src/storage/duckdb/DuckDBRepository.ts +0 -208
- package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
- package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
- package/src/storage/duckdb/MaterializationRepository.ts +0 -336
- package/src/storage/duckdb/PackageRepository.ts +0 -135
- package/src/storage/duckdb/schema.ts +0 -229
- package/src/stream_helpers.spec.ts +0 -280
- package/src/stream_helpers.ts +0 -162
- package/src/telemetry.ts +0 -20
- package/src/test_helpers/metrics_harness.ts +0 -166
- package/src/theme_key_parity.spec.ts +0 -57
- package/src/utils.ts +0 -29
- package/tests/fixtures/authorize-compile/model.malloy +0 -9
- package/tests/fixtures/authorize-compile/publisher.json +0 -4
- package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
- package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
- package/tests/fixtures/html-pages-test/data.csv +0 -3
- package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
- package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
- package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
- package/tests/fixtures/html-pages-test/public/data.json +0 -1
- package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
- package/tests/fixtures/html-pages-test/public/index.html +0 -9
- package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
- package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
- package/tests/fixtures/html-pages-test/public/slides.html +0 -12
- package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
- package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
- package/tests/fixtures/html-pages-test/publisher.json +0 -5
- package/tests/fixtures/html-pages-test/report.malloy +0 -1
- package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
- package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
- package/tests/fixtures/persist-multi-level/publisher.json +0 -5
- package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
- package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
- package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
- package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
- package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
- package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
- package/tests/fixtures/persist-test/data/orders.csv +0 -5
- package/tests/fixtures/persist-test/persist_test.malloy +0 -11
- package/tests/fixtures/persist-test/publisher.json +0 -5
- package/tests/fixtures/publisher.config.json +0 -15
- package/tests/fixtures/query-givens/data/orders.csv +0 -7
- package/tests/fixtures/query-givens/model.malloy +0 -34
- package/tests/fixtures/query-givens/publisher.json +0 -5
- package/tests/fixtures/xlsx/database.xlsx +0 -0
- package/tests/harness/mcp_test_setup.ts +0 -332
- package/tests/harness/rest_e2e.ts +0 -68
- package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
- package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
- package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
- package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
- package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
- package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
- package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
- package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
- package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
- package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
- package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
- package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
- package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
- package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
- package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
- package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
- package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
- package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
- package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
- package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
- package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
- package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
- package/tests/unit/controller/theme.controller.spec.ts +0 -141
- package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
- package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
- package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
- package/tests/unit/duckdb/repositories.test.ts +0 -208
- package/tests/unit/ducklake/ducklake.test.ts +0 -952
- package/tests/unit/service/theme_store.spec.ts +0 -139
- package/tsconfig.json +0 -32
package/dist/server.mjs
CHANGED
|
@@ -120280,8 +120280,8 @@ var require_node42 = __commonJS((exports, module) => {
|
|
|
120280
120280
|
}
|
|
120281
120281
|
break;
|
|
120282
120282
|
case "FILE":
|
|
120283
|
-
var
|
|
120284
|
-
stream2 = new
|
|
120283
|
+
var fs2 = __require("fs");
|
|
120284
|
+
stream2 = new fs2.SyncWriteStream(fd2, { autoClose: false });
|
|
120285
120285
|
stream2._type = "fs";
|
|
120286
120286
|
break;
|
|
120287
120287
|
case "PIPE":
|
|
@@ -136347,8 +136347,8 @@ var require_node43 = __commonJS((exports, module) => {
|
|
|
136347
136347
|
}
|
|
136348
136348
|
break;
|
|
136349
136349
|
case "FILE":
|
|
136350
|
-
var
|
|
136351
|
-
stream2 = new
|
|
136350
|
+
var fs2 = __require("fs");
|
|
136351
|
+
stream2 = new fs2.SyncWriteStream(fd2, { autoClose: false });
|
|
136352
136352
|
stream2._type = "fs";
|
|
136353
136353
|
break;
|
|
136354
136354
|
case "PIPE":
|
|
@@ -137073,8 +137073,8 @@ var require_node44 = __commonJS((exports, module) => {
|
|
|
137073
137073
|
}
|
|
137074
137074
|
break;
|
|
137075
137075
|
case "FILE":
|
|
137076
|
-
var
|
|
137077
|
-
stream2 = new
|
|
137076
|
+
var fs2 = __require("fs");
|
|
137077
|
+
stream2 = new fs2.SyncWriteStream(fd2, { autoClose: false });
|
|
137078
137078
|
stream2._type = "fs";
|
|
137079
137079
|
break;
|
|
137080
137080
|
case "PIPE":
|
|
@@ -137986,7 +137986,7 @@ var require_view = __commonJS((exports, module) => {
|
|
|
137986
137986
|
*/
|
|
137987
137987
|
var debug = require_src109()("express:view");
|
|
137988
137988
|
var path = __require("path");
|
|
137989
|
-
var
|
|
137989
|
+
var fs2 = __require("fs");
|
|
137990
137990
|
var dirname = path.dirname;
|
|
137991
137991
|
var basename = path.basename;
|
|
137992
137992
|
var extname = path.extname;
|
|
@@ -138052,7 +138052,7 @@ var require_view = __commonJS((exports, module) => {
|
|
|
138052
138052
|
function tryStat(path2) {
|
|
138053
138053
|
debug('stat "%s"', path2);
|
|
138054
138054
|
try {
|
|
138055
|
-
return
|
|
138055
|
+
return fs2.statSync(path2);
|
|
138056
138056
|
} catch (e) {
|
|
138057
138057
|
return;
|
|
138058
138058
|
}
|
|
@@ -138595,8 +138595,8 @@ var require_node45 = __commonJS((exports, module) => {
|
|
|
138595
138595
|
}
|
|
138596
138596
|
break;
|
|
138597
138597
|
case "FILE":
|
|
138598
|
-
var
|
|
138599
|
-
stream2 = new
|
|
138598
|
+
var fs2 = __require("fs");
|
|
138599
|
+
stream2 = new fs2.SyncWriteStream(fd2, { autoClose: false });
|
|
138600
138600
|
stream2._type = "fs";
|
|
138601
138601
|
break;
|
|
138602
138602
|
case "PIPE":
|
|
@@ -138784,7 +138784,7 @@ var require_types45 = __commonJS((exports, module) => {
|
|
|
138784
138784
|
// ../../node_modules/send/node_modules/mime/mime.js
|
|
138785
138785
|
var require_mime = __commonJS((exports, module) => {
|
|
138786
138786
|
var path = __require("path");
|
|
138787
|
-
var
|
|
138787
|
+
var fs2 = __require("fs");
|
|
138788
138788
|
function Mime() {
|
|
138789
138789
|
this.types = Object.create(null);
|
|
138790
138790
|
this.extensions = Object.create(null);
|
|
@@ -138805,7 +138805,7 @@ var require_mime = __commonJS((exports, module) => {
|
|
|
138805
138805
|
};
|
|
138806
138806
|
Mime.prototype.load = function(file) {
|
|
138807
138807
|
this._loading = file;
|
|
138808
|
-
var map = {}, content =
|
|
138808
|
+
var map = {}, content = fs2.readFileSync(file, "ascii"), lines = content.split(/[\r\n]+/);
|
|
138809
138809
|
lines.forEach(function(line) {
|
|
138810
138810
|
var fields = line.replace(/\s*#.*|^\s*|\s*$/g, "").split(/\s+/);
|
|
138811
138811
|
map[fields.shift()] = fields;
|
|
@@ -138933,7 +138933,7 @@ var require_send = __commonJS((exports, module) => {
|
|
|
138933
138933
|
var escapeHtml = require_escape_html();
|
|
138934
138934
|
var etag = require_etag();
|
|
138935
138935
|
var fresh = require_fresh();
|
|
138936
|
-
var
|
|
138936
|
+
var fs2 = __require("fs");
|
|
138937
138937
|
var mime = require_mime();
|
|
138938
138938
|
var ms = require_ms();
|
|
138939
138939
|
var onFinished = require_on_finished();
|
|
@@ -139261,7 +139261,7 @@ var require_send = __commonJS((exports, module) => {
|
|
|
139261
139261
|
var i = 0;
|
|
139262
139262
|
var self2 = this;
|
|
139263
139263
|
debug('stat "%s"', path2);
|
|
139264
|
-
|
|
139264
|
+
fs2.stat(path2, function onstat(err, stat) {
|
|
139265
139265
|
if (err && err.code === "ENOENT" && !extname(path2) && path2[path2.length - 1] !== sep) {
|
|
139266
139266
|
return next(err);
|
|
139267
139267
|
}
|
|
@@ -139278,7 +139278,7 @@ var require_send = __commonJS((exports, module) => {
|
|
|
139278
139278
|
}
|
|
139279
139279
|
var p = path2 + "." + self2._extensions[i++];
|
|
139280
139280
|
debug('stat "%s"', p);
|
|
139281
|
-
|
|
139281
|
+
fs2.stat(p, function(err2, stat) {
|
|
139282
139282
|
if (err2)
|
|
139283
139283
|
return next(err2);
|
|
139284
139284
|
if (stat.isDirectory())
|
|
@@ -139299,7 +139299,7 @@ var require_send = __commonJS((exports, module) => {
|
|
|
139299
139299
|
}
|
|
139300
139300
|
var p = join(path2, self2._index[i]);
|
|
139301
139301
|
debug('stat "%s"', p);
|
|
139302
|
-
|
|
139302
|
+
fs2.stat(p, function(err2, stat) {
|
|
139303
139303
|
if (err2)
|
|
139304
139304
|
return next(err2);
|
|
139305
139305
|
if (stat.isDirectory())
|
|
@@ -139313,7 +139313,7 @@ var require_send = __commonJS((exports, module) => {
|
|
|
139313
139313
|
SendStream.prototype.stream = function stream(path2, options) {
|
|
139314
139314
|
var self2 = this;
|
|
139315
139315
|
var res = this.res;
|
|
139316
|
-
var stream2 =
|
|
139316
|
+
var stream2 = fs2.createReadStream(path2, options);
|
|
139317
139317
|
this.emit("stream", stream2);
|
|
139318
139318
|
stream2.pipe(res);
|
|
139319
139319
|
function cleanup() {
|
|
@@ -156020,6 +156020,8 @@ function internalErrorToHttpError(error) {
|
|
|
156020
156020
|
return httpError(400, error.message);
|
|
156021
156021
|
} else if (error instanceof ConnectionNotFoundError) {
|
|
156022
156022
|
return httpError(404, error.message);
|
|
156023
|
+
} else if (error instanceof DestinationNotFoundError) {
|
|
156024
|
+
return httpError(422, error.message);
|
|
156023
156025
|
} else if (error instanceof ConnectionAuthError) {
|
|
156024
156026
|
return httpError(422, error.message);
|
|
156025
156027
|
} else if (error instanceof UnsupportedCatalogFormatError) {
|
|
@@ -156055,7 +156057,7 @@ function httpError(code, message) {
|
|
|
156055
156057
|
}
|
|
156056
156058
|
};
|
|
156057
156059
|
}
|
|
156058
|
-
var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, ConnectionAuthError, UnsupportedCatalogFormatError, ModelCompilationError, MaterializationEligibilityError, FrozenConfigError, AccessDeniedError, NotQueryableError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, QueryTimeoutError;
|
|
156060
|
+
var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, DestinationNotFoundError, ConnectionAuthError, UnsupportedCatalogFormatError, ModelCompilationError, MaterializationEligibilityError, FrozenConfigError, AccessDeniedError, NotQueryableError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, ResponseUnserializableError, QueryTimeoutError;
|
|
156059
156061
|
var init_errors = __esm(() => {
|
|
156060
156062
|
init_constants();
|
|
156061
156063
|
NotImplementedError = class NotImplementedError extends Error {
|
|
@@ -156093,6 +156095,11 @@ var init_errors = __esm(() => {
|
|
|
156093
156095
|
super(message);
|
|
156094
156096
|
}
|
|
156095
156097
|
};
|
|
156098
|
+
DestinationNotFoundError = class DestinationNotFoundError extends Error {
|
|
156099
|
+
constructor(message) {
|
|
156100
|
+
super(message);
|
|
156101
|
+
}
|
|
156102
|
+
};
|
|
156096
156103
|
ConnectionAuthError = class ConnectionAuthError extends Error {
|
|
156097
156104
|
constructor(message) {
|
|
156098
156105
|
super(message);
|
|
@@ -156154,6 +156161,13 @@ var init_errors = __esm(() => {
|
|
|
156154
156161
|
PayloadTooLargeError = class PayloadTooLargeError extends Error {
|
|
156155
156162
|
constructor(message) {
|
|
156156
156163
|
super(message);
|
|
156164
|
+
this.name = "PayloadTooLargeError";
|
|
156165
|
+
}
|
|
156166
|
+
};
|
|
156167
|
+
ResponseUnserializableError = class ResponseUnserializableError extends PayloadTooLargeError {
|
|
156168
|
+
constructor(message) {
|
|
156169
|
+
super(message);
|
|
156170
|
+
this.name = "ResponseUnserializableError";
|
|
156157
156171
|
}
|
|
156158
156172
|
};
|
|
156159
156173
|
QueryTimeoutError = class QueryTimeoutError extends Error {
|
|
@@ -165281,7 +165295,7 @@ var require_form_data = __commonJS((exports, module) => {
|
|
|
165281
165295
|
var http = __require("http");
|
|
165282
165296
|
var https = __require("https");
|
|
165283
165297
|
var parseUrl = __require("url").parse;
|
|
165284
|
-
var
|
|
165298
|
+
var fs3 = __require("fs");
|
|
165285
165299
|
var Stream = __require("stream").Stream;
|
|
165286
165300
|
var crypto2 = __require("crypto");
|
|
165287
165301
|
var mime = require_mime_types3();
|
|
@@ -165349,7 +165363,7 @@ var require_form_data = __commonJS((exports, module) => {
|
|
|
165349
165363
|
if (value.end != null && value.end != Infinity && value.start != null) {
|
|
165350
165364
|
callback(null, value.end + 1 - (value.start ? value.start : 0));
|
|
165351
165365
|
} else {
|
|
165352
|
-
|
|
165366
|
+
fs3.stat(value.path, function(err, stat) {
|
|
165353
165367
|
if (err) {
|
|
165354
165368
|
callback(err);
|
|
165355
165369
|
return;
|
|
@@ -183150,8 +183164,8 @@ ${suffix}`;
|
|
|
183150
183164
|
// ../../node_modules/ssh2/lib/protocol/SFTP.js
|
|
183151
183165
|
var require_SFTP = __commonJS((exports, module) => {
|
|
183152
183166
|
var EventEmitter2 = __require("events");
|
|
183153
|
-
var
|
|
183154
|
-
var { constants } =
|
|
183167
|
+
var fs3 = __require("fs");
|
|
183168
|
+
var { constants } = fs3;
|
|
183155
183169
|
var {
|
|
183156
183170
|
Readable: ReadableStream3,
|
|
183157
183171
|
Writable: WritableStream
|
|
@@ -183550,12 +183564,12 @@ var require_SFTP = __commonJS((exports, module) => {
|
|
|
183550
183564
|
fastGet(remotePath, localPath, opts, cb) {
|
|
183551
183565
|
if (this.server)
|
|
183552
183566
|
throw new Error("Client-only method called in server mode");
|
|
183553
|
-
fastXfer(this,
|
|
183567
|
+
fastXfer(this, fs3, remotePath, localPath, opts, cb);
|
|
183554
183568
|
}
|
|
183555
183569
|
fastPut(localPath, remotePath, opts, cb) {
|
|
183556
183570
|
if (this.server)
|
|
183557
183571
|
throw new Error("Client-only method called in server mode");
|
|
183558
|
-
fastXfer(
|
|
183572
|
+
fastXfer(fs3, this, localPath, remotePath, opts, cb);
|
|
183559
183573
|
}
|
|
183560
183574
|
readFile(path3, options, callback_) {
|
|
183561
183575
|
if (this.server)
|
|
@@ -184764,13 +184778,13 @@ var require_SFTP = __commonJS((exports, module) => {
|
|
|
184764
184778
|
if (--left === 0)
|
|
184765
184779
|
cb(err);
|
|
184766
184780
|
};
|
|
184767
|
-
if (srcHandle && (src ===
|
|
184781
|
+
if (srcHandle && (src === fs3 || src.outgoing.state === "open"))
|
|
184768
184782
|
++left;
|
|
184769
|
-
if (dstHandle && (dst ===
|
|
184783
|
+
if (dstHandle && (dst === fs3 || dst.outgoing.state === "open"))
|
|
184770
184784
|
++left;
|
|
184771
|
-
if (srcHandle && (src ===
|
|
184785
|
+
if (srcHandle && (src === fs3 || src.outgoing.state === "open"))
|
|
184772
184786
|
src.close(srcHandle, cbfinal);
|
|
184773
|
-
if (dstHandle && (dst ===
|
|
184787
|
+
if (dstHandle && (dst === fs3 || dst.outgoing.state === "open"))
|
|
184774
184788
|
dst.close(dstHandle, cbfinal);
|
|
184775
184789
|
} else {
|
|
184776
184790
|
cb(err);
|
|
@@ -184786,7 +184800,7 @@ var require_SFTP = __commonJS((exports, module) => {
|
|
|
184786
184800
|
tryStat(null, { size: fileSize });
|
|
184787
184801
|
function tryStat(err2, attrs) {
|
|
184788
184802
|
if (err2) {
|
|
184789
|
-
if (src !==
|
|
184803
|
+
if (src !== fs3) {
|
|
184790
184804
|
src.stat(srcPath, (err_, attrs_) => {
|
|
184791
184805
|
if (err_)
|
|
184792
184806
|
return onerror(err2);
|
|
@@ -190358,23 +190372,23 @@ var require_crypto2 = __commonJS((exports) => {
|
|
|
190358
190372
|
var require_crypto3 = __commonJS((exports) => {
|
|
190359
190373
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
190360
190374
|
exports.NodeCrypto = undefined;
|
|
190361
|
-
var
|
|
190375
|
+
var crypto4 = __require("crypto");
|
|
190362
190376
|
|
|
190363
190377
|
class NodeCrypto {
|
|
190364
190378
|
async sha256DigestBase64(str) {
|
|
190365
|
-
return
|
|
190379
|
+
return crypto4.createHash("sha256").update(str).digest("base64");
|
|
190366
190380
|
}
|
|
190367
190381
|
randomBytesBase64(count) {
|
|
190368
|
-
return
|
|
190382
|
+
return crypto4.randomBytes(count).toString("base64");
|
|
190369
190383
|
}
|
|
190370
190384
|
async verify(pubkey, data, signature) {
|
|
190371
|
-
const verifier =
|
|
190385
|
+
const verifier = crypto4.createVerify("RSA-SHA256");
|
|
190372
190386
|
verifier.update(data);
|
|
190373
190387
|
verifier.end();
|
|
190374
190388
|
return verifier.verify(pubkey, signature, "base64");
|
|
190375
190389
|
}
|
|
190376
190390
|
async sign(privateKey, data) {
|
|
190377
|
-
const signer =
|
|
190391
|
+
const signer = crypto4.createSign("RSA-SHA256");
|
|
190378
190392
|
signer.update(data);
|
|
190379
190393
|
signer.end();
|
|
190380
190394
|
return signer.sign(privateKey, "base64");
|
|
@@ -190386,11 +190400,11 @@ var require_crypto3 = __commonJS((exports) => {
|
|
|
190386
190400
|
return Buffer.from(text, "utf-8").toString("base64");
|
|
190387
190401
|
}
|
|
190388
190402
|
async sha256DigestHex(str) {
|
|
190389
|
-
return
|
|
190403
|
+
return crypto4.createHash("sha256").update(str).digest("hex");
|
|
190390
190404
|
}
|
|
190391
190405
|
async signWithHmacSha256(key, msg) {
|
|
190392
190406
|
const cryptoKey = typeof key === "string" ? key : toBuffer(key);
|
|
190393
|
-
return toArrayBuffer(
|
|
190407
|
+
return toArrayBuffer(crypto4.createHmac("sha256", cryptoKey).update(msg).digest());
|
|
190394
190408
|
}
|
|
190395
190409
|
}
|
|
190396
190410
|
exports.NodeCrypto = NodeCrypto;
|
|
@@ -191006,10 +191020,10 @@ var require_oauth2client = __commonJS((exports) => {
|
|
|
191006
191020
|
throw new Error("generateCodeVerifier is removed, please use generateCodeVerifierAsync instead.");
|
|
191007
191021
|
}
|
|
191008
191022
|
async generateCodeVerifierAsync() {
|
|
191009
|
-
const
|
|
191010
|
-
const randomString =
|
|
191023
|
+
const crypto4 = (0, crypto_1.createCrypto)();
|
|
191024
|
+
const randomString = crypto4.randomBytesBase64(96);
|
|
191011
191025
|
const codeVerifier = randomString.replace(/\+/g, "~").replace(/=/g, "_").replace(/\//g, "-");
|
|
191012
|
-
const unencodedCodeChallenge = await
|
|
191026
|
+
const unencodedCodeChallenge = await crypto4.sha256DigestBase64(codeVerifier);
|
|
191013
191027
|
const codeChallenge = unencodedCodeChallenge.split("=")[0].replace(/\+/g, "-").replace(/\//g, "_");
|
|
191014
191028
|
return { codeVerifier, codeChallenge };
|
|
191015
191029
|
}
|
|
@@ -191411,7 +191425,7 @@ var require_oauth2client = __commonJS((exports) => {
|
|
|
191411
191425
|
throw new Error("verifySignedJwtWithCerts is removed, please use verifySignedJwtWithCertsAsync instead.");
|
|
191412
191426
|
}
|
|
191413
191427
|
async verifySignedJwtWithCertsAsync(jwt, certs, requiredAudience, issuers, maxExpiry) {
|
|
191414
|
-
const
|
|
191428
|
+
const crypto4 = (0, crypto_1.createCrypto)();
|
|
191415
191429
|
if (!maxExpiry) {
|
|
191416
191430
|
maxExpiry = OAuth2Client.DEFAULT_MAX_TOKEN_LIFETIME_SECS_;
|
|
191417
191431
|
}
|
|
@@ -191424,7 +191438,7 @@ var require_oauth2client = __commonJS((exports) => {
|
|
|
191424
191438
|
let envelope;
|
|
191425
191439
|
let payload;
|
|
191426
191440
|
try {
|
|
191427
|
-
envelope = JSON.parse(
|
|
191441
|
+
envelope = JSON.parse(crypto4.decodeBase64StringUtf8(segments[0]));
|
|
191428
191442
|
} catch (err) {
|
|
191429
191443
|
if (err instanceof Error) {
|
|
191430
191444
|
err.message = `Can't parse token envelope: ${segments[0]}': ${err.message}`;
|
|
@@ -191435,7 +191449,7 @@ var require_oauth2client = __commonJS((exports) => {
|
|
|
191435
191449
|
throw new Error("Can't parse token envelope: " + segments[0]);
|
|
191436
191450
|
}
|
|
191437
191451
|
try {
|
|
191438
|
-
payload = JSON.parse(
|
|
191452
|
+
payload = JSON.parse(crypto4.decodeBase64StringUtf8(segments[1]));
|
|
191439
191453
|
} catch (err) {
|
|
191440
191454
|
if (err instanceof Error) {
|
|
191441
191455
|
err.message = `Can't parse token payload '${segments[0]}`;
|
|
@@ -191452,7 +191466,7 @@ var require_oauth2client = __commonJS((exports) => {
|
|
|
191452
191466
|
if (envelope.alg === "ES256") {
|
|
191453
191467
|
signature = formatEcdsa.joseToDer(signature, "ES256").toString("base64");
|
|
191454
191468
|
}
|
|
191455
|
-
const verified = await
|
|
191469
|
+
const verified = await crypto4.verify(cert, signed, signature);
|
|
191456
191470
|
if (!verified) {
|
|
191457
191471
|
throw new Error("Invalid token signature: " + jwt);
|
|
191458
191472
|
}
|
|
@@ -191776,7 +191790,7 @@ var require_buffer_equal_constant_time = __commonJS((exports, module) => {
|
|
|
191776
191790
|
// ../../node_modules/jwa/index.js
|
|
191777
191791
|
var require_jwa = __commonJS((exports, module) => {
|
|
191778
191792
|
var Buffer2 = require_safe_buffer().Buffer;
|
|
191779
|
-
var
|
|
191793
|
+
var crypto4 = __require("crypto");
|
|
191780
191794
|
var formatEcdsa = require_ecdsa_sig_formatter();
|
|
191781
191795
|
var util5 = __require("util");
|
|
191782
191796
|
var MSG_INVALID_ALGORITHM = `"%s" is not a valid algorithm.
|
|
@@ -191785,7 +191799,7 @@ var require_jwa = __commonJS((exports, module) => {
|
|
|
191785
191799
|
var MSG_INVALID_SECRET = "secret must be a string or buffer";
|
|
191786
191800
|
var MSG_INVALID_VERIFIER_KEY = "key must be a string or a buffer";
|
|
191787
191801
|
var MSG_INVALID_SIGNER_KEY = "key must be a string, a buffer or an object";
|
|
191788
|
-
var supportsKeyObjects = typeof
|
|
191802
|
+
var supportsKeyObjects = typeof crypto4.createPublicKey === "function";
|
|
191789
191803
|
if (supportsKeyObjects) {
|
|
191790
191804
|
MSG_INVALID_VERIFIER_KEY += " or a KeyObject";
|
|
191791
191805
|
MSG_INVALID_SECRET += "or a KeyObject";
|
|
@@ -191875,17 +191889,17 @@ var require_jwa = __commonJS((exports, module) => {
|
|
|
191875
191889
|
return function sign(thing, secret) {
|
|
191876
191890
|
checkIsSecretKey(secret);
|
|
191877
191891
|
thing = normalizeInput(thing);
|
|
191878
|
-
var hmac =
|
|
191892
|
+
var hmac = crypto4.createHmac("sha" + bits, secret);
|
|
191879
191893
|
var sig = (hmac.update(thing), hmac.digest("base64"));
|
|
191880
191894
|
return fromBase64(sig);
|
|
191881
191895
|
};
|
|
191882
191896
|
}
|
|
191883
191897
|
var bufferEqual;
|
|
191884
|
-
var timingSafeEqual = "timingSafeEqual" in
|
|
191898
|
+
var timingSafeEqual = "timingSafeEqual" in crypto4 ? function timingSafeEqual2(a, b) {
|
|
191885
191899
|
if (a.byteLength !== b.byteLength) {
|
|
191886
191900
|
return false;
|
|
191887
191901
|
}
|
|
191888
|
-
return
|
|
191902
|
+
return crypto4.timingSafeEqual(a, b);
|
|
191889
191903
|
} : function timingSafeEqual2(a, b) {
|
|
191890
191904
|
if (!bufferEqual) {
|
|
191891
191905
|
bufferEqual = require_buffer_equal_constant_time();
|
|
@@ -191902,7 +191916,7 @@ var require_jwa = __commonJS((exports, module) => {
|
|
|
191902
191916
|
return function sign(thing, privateKey) {
|
|
191903
191917
|
checkIsPrivateKey(privateKey);
|
|
191904
191918
|
thing = normalizeInput(thing);
|
|
191905
|
-
var signer =
|
|
191919
|
+
var signer = crypto4.createSign("RSA-SHA" + bits);
|
|
191906
191920
|
var sig = (signer.update(thing), signer.sign(privateKey, "base64"));
|
|
191907
191921
|
return fromBase64(sig);
|
|
191908
191922
|
};
|
|
@@ -191912,7 +191926,7 @@ var require_jwa = __commonJS((exports, module) => {
|
|
|
191912
191926
|
checkIsPublicKey(publicKey);
|
|
191913
191927
|
thing = normalizeInput(thing);
|
|
191914
191928
|
signature = toBase64(signature);
|
|
191915
|
-
var verifier =
|
|
191929
|
+
var verifier = crypto4.createVerify("RSA-SHA" + bits);
|
|
191916
191930
|
verifier.update(thing);
|
|
191917
191931
|
return verifier.verify(publicKey, signature, "base64");
|
|
191918
191932
|
};
|
|
@@ -191921,11 +191935,11 @@ var require_jwa = __commonJS((exports, module) => {
|
|
|
191921
191935
|
return function sign(thing, privateKey) {
|
|
191922
191936
|
checkIsPrivateKey(privateKey);
|
|
191923
191937
|
thing = normalizeInput(thing);
|
|
191924
|
-
var signer =
|
|
191938
|
+
var signer = crypto4.createSign("RSA-SHA" + bits);
|
|
191925
191939
|
var sig = (signer.update(thing), signer.sign({
|
|
191926
191940
|
key: privateKey,
|
|
191927
|
-
padding:
|
|
191928
|
-
saltLength:
|
|
191941
|
+
padding: crypto4.constants.RSA_PKCS1_PSS_PADDING,
|
|
191942
|
+
saltLength: crypto4.constants.RSA_PSS_SALTLEN_DIGEST
|
|
191929
191943
|
}, "base64"));
|
|
191930
191944
|
return fromBase64(sig);
|
|
191931
191945
|
};
|
|
@@ -191935,12 +191949,12 @@ var require_jwa = __commonJS((exports, module) => {
|
|
|
191935
191949
|
checkIsPublicKey(publicKey);
|
|
191936
191950
|
thing = normalizeInput(thing);
|
|
191937
191951
|
signature = toBase64(signature);
|
|
191938
|
-
var verifier =
|
|
191952
|
+
var verifier = crypto4.createVerify("RSA-SHA" + bits);
|
|
191939
191953
|
verifier.update(thing);
|
|
191940
191954
|
return verifier.verify({
|
|
191941
191955
|
key: publicKey,
|
|
191942
|
-
padding:
|
|
191943
|
-
saltLength:
|
|
191956
|
+
padding: crypto4.constants.RSA_PKCS1_PSS_PADDING,
|
|
191957
|
+
saltLength: crypto4.constants.RSA_PSS_SALTLEN_DIGEST
|
|
191944
191958
|
}, signature, "base64");
|
|
191945
191959
|
};
|
|
191946
191960
|
}
|
|
@@ -192244,12 +192258,12 @@ var require_src113 = __commonJS((exports) => {
|
|
|
192244
192258
|
var _GoogleToken_requestToken;
|
|
192245
192259
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
192246
192260
|
exports.GoogleToken = undefined;
|
|
192247
|
-
var
|
|
192261
|
+
var fs4 = __require("fs");
|
|
192248
192262
|
var gaxios_1 = require_src53();
|
|
192249
192263
|
var jws = require_jws();
|
|
192250
192264
|
var path4 = __require("path");
|
|
192251
192265
|
var util_1 = __require("util");
|
|
192252
|
-
var readFile =
|
|
192266
|
+
var readFile = fs4.readFile ? (0, util_1.promisify)(fs4.readFile) : async () => {
|
|
192253
192267
|
throw new ErrorWithCode("use key rather than keyFile.", "MISSING_CREDENTIALS");
|
|
192254
192268
|
};
|
|
192255
192269
|
var GOOGLE_TOKEN_URL = "https://www.googleapis.com/oauth2/v4/token";
|
|
@@ -193462,10 +193476,10 @@ var require_filesubjecttokensupplier = __commonJS((exports) => {
|
|
|
193462
193476
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
193463
193477
|
exports.FileSubjectTokenSupplier = undefined;
|
|
193464
193478
|
var util_1 = __require("util");
|
|
193465
|
-
var
|
|
193466
|
-
var readFile = (0, util_1.promisify)((_a =
|
|
193467
|
-
var realpath = (0, util_1.promisify)((_b =
|
|
193468
|
-
var lstat = (0, util_1.promisify)((_c =
|
|
193479
|
+
var fs4 = __require("fs");
|
|
193480
|
+
var readFile = (0, util_1.promisify)((_a = fs4.readFile) !== null && _a !== undefined ? _a : () => {});
|
|
193481
|
+
var realpath = (0, util_1.promisify)((_b = fs4.realpath) !== null && _b !== undefined ? _b : () => {});
|
|
193482
|
+
var lstat = (0, util_1.promisify)((_c = fs4.lstat) !== null && _c !== undefined ? _c : () => {});
|
|
193469
193483
|
|
|
193470
193484
|
class FileSubjectTokenSupplier {
|
|
193471
193485
|
constructor(opts) {
|
|
@@ -193666,14 +193680,14 @@ var require_awsrequestsigner = __commonJS((exports) => {
|
|
|
193666
193680
|
}
|
|
193667
193681
|
}
|
|
193668
193682
|
exports.AwsRequestSigner = AwsRequestSigner;
|
|
193669
|
-
async function sign(
|
|
193670
|
-
return await
|
|
193671
|
-
}
|
|
193672
|
-
async function getSigningKey(
|
|
193673
|
-
const kDate = await sign(
|
|
193674
|
-
const kRegion = await sign(
|
|
193675
|
-
const kService = await sign(
|
|
193676
|
-
const kSigning = await sign(
|
|
193683
|
+
async function sign(crypto4, key, msg) {
|
|
193684
|
+
return await crypto4.signWithHmacSha256(key, msg);
|
|
193685
|
+
}
|
|
193686
|
+
async function getSigningKey(crypto4, key, dateStamp, region, serviceName) {
|
|
193687
|
+
const kDate = await sign(crypto4, `AWS4${key}`, dateStamp);
|
|
193688
|
+
const kRegion = await sign(crypto4, kDate, region);
|
|
193689
|
+
const kService = await sign(crypto4, kRegion, serviceName);
|
|
193690
|
+
const kSigning = await sign(crypto4, kService, "aws4_request");
|
|
193677
193691
|
return kSigning;
|
|
193678
193692
|
}
|
|
193679
193693
|
async function generateAuthenticationHeaderMap(options) {
|
|
@@ -194029,7 +194043,7 @@ var require_pluggable_auth_handler = __commonJS((exports) => {
|
|
|
194029
194043
|
var pluggable_auth_client_1 = require_pluggable_auth_client();
|
|
194030
194044
|
var executable_response_1 = require_executable_response();
|
|
194031
194045
|
var childProcess = __require("child_process");
|
|
194032
|
-
var
|
|
194046
|
+
var fs4 = __require("fs");
|
|
194033
194047
|
|
|
194034
194048
|
class PluggableAuthHandler {
|
|
194035
194049
|
constructor(options) {
|
|
@@ -194085,14 +194099,14 @@ var require_pluggable_auth_handler = __commonJS((exports) => {
|
|
|
194085
194099
|
}
|
|
194086
194100
|
let filePath;
|
|
194087
194101
|
try {
|
|
194088
|
-
filePath = await
|
|
194102
|
+
filePath = await fs4.promises.realpath(this.outputFile);
|
|
194089
194103
|
} catch (_a) {
|
|
194090
194104
|
return;
|
|
194091
194105
|
}
|
|
194092
|
-
if (!(await
|
|
194106
|
+
if (!(await fs4.promises.lstat(filePath)).isFile()) {
|
|
194093
194107
|
return;
|
|
194094
194108
|
}
|
|
194095
|
-
const responseString = await
|
|
194109
|
+
const responseString = await fs4.promises.readFile(filePath, {
|
|
194096
194110
|
encoding: "utf8"
|
|
194097
194111
|
});
|
|
194098
194112
|
if (responseString === "") {
|
|
@@ -194419,7 +194433,7 @@ var require_googleauth = __commonJS((exports) => {
|
|
|
194419
194433
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
194420
194434
|
exports.GoogleAuth = exports.GoogleAuthExceptionMessages = exports.CLOUD_SDK_CLIENT_ID = undefined;
|
|
194421
194435
|
var child_process_1 = __require("child_process");
|
|
194422
|
-
var
|
|
194436
|
+
var fs4 = __require("fs");
|
|
194423
194437
|
var gcpMetadata = require_src55();
|
|
194424
194438
|
var os2 = __require("os");
|
|
194425
194439
|
var path4 = __require("path");
|
|
@@ -194623,7 +194637,7 @@ var require_googleauth = __commonJS((exports) => {
|
|
|
194623
194637
|
}
|
|
194624
194638
|
if (location) {
|
|
194625
194639
|
location = path4.join(location, "gcloud", "application_default_credentials.json");
|
|
194626
|
-
if (!
|
|
194640
|
+
if (!fs4.existsSync(location)) {
|
|
194627
194641
|
location = null;
|
|
194628
194642
|
}
|
|
194629
194643
|
}
|
|
@@ -194638,8 +194652,8 @@ var require_googleauth = __commonJS((exports) => {
|
|
|
194638
194652
|
throw new Error("The file path is invalid.");
|
|
194639
194653
|
}
|
|
194640
194654
|
try {
|
|
194641
|
-
filePath =
|
|
194642
|
-
if (!
|
|
194655
|
+
filePath = fs4.realpathSync(filePath);
|
|
194656
|
+
if (!fs4.lstatSync(filePath).isFile()) {
|
|
194643
194657
|
throw new Error;
|
|
194644
194658
|
}
|
|
194645
194659
|
} catch (err) {
|
|
@@ -194648,7 +194662,7 @@ var require_googleauth = __commonJS((exports) => {
|
|
|
194648
194662
|
}
|
|
194649
194663
|
throw err;
|
|
194650
194664
|
}
|
|
194651
|
-
const readStream2 =
|
|
194665
|
+
const readStream2 = fs4.createReadStream(filePath);
|
|
194652
194666
|
return this.fromStream(readStream2, options);
|
|
194653
194667
|
}
|
|
194654
194668
|
fromImpersonatedJSON(json) {
|
|
@@ -194900,24 +194914,24 @@ var require_googleauth = __commonJS((exports) => {
|
|
|
194900
194914
|
const signed = await client.sign(data);
|
|
194901
194915
|
return signed.signedBlob;
|
|
194902
194916
|
}
|
|
194903
|
-
const
|
|
194917
|
+
const crypto4 = (0, crypto_1.createCrypto)();
|
|
194904
194918
|
if (client instanceof jwtclient_1.JWT && client.key) {
|
|
194905
|
-
const sign = await
|
|
194919
|
+
const sign = await crypto4.sign(client.key, data);
|
|
194906
194920
|
return sign;
|
|
194907
194921
|
}
|
|
194908
194922
|
const creds = await this.getCredentials();
|
|
194909
194923
|
if (!creds.client_email) {
|
|
194910
194924
|
throw new Error("Cannot sign data without `client_email`.");
|
|
194911
194925
|
}
|
|
194912
|
-
return this.signBlob(
|
|
194926
|
+
return this.signBlob(crypto4, creds.client_email, data, endpoint);
|
|
194913
194927
|
}
|
|
194914
|
-
async signBlob(
|
|
194928
|
+
async signBlob(crypto4, emailOrUniqueId, data, endpoint) {
|
|
194915
194929
|
const url2 = new URL(endpoint + `${emailOrUniqueId}:signBlob`);
|
|
194916
194930
|
const res = await this.request({
|
|
194917
194931
|
method: "POST",
|
|
194918
194932
|
url: url2.href,
|
|
194919
194933
|
data: {
|
|
194920
|
-
payload:
|
|
194934
|
+
payload: crypto4.encodeBase64StringUtf8(data)
|
|
194921
194935
|
},
|
|
194922
194936
|
retry: true,
|
|
194923
194937
|
retryConfig: {
|
|
@@ -194940,7 +194954,7 @@ var require_googleauth = __commonJS((exports) => {
|
|
|
194940
194954
|
return this._cacheClientFromJSON(this.jsonContent, this.clientOptions);
|
|
194941
194955
|
} else if (this.keyFilename) {
|
|
194942
194956
|
const filePath = path4.resolve(this.keyFilename);
|
|
194943
|
-
const stream4 =
|
|
194957
|
+
const stream4 = fs4.createReadStream(filePath);
|
|
194944
194958
|
return await this.fromStreamAsync(stream4, this.clientOptions);
|
|
194945
194959
|
} else if (this.apiKey) {
|
|
194946
194960
|
const client = await this.fromAPIKey(this.apiKey, this.clientOptions);
|
|
@@ -202694,7 +202708,7 @@ var require_table = __commonJS((exports) => {
|
|
|
202694
202708
|
var Big = require_big();
|
|
202695
202709
|
var extend2 = require_extend();
|
|
202696
202710
|
var events_1 = __require("events");
|
|
202697
|
-
var
|
|
202711
|
+
var fs4 = __require("fs");
|
|
202698
202712
|
var is = require_is();
|
|
202699
202713
|
var path4 = __require("path");
|
|
202700
202714
|
var streamEvents = require_stream_events();
|
|
@@ -203004,7 +203018,7 @@ var require_table = __commonJS((exports) => {
|
|
|
203004
203018
|
if (!metadata.sourceFormat && detectedFormat) {
|
|
203005
203019
|
metadata.sourceFormat = detectedFormat;
|
|
203006
203020
|
}
|
|
203007
|
-
const jobWritable =
|
|
203021
|
+
const jobWritable = fs4.createReadStream(source).pipe(this.createWriteStream_(metadata));
|
|
203008
203022
|
const [jobResponse] = await (0, events_1.once)(jobWritable, "job");
|
|
203009
203023
|
return [jobResponse, jobResponse.metadata];
|
|
203010
203024
|
}
|
|
@@ -208804,11 +208818,11 @@ var require_randomUUID = __commonJS((exports) => {
|
|
|
208804
208818
|
|
|
208805
208819
|
// ../../node_modules/@smithy/uuid/dist-cjs/index.js
|
|
208806
208820
|
var require_dist_cjs16 = __commonJS((exports) => {
|
|
208807
|
-
var
|
|
208821
|
+
var randomUUID3 = require_randomUUID();
|
|
208808
208822
|
var decimalToHex = Array.from({ length: 256 }, (_, i) => i.toString(16).padStart(2, "0"));
|
|
208809
208823
|
var v4 = () => {
|
|
208810
|
-
if (
|
|
208811
|
-
return
|
|
208824
|
+
if (randomUUID3.randomUUID) {
|
|
208825
|
+
return randomUUID3.randomUUID();
|
|
208812
208826
|
}
|
|
208813
208827
|
const rnds = new Uint8Array(16);
|
|
208814
208828
|
crypto.getRandomValues(rnds);
|
|
@@ -223129,7 +223143,7 @@ var require_dist_cjs55 = __commonJS((exports) => {
|
|
|
223129
223143
|
var utilBufferFrom = require_dist_cjs7();
|
|
223130
223144
|
var utilUtf8 = require_dist_cjs8();
|
|
223131
223145
|
var buffer = __require("buffer");
|
|
223132
|
-
var
|
|
223146
|
+
var crypto4 = __require("crypto");
|
|
223133
223147
|
|
|
223134
223148
|
class Hash {
|
|
223135
223149
|
algorithmIdentifier;
|
|
@@ -223147,7 +223161,7 @@ var require_dist_cjs55 = __commonJS((exports) => {
|
|
|
223147
223161
|
return Promise.resolve(this.hash.digest());
|
|
223148
223162
|
}
|
|
223149
223163
|
reset() {
|
|
223150
|
-
this.hash = this.secret ?
|
|
223164
|
+
this.hash = this.secret ? crypto4.createHmac(this.algorithmIdentifier, castSourceData(this.secret)) : crypto4.createHash(this.algorithmIdentifier);
|
|
223151
223165
|
}
|
|
223152
223166
|
}
|
|
223153
223167
|
function castSourceData(toCast, encoding) {
|
|
@@ -225787,7 +225801,7 @@ var require_dist_cjs59 = __commonJS((exports) => {
|
|
|
225787
225801
|
var httpAuthSchemes = require_httpAuthSchemes();
|
|
225788
225802
|
var propertyProvider = require_dist_cjs18();
|
|
225789
225803
|
var sharedIniFileLoader = require_dist_cjs44();
|
|
225790
|
-
var
|
|
225804
|
+
var fs4 = __require("fs");
|
|
225791
225805
|
var fromEnvSigningName = ({ logger: logger2, signingName } = {}) => async () => {
|
|
225792
225806
|
logger2?.debug?.("@aws-sdk/token-providers - fromEnvSigningName");
|
|
225793
225807
|
if (!signingName) {
|
|
@@ -225833,7 +225847,7 @@ var require_dist_cjs59 = __commonJS((exports) => {
|
|
|
225833
225847
|
throw new propertyProvider.TokenProviderError(`Value not present for '${key}' in SSO Token${forRefresh ? ". Cannot refresh" : ""}. ${REFRESH_MESSAGE}`, false);
|
|
225834
225848
|
}
|
|
225835
225849
|
};
|
|
225836
|
-
var { writeFile } =
|
|
225850
|
+
var { writeFile } = fs4.promises;
|
|
225837
225851
|
var writeSSOTokenToFile = (id, ssoToken) => {
|
|
225838
225852
|
const tokenFilepath = sharedIniFileLoader.getSSOTokenFilepath(id);
|
|
225839
225853
|
const tokenString = JSON.stringify(ssoToken, null, 2);
|
|
@@ -230045,7 +230059,7 @@ var require_dist_cjs70 = __commonJS((exports) => {
|
|
|
230045
230059
|
|
|
230046
230060
|
// ../../node_modules/@smithy/hash-stream-node/dist-cjs/index.js
|
|
230047
230061
|
var require_dist_cjs71 = __commonJS((exports) => {
|
|
230048
|
-
var
|
|
230062
|
+
var fs4 = __require("fs");
|
|
230049
230063
|
var utilUtf8 = require_dist_cjs8();
|
|
230050
230064
|
var stream4 = __require("stream");
|
|
230051
230065
|
|
|
@@ -230069,7 +230083,7 @@ var require_dist_cjs71 = __commonJS((exports) => {
|
|
|
230069
230083
|
reject(new Error("Unable to calculate hash for non-file streams."));
|
|
230070
230084
|
return;
|
|
230071
230085
|
}
|
|
230072
|
-
const fileStreamTee =
|
|
230086
|
+
const fileStreamTee = fs4.createReadStream(fileStream.path, {
|
|
230073
230087
|
start: fileStream.start,
|
|
230074
230088
|
end: fileStream.end
|
|
230075
230089
|
});
|
|
@@ -238478,9 +238492,9 @@ var require_dist11 = __commonJS((exports) => {
|
|
|
238478
238492
|
var require_pump = __commonJS((exports, module) => {
|
|
238479
238493
|
var once = require_once2();
|
|
238480
238494
|
var eos = require_end_of_stream4();
|
|
238481
|
-
var
|
|
238495
|
+
var fs4;
|
|
238482
238496
|
try {
|
|
238483
|
-
|
|
238497
|
+
fs4 = __require("fs");
|
|
238484
238498
|
} catch (e) {}
|
|
238485
238499
|
var noop2 = function() {};
|
|
238486
238500
|
var ancient = typeof process === "undefined" ? false : /^v?\.0/.test(process.version);
|
|
@@ -238490,9 +238504,9 @@ var require_pump = __commonJS((exports, module) => {
|
|
|
238490
238504
|
var isFS = function(stream4) {
|
|
238491
238505
|
if (!ancient)
|
|
238492
238506
|
return false;
|
|
238493
|
-
if (!
|
|
238507
|
+
if (!fs4)
|
|
238494
238508
|
return false;
|
|
238495
|
-
return (stream4 instanceof (
|
|
238509
|
+
return (stream4 instanceof (fs4.ReadStream || noop2) || stream4 instanceof (fs4.WriteStream || noop2)) && isFn(stream4.close);
|
|
238496
238510
|
};
|
|
238497
238511
|
var isRequest2 = function(stream4) {
|
|
238498
238512
|
return stream4.setHeader && isFn(stream4.abort);
|
|
@@ -238707,7 +238721,7 @@ var require_pend = __commonJS((exports, module) => {
|
|
|
238707
238721
|
|
|
238708
238722
|
// ../../node_modules/fd-slicer/index.js
|
|
238709
238723
|
var require_fd_slicer = __commonJS((exports) => {
|
|
238710
|
-
var
|
|
238724
|
+
var fs4 = __require("fs");
|
|
238711
238725
|
var util5 = __require("util");
|
|
238712
238726
|
var stream4 = __require("stream");
|
|
238713
238727
|
var Readable3 = stream4.Readable;
|
|
@@ -238732,7 +238746,7 @@ var require_fd_slicer = __commonJS((exports) => {
|
|
|
238732
238746
|
FdSlicer.prototype.read = function(buffer, offset, length, position, callback) {
|
|
238733
238747
|
var self2 = this;
|
|
238734
238748
|
self2.pend.go(function(cb) {
|
|
238735
|
-
|
|
238749
|
+
fs4.read(self2.fd, buffer, offset, length, position, function(err, bytesRead, buffer2) {
|
|
238736
238750
|
cb();
|
|
238737
238751
|
callback(err, bytesRead, buffer2);
|
|
238738
238752
|
});
|
|
@@ -238741,7 +238755,7 @@ var require_fd_slicer = __commonJS((exports) => {
|
|
|
238741
238755
|
FdSlicer.prototype.write = function(buffer, offset, length, position, callback) {
|
|
238742
238756
|
var self2 = this;
|
|
238743
238757
|
self2.pend.go(function(cb) {
|
|
238744
|
-
|
|
238758
|
+
fs4.write(self2.fd, buffer, offset, length, position, function(err, written, buffer2) {
|
|
238745
238759
|
cb();
|
|
238746
238760
|
callback(err, written, buffer2);
|
|
238747
238761
|
});
|
|
@@ -238764,7 +238778,7 @@ var require_fd_slicer = __commonJS((exports) => {
|
|
|
238764
238778
|
if (self2.refCount < 0)
|
|
238765
238779
|
throw new Error("invalid unref");
|
|
238766
238780
|
if (self2.autoClose) {
|
|
238767
|
-
|
|
238781
|
+
fs4.close(self2.fd, onCloseDone);
|
|
238768
238782
|
}
|
|
238769
238783
|
function onCloseDone(err) {
|
|
238770
238784
|
if (err) {
|
|
@@ -238803,7 +238817,7 @@ var require_fd_slicer = __commonJS((exports) => {
|
|
|
238803
238817
|
if (self2.destroyed)
|
|
238804
238818
|
return cb();
|
|
238805
238819
|
var buffer = new Buffer(toRead);
|
|
238806
|
-
|
|
238820
|
+
fs4.read(self2.context.fd, buffer, 0, toRead, self2.pos, function(err, bytesRead) {
|
|
238807
238821
|
if (err) {
|
|
238808
238822
|
self2.destroy(err);
|
|
238809
238823
|
} else if (bytesRead === 0) {
|
|
@@ -238853,7 +238867,7 @@ var require_fd_slicer = __commonJS((exports) => {
|
|
|
238853
238867
|
self2.context.pend.go(function(cb) {
|
|
238854
238868
|
if (self2.destroyed)
|
|
238855
238869
|
return cb();
|
|
238856
|
-
|
|
238870
|
+
fs4.write(self2.context.fd, buffer, 0, buffer.length, self2.pos, function(err2, bytes) {
|
|
238857
238871
|
if (err2) {
|
|
238858
238872
|
self2.destroy();
|
|
238859
238873
|
cb();
|
|
@@ -239278,7 +239292,7 @@ var require_buffer_crc32 = __commonJS((exports, module) => {
|
|
|
239278
239292
|
|
|
239279
239293
|
// ../../node_modules/yauzl/index.js
|
|
239280
239294
|
var require_yauzl = __commonJS((exports) => {
|
|
239281
|
-
var
|
|
239295
|
+
var fs4 = __require("fs");
|
|
239282
239296
|
var zlib2 = __require("zlib");
|
|
239283
239297
|
var fd_slicer = require_fd_slicer();
|
|
239284
239298
|
var crc32 = require_buffer_crc32();
|
|
@@ -239315,12 +239329,12 @@ var require_yauzl = __commonJS((exports) => {
|
|
|
239315
239329
|
options.strictFileNames = false;
|
|
239316
239330
|
if (callback == null)
|
|
239317
239331
|
callback = defaultCallback;
|
|
239318
|
-
|
|
239332
|
+
fs4.open(path4, "r", function(err, fd) {
|
|
239319
239333
|
if (err)
|
|
239320
239334
|
return callback(err);
|
|
239321
239335
|
fromFd(fd, options, function(err2, zipfile) {
|
|
239322
239336
|
if (err2)
|
|
239323
|
-
|
|
239337
|
+
fs4.close(fd, defaultCallback);
|
|
239324
239338
|
callback(err2, zipfile);
|
|
239325
239339
|
});
|
|
239326
239340
|
});
|
|
@@ -239344,7 +239358,7 @@ var require_yauzl = __commonJS((exports) => {
|
|
|
239344
239358
|
options.strictFileNames = false;
|
|
239345
239359
|
if (callback == null)
|
|
239346
239360
|
callback = defaultCallback;
|
|
239347
|
-
|
|
239361
|
+
fs4.fstat(fd, function(err, stats) {
|
|
239348
239362
|
if (err)
|
|
239349
239363
|
return callback(err);
|
|
239350
239364
|
var reader = fd_slicer.createFromFd(fd, { autoClose: true });
|
|
@@ -239977,7 +239991,7 @@ var require_yauzl = __commonJS((exports) => {
|
|
|
239977
239991
|
// ../../node_modules/extract-zip/index.js
|
|
239978
239992
|
var require_extract_zip = __commonJS((exports, module) => {
|
|
239979
239993
|
var debug = require_src5()("extract-zip");
|
|
239980
|
-
var { createWriteStream, promises:
|
|
239994
|
+
var { createWriteStream, promises: fs4 } = __require("fs");
|
|
239981
239995
|
var getStream = require_get_stream();
|
|
239982
239996
|
var path4 = __require("path");
|
|
239983
239997
|
var { promisify } = __require("util");
|
|
@@ -240019,8 +240033,8 @@ var require_extract_zip = __commonJS((exports, module) => {
|
|
|
240019
240033
|
}
|
|
240020
240034
|
const destDir = path4.dirname(path4.join(this.opts.dir, entry.fileName));
|
|
240021
240035
|
try {
|
|
240022
|
-
await
|
|
240023
|
-
const canonicalDestDir = await
|
|
240036
|
+
await fs4.mkdir(destDir, { recursive: true });
|
|
240037
|
+
const canonicalDestDir = await fs4.realpath(destDir);
|
|
240024
240038
|
const relativeDestDir = path4.relative(this.opts.dir, canonicalDestDir);
|
|
240025
240039
|
if (relativeDestDir.split(path4.sep).includes("..")) {
|
|
240026
240040
|
throw new Error(`Out of bound path "${canonicalDestDir}" found while processing file ${entry.fileName}`);
|
|
@@ -240065,7 +240079,7 @@ var require_extract_zip = __commonJS((exports, module) => {
|
|
|
240065
240079
|
mkdirOptions.mode = procMode;
|
|
240066
240080
|
}
|
|
240067
240081
|
debug("mkdir", { dir: destDir, ...mkdirOptions });
|
|
240068
|
-
await
|
|
240082
|
+
await fs4.mkdir(destDir, mkdirOptions);
|
|
240069
240083
|
if (isDir)
|
|
240070
240084
|
return;
|
|
240071
240085
|
debug("opening read stream", dest);
|
|
@@ -240073,7 +240087,7 @@ var require_extract_zip = __commonJS((exports, module) => {
|
|
|
240073
240087
|
if (symlink) {
|
|
240074
240088
|
const link = await getStream(readStream2);
|
|
240075
240089
|
debug("creating symlink", link, dest);
|
|
240076
|
-
await
|
|
240090
|
+
await fs4.symlink(link, dest);
|
|
240077
240091
|
} else {
|
|
240078
240092
|
await pipeline(readStream2, createWriteStream(dest, { mode: procMode }));
|
|
240079
240093
|
}
|
|
@@ -240105,8 +240119,8 @@ var require_extract_zip = __commonJS((exports, module) => {
|
|
|
240105
240119
|
if (!path4.isAbsolute(opts.dir)) {
|
|
240106
240120
|
throw new Error("Target directory is expected to be absolute");
|
|
240107
240121
|
}
|
|
240108
|
-
await
|
|
240109
|
-
opts.dir = await
|
|
240122
|
+
await fs4.mkdir(opts.dir, { recursive: true });
|
|
240123
|
+
opts.dir = await fs4.realpath(opts.dir);
|
|
240110
240124
|
return new Extractor(zipPath, opts).extract();
|
|
240111
240125
|
};
|
|
240112
240126
|
});
|
|
@@ -240690,10 +240704,10 @@ var init_package_load_pool = __esm(() => {
|
|
|
240690
240704
|
})();
|
|
240691
240705
|
defaultUrlReader = {
|
|
240692
240706
|
readURL: async (url2) => {
|
|
240693
|
-
const { promises:
|
|
240707
|
+
const { promises: fs4 } = await import("fs");
|
|
240694
240708
|
const { fileURLToPath: fileURLToPath3 } = await import("url");
|
|
240695
240709
|
const filePath = url2.protocol === "file:" ? fileURLToPath3(url2) : url2.toString();
|
|
240696
|
-
return
|
|
240710
|
+
return fs4.readFile(filePath, "utf8");
|
|
240697
240711
|
}
|
|
240698
240712
|
};
|
|
240699
240713
|
});
|
|
@@ -241504,7 +241518,7 @@ globstar while`, file, fr, pattern, pr, swallowee);
|
|
|
241504
241518
|
|
|
241505
241519
|
// ../../node_modules/recursive-readdir/index.js
|
|
241506
241520
|
var require_recursive_readdir = __commonJS((exports, module) => {
|
|
241507
|
-
var
|
|
241521
|
+
var fs6 = __require("fs");
|
|
241508
241522
|
var p = __require("path");
|
|
241509
241523
|
var minimatch = require_minimatch();
|
|
241510
241524
|
function patternMatcher(pattern) {
|
|
@@ -241538,7 +241552,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
241538
241552
|
}
|
|
241539
241553
|
ignores = ignores.map(toMatcherFunction);
|
|
241540
241554
|
var list = [];
|
|
241541
|
-
|
|
241555
|
+
fs6.readdir(path6, function(err, files) {
|
|
241542
241556
|
if (err) {
|
|
241543
241557
|
return callback(err);
|
|
241544
241558
|
}
|
|
@@ -241548,7 +241562,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
241548
241562
|
}
|
|
241549
241563
|
files.forEach(function(file) {
|
|
241550
241564
|
var filePath = p.join(path6, file);
|
|
241551
|
-
|
|
241565
|
+
fs6.stat(filePath, function(_err, stats) {
|
|
241552
241566
|
if (_err) {
|
|
241553
241567
|
return callback(_err);
|
|
241554
241568
|
}
|
|
@@ -256134,6 +256148,68 @@ function httpMetricsMiddleware(req, res, next) {
|
|
|
256134
256148
|
next();
|
|
256135
256149
|
}
|
|
256136
256150
|
|
|
256151
|
+
// src/node_version_check.ts
|
|
256152
|
+
import * as fs from "node:fs";
|
|
256153
|
+
var MIN_NODE_MAJOR = 20;
|
|
256154
|
+
var REQUIRED_NODE_RANGE = `>=${MIN_NODE_MAJOR}`;
|
|
256155
|
+
var UNSUPPORTED_NODE_TOKEN = "PUBLISHER_UNSUPPORTED_NODE";
|
|
256156
|
+
function evaluateRuntime(versions) {
|
|
256157
|
+
if (versions.bunVersion) {
|
|
256158
|
+
return { supported: true };
|
|
256159
|
+
}
|
|
256160
|
+
const major = parseMajor(versions.nodeVersion);
|
|
256161
|
+
if (major === undefined || major >= MIN_NODE_MAJOR) {
|
|
256162
|
+
return { supported: true };
|
|
256163
|
+
}
|
|
256164
|
+
return {
|
|
256165
|
+
supported: false,
|
|
256166
|
+
message: unsupportedMessage(versions.nodeVersion)
|
|
256167
|
+
};
|
|
256168
|
+
}
|
|
256169
|
+
function parseMajor(version) {
|
|
256170
|
+
const match = /^v?(\d+)\./.exec(version.trim());
|
|
256171
|
+
if (!match) {
|
|
256172
|
+
return;
|
|
256173
|
+
}
|
|
256174
|
+
const major = Number(match[1]);
|
|
256175
|
+
return Number.isFinite(major) ? major : undefined;
|
|
256176
|
+
}
|
|
256177
|
+
function unsupportedMessage(nodeVersion) {
|
|
256178
|
+
return [
|
|
256179
|
+
`${UNSUPPORTED_NODE_TOKEN} required=${REQUIRED_NODE_RANGE} detected=${nodeVersion}`,
|
|
256180
|
+
`Malloy Publisher supports Node.js ${MIN_NODE_MAJOR} and newer, but this process is running Node.js ${nodeVersion}.`,
|
|
256181
|
+
`Publisher and every Malloy library it depends on declare Node ${REQUIRED_NODE_RANGE}, so this`,
|
|
256182
|
+
`runtime is untested and has failed in ways that never mention Node.`,
|
|
256183
|
+
`Upgrade Node, then run the command again: https://nodejs.org`,
|
|
256184
|
+
` nvm: nvm install ${MIN_NODE_MAJOR} && nvm use ${MIN_NODE_MAJOR}`,
|
|
256185
|
+
` mise: mise use -g node@${MIN_NODE_MAJOR} then open a new shell before running the command`,
|
|
256186
|
+
""
|
|
256187
|
+
].join(`
|
|
256188
|
+
`);
|
|
256189
|
+
}
|
|
256190
|
+
function assertSupportedNodeVersion(options = {}) {
|
|
256191
|
+
const verdict = evaluateRuntime({
|
|
256192
|
+
nodeVersion: "nodeVersion" in options && options.nodeVersion !== undefined ? options.nodeVersion : process.version,
|
|
256193
|
+
bunVersion: "bunVersion" in options ? options.bunVersion : process.versions.bun
|
|
256194
|
+
});
|
|
256195
|
+
if (verdict.supported) {
|
|
256196
|
+
return;
|
|
256197
|
+
}
|
|
256198
|
+
const write = options.write ?? writeToStderr;
|
|
256199
|
+
const exit = options.exit ?? ((code) => process.exit(code));
|
|
256200
|
+
try {
|
|
256201
|
+
write(verdict.message ?? "");
|
|
256202
|
+
} catch {}
|
|
256203
|
+
exit(1);
|
|
256204
|
+
}
|
|
256205
|
+
function writeToStderr(text) {
|
|
256206
|
+
try {
|
|
256207
|
+
fs.writeSync(2, text);
|
|
256208
|
+
} catch {
|
|
256209
|
+
process.stderr.write(text);
|
|
256210
|
+
}
|
|
256211
|
+
}
|
|
256212
|
+
|
|
256137
256213
|
// ../../node_modules/zod/v3/external.js
|
|
256138
256214
|
var exports_external = {};
|
|
256139
256215
|
__export(exports_external, {
|
|
@@ -261175,23 +261251,23 @@ class CompileController {
|
|
|
261175
261251
|
// src/config.ts
|
|
261176
261252
|
init_constants();
|
|
261177
261253
|
init_logger();
|
|
261178
|
-
import
|
|
261254
|
+
import fs2 from "fs";
|
|
261179
261255
|
import path from "path";
|
|
261180
261256
|
import { fileURLToPath } from "url";
|
|
261181
261257
|
var BUNDLED_DEFAULT_CONFIG_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), "default-publisher.config.json");
|
|
261182
261258
|
function resolvePublisherConfigPath(serverRoot) {
|
|
261183
261259
|
const explicitPath = process.env.PUBLISHER_CONFIG_PATH;
|
|
261184
261260
|
if (explicitPath && explicitPath.length > 0) {
|
|
261185
|
-
if (!
|
|
261261
|
+
if (!fs2.existsSync(explicitPath)) {
|
|
261186
261262
|
return null;
|
|
261187
261263
|
}
|
|
261188
261264
|
return { path: explicitPath, isBundledDefault: false };
|
|
261189
261265
|
}
|
|
261190
261266
|
const serverRootPath = path.join(serverRoot, PUBLISHER_CONFIG_NAME);
|
|
261191
|
-
if (
|
|
261267
|
+
if (fs2.existsSync(serverRootPath)) {
|
|
261192
261268
|
return { path: serverRootPath, isBundledDefault: false };
|
|
261193
261269
|
}
|
|
261194
|
-
if (process.env.PUBLISHER_USE_BUNDLED_DEFAULT === "true" &&
|
|
261270
|
+
if (process.env.PUBLISHER_USE_BUNDLED_DEFAULT === "true" && fs2.existsSync(BUNDLED_DEFAULT_CONFIG_PATH)) {
|
|
261195
261271
|
return { path: BUNDLED_DEFAULT_CONFIG_PATH, isBundledDefault: true };
|
|
261196
261272
|
}
|
|
261197
261273
|
return null;
|
|
@@ -261423,7 +261499,7 @@ var getPublisherConfigDir = (serverRoot) => {
|
|
|
261423
261499
|
return null;
|
|
261424
261500
|
}
|
|
261425
261501
|
try {
|
|
261426
|
-
if (!
|
|
261502
|
+
if (!fs2.statSync(resolved.path).isFile()) {
|
|
261427
261503
|
return null;
|
|
261428
261504
|
}
|
|
261429
261505
|
} catch {
|
|
@@ -261448,7 +261524,7 @@ var getPublisherConfig = (serverRoot) => {
|
|
|
261448
261524
|
}
|
|
261449
261525
|
let rawConfig;
|
|
261450
261526
|
try {
|
|
261451
|
-
const fileContent =
|
|
261527
|
+
const fileContent = fs2.readFileSync(publisherConfigPath, "utf8");
|
|
261452
261528
|
rawConfig = JSON.parse(fileContent);
|
|
261453
261529
|
} catch (error) {
|
|
261454
261530
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -261655,6 +261731,7 @@ var getProcessedPublisherConfig = (serverRoot) => {
|
|
|
261655
261731
|
name: environment.name,
|
|
261656
261732
|
packages: validPackages,
|
|
261657
261733
|
connections: convertConnectionsToApiConnections(environment.connections || []),
|
|
261734
|
+
storageDestinations: Array.isArray(environment.storageDestinations) ? environment.storageDestinations : [],
|
|
261658
261735
|
...resolvedTheme ? { theme: resolvedTheme } : {}
|
|
261659
261736
|
});
|
|
261660
261737
|
}
|
|
@@ -265064,7 +265141,7 @@ var {
|
|
|
265064
265141
|
} = axios_default;
|
|
265065
265142
|
|
|
265066
265143
|
// src/service/connection.ts
|
|
265067
|
-
import
|
|
265144
|
+
import fs3 from "fs/promises";
|
|
265068
265145
|
|
|
265069
265146
|
// src/ducklake_version.ts
|
|
265070
265147
|
var MIN_CATALOG_FORMAT = "1.0";
|
|
@@ -265185,6 +265262,7 @@ function safeJoinUnderRoot(root, ...segments) {
|
|
|
265185
265262
|
}
|
|
265186
265263
|
|
|
265187
265264
|
// src/service/connection_config.ts
|
|
265265
|
+
init_errors();
|
|
265188
265266
|
init_logger();
|
|
265189
265267
|
import { createPrivateKey } from "crypto";
|
|
265190
265268
|
import { existsSync } from "fs";
|
|
@@ -265317,6 +265395,9 @@ function openSshProxy(ssh, target) {
|
|
|
265317
265395
|
});
|
|
265318
265396
|
}
|
|
265319
265397
|
|
|
265398
|
+
// src/service/query_metadata.ts
|
|
265399
|
+
import * as crypto3 from "node:crypto";
|
|
265400
|
+
|
|
265320
265401
|
// src/query_metadata_metrics.ts
|
|
265321
265402
|
var resetHooks = [];
|
|
265322
265403
|
function lazyCounter(name, description) {
|
|
@@ -265399,7 +265480,7 @@ function queryMetadataBudgetWarning(declared) {
|
|
|
265399
265480
|
return `queryMetadata declares ${declared} properties; the server adds up to ` + `${RESERVED_CONTEXT_PROPERTIES} of its own to every statement, so a bag over ` + `${authorBudget} loses its least specific properties at query time`;
|
|
265400
265481
|
}
|
|
265401
265482
|
function mintCorrelationId() {
|
|
265402
|
-
return
|
|
265483
|
+
return crypto3.randomUUID();
|
|
265403
265484
|
}
|
|
265404
265485
|
var CONTEXT_SHED_ORDER = [
|
|
265405
265486
|
"version",
|
|
@@ -265865,6 +265946,106 @@ function validateConnectionShape(connection) {
|
|
|
265865
265946
|
}
|
|
265866
265947
|
}
|
|
265867
265948
|
}
|
|
265949
|
+
var DECLARABLE_STORAGE_DESTINATION_TYPES = new Set(["ducklake"]);
|
|
265950
|
+
var STORAGE_DESTINATIONS_DIR = ".storage-destinations";
|
|
265951
|
+
function storageDestinationRoot(environmentPath) {
|
|
265952
|
+
return path3.join(environmentPath, STORAGE_DESTINATIONS_DIR);
|
|
265953
|
+
}
|
|
265954
|
+
function validateStorageDestinations(destinations = []) {
|
|
265955
|
+
const accepted = [];
|
|
265956
|
+
const rejected = [];
|
|
265957
|
+
if (!Array.isArray(destinations)) {
|
|
265958
|
+
return {
|
|
265959
|
+
accepted,
|
|
265960
|
+
rejected: [{ reason: "the value is not a list of destinations." }]
|
|
265961
|
+
};
|
|
265962
|
+
}
|
|
265963
|
+
const seen = new Set;
|
|
265964
|
+
for (const destination of destinations) {
|
|
265965
|
+
if (!destination || typeof destination !== "object") {
|
|
265966
|
+
rejected.push({ reason: "the entry is not an object." });
|
|
265967
|
+
continue;
|
|
265968
|
+
}
|
|
265969
|
+
const { name, type } = destination;
|
|
265970
|
+
if (!name || typeof name !== "string") {
|
|
265971
|
+
rejected.push({
|
|
265972
|
+
reason: `the "name" field is missing or not a string.`
|
|
265973
|
+
});
|
|
265974
|
+
continue;
|
|
265975
|
+
}
|
|
265976
|
+
if (!type || !DECLARABLE_STORAGE_DESTINATION_TYPES.has(type)) {
|
|
265977
|
+
rejected.push({
|
|
265978
|
+
name,
|
|
265979
|
+
reason: `the type "${type}" is not a supported destination type ` + `(supported: ${[...DECLARABLE_STORAGE_DESTINATION_TYPES].join(", ")}).`
|
|
265980
|
+
});
|
|
265981
|
+
continue;
|
|
265982
|
+
}
|
|
265983
|
+
if (name === "duckdb") {
|
|
265984
|
+
rejected.push({
|
|
265985
|
+
name,
|
|
265986
|
+
reason: `the name "duckdb" is reserved for per-package sandboxes.`
|
|
265987
|
+
});
|
|
265988
|
+
continue;
|
|
265989
|
+
}
|
|
265990
|
+
if (seen.has(name)) {
|
|
265991
|
+
rejected.push({
|
|
265992
|
+
name,
|
|
265993
|
+
reason: "an earlier entry already uses this name."
|
|
265994
|
+
});
|
|
265995
|
+
continue;
|
|
265996
|
+
}
|
|
265997
|
+
try {
|
|
265998
|
+
validateConnectionShape(destination);
|
|
265999
|
+
} catch (error) {
|
|
266000
|
+
rejected.push({ name, reason: error.message });
|
|
266001
|
+
continue;
|
|
266002
|
+
}
|
|
266003
|
+
seen.add(name);
|
|
266004
|
+
const { resource: _resource, ...body } = destination;
|
|
266005
|
+
accepted.push(body);
|
|
266006
|
+
}
|
|
266007
|
+
return { accepted, rejected };
|
|
266008
|
+
}
|
|
266009
|
+
function describeRejection(rejection) {
|
|
266010
|
+
return rejection.name ? `storage destination "${rejection.name}": ${rejection.reason}` : `storage destination: ${rejection.reason}`;
|
|
266011
|
+
}
|
|
266012
|
+
function processStorageDestinations(destinations = []) {
|
|
266013
|
+
const { accepted, rejected } = validateStorageDestinations(destinations);
|
|
266014
|
+
for (const rejection of rejected) {
|
|
266015
|
+
logger.warn(`Invalid ${describeRejection(rejection)} Skipping.`);
|
|
266016
|
+
}
|
|
266017
|
+
return accepted;
|
|
266018
|
+
}
|
|
266019
|
+
function processStorageDestinationsOrThrow(destinations = []) {
|
|
266020
|
+
const { accepted, rejected } = validateStorageDestinations(destinations);
|
|
266021
|
+
if (rejected.length > 0) {
|
|
266022
|
+
throw new BadRequestError(`storageDestinations was not accepted — ${rejected.map(describeRejection).join(" ")}`);
|
|
266023
|
+
}
|
|
266024
|
+
return accepted;
|
|
266025
|
+
}
|
|
266026
|
+
function storageDestinationsEqual(left, right) {
|
|
266027
|
+
if (left.length !== right.length) {
|
|
266028
|
+
return false;
|
|
266029
|
+
}
|
|
266030
|
+
return canonicalDestinationList(left) === canonicalDestinationList(right);
|
|
266031
|
+
}
|
|
266032
|
+
function canonicalDestinationList(destinations) {
|
|
266033
|
+
return JSON.stringify([...destinations].sort((left, right) => (left.name ?? "") < (right.name ?? "") ? -1 : 1).map(canonicalizeJsonValue));
|
|
266034
|
+
}
|
|
266035
|
+
function canonicalizeJsonValue(value) {
|
|
266036
|
+
if (Array.isArray(value)) {
|
|
266037
|
+
return value.map(canonicalizeJsonValue);
|
|
266038
|
+
}
|
|
266039
|
+
if (value === null || typeof value !== "object") {
|
|
266040
|
+
return value;
|
|
266041
|
+
}
|
|
266042
|
+
const entries = value;
|
|
266043
|
+
const sorted = {};
|
|
266044
|
+
for (const key of Object.keys(entries).sort()) {
|
|
266045
|
+
sorted[key] = canonicalizeJsonValue(entries[key]);
|
|
266046
|
+
}
|
|
266047
|
+
return sorted;
|
|
266048
|
+
}
|
|
265868
266049
|
function assembleEnvironmentConnections(connections = [], environmentPath = "") {
|
|
265869
266050
|
const pojo = { connections: {} };
|
|
265870
266051
|
const metadata = new Map;
|
|
@@ -266787,8 +266968,8 @@ async function deleteDuckLakeConnectionFile(connectionName, environmentPath) {
|
|
|
266787
266968
|
assertSafeEnvironmentPath(environmentPath);
|
|
266788
266969
|
const ducklakePath = safeJoinUnderRoot(environmentPath, `${connectionName}_ducklake.duckdb`);
|
|
266789
266970
|
try {
|
|
266790
|
-
await
|
|
266791
|
-
await
|
|
266971
|
+
await fs3.access(ducklakePath);
|
|
266972
|
+
await fs3.rm(ducklakePath);
|
|
266792
266973
|
logger.info(`Removed DuckLake connection file ${connectionName}_ducklake.duckdb from ${environmentPath}`);
|
|
266793
266974
|
} catch (error) {
|
|
266794
266975
|
if (error.code === "ENOENT") {
|
|
@@ -266798,6 +266979,18 @@ async function deleteDuckLakeConnectionFile(connectionName, environmentPath) {
|
|
|
266798
266979
|
}
|
|
266799
266980
|
}
|
|
266800
266981
|
}
|
|
266982
|
+
function restrictMalloyConfigToConnections(base, allowed) {
|
|
266983
|
+
const restricted = new MalloyConfig({ connections: {} });
|
|
266984
|
+
restricted.wrapConnections(() => ({
|
|
266985
|
+
lookupConnection: async (name) => {
|
|
266986
|
+
if (!name || !allowed.has(name)) {
|
|
266987
|
+
throw new ConnectionNotFoundError(`Connection ${name ?? "(default)"} is not available to a materialization serve shape`);
|
|
266988
|
+
}
|
|
266989
|
+
return base.connections.lookupConnection(name);
|
|
266990
|
+
}
|
|
266991
|
+
}));
|
|
266992
|
+
return restricted;
|
|
266993
|
+
}
|
|
266801
266994
|
function entryToDuckDBOptions(name, entry, workingDirectory) {
|
|
266802
266995
|
const { is: _is, ...rest } = entry;
|
|
266803
266996
|
if (workingDirectory !== undefined) {
|
|
@@ -269082,6 +269275,7 @@ class QueryController {
|
|
|
269082
269275
|
} else {
|
|
269083
269276
|
const {
|
|
269084
269277
|
result,
|
|
269278
|
+
serializedResult,
|
|
269085
269279
|
compactResult,
|
|
269086
269280
|
rowLimit,
|
|
269087
269281
|
rowLimitSource,
|
|
@@ -269107,7 +269301,7 @@ class QueryController {
|
|
|
269107
269301
|
}), getQueryTimeoutMs());
|
|
269108
269302
|
const renderLogs = import_render_validator.validateRenderTags(result);
|
|
269109
269303
|
return {
|
|
269110
|
-
result: compactJson ? JSON.stringify(compactResult, bigIntReplacer) :
|
|
269304
|
+
result: compactJson ? JSON.stringify(compactResult, bigIntReplacer) : serializedResult,
|
|
269111
269305
|
resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}/models/${modelPath}/query`,
|
|
269112
269306
|
renderLogs: renderLogs.length > 0 ? renderLogs : undefined,
|
|
269113
269307
|
queryRowLimit: rowLimit,
|
|
@@ -271027,8 +271221,8 @@ function tryAcquire(sync, alreadyAcquiredError = E_ALREADY_LOCKED) {
|
|
|
271027
271221
|
|
|
271028
271222
|
// src/service/environment_store.ts
|
|
271029
271223
|
var import_extract_zip = __toESM(require_extract_zip(), 1);
|
|
271030
|
-
import
|
|
271031
|
-
import * as
|
|
271224
|
+
import crypto5 from "crypto";
|
|
271225
|
+
import * as fs9 from "fs";
|
|
271032
271226
|
import * as os2 from "os";
|
|
271033
271227
|
import * as path9 from "path";
|
|
271034
271228
|
|
|
@@ -275885,6 +276079,66 @@ class EnvironmentRepository {
|
|
|
275885
276079
|
}
|
|
275886
276080
|
}
|
|
275887
276081
|
|
|
276082
|
+
// src/storage/duckdb/StorageDestinationRepository.ts
|
|
276083
|
+
class StorageDestinationRepository {
|
|
276084
|
+
db;
|
|
276085
|
+
constructor(db) {
|
|
276086
|
+
this.db = db;
|
|
276087
|
+
}
|
|
276088
|
+
generateId() {
|
|
276089
|
+
return `${Date.now()}-${Math.random().toString(36).slice(2, 11)}`;
|
|
276090
|
+
}
|
|
276091
|
+
async list(environmentId) {
|
|
276092
|
+
const rows = await this.db.all("SELECT * FROM storage_destinations WHERE environment_id = ? ORDER BY name", [environmentId]);
|
|
276093
|
+
return rows.map(this.mapToDestination);
|
|
276094
|
+
}
|
|
276095
|
+
async getByName(environmentId, name) {
|
|
276096
|
+
const row = await this.db.get("SELECT * FROM storage_destinations WHERE environment_id = ? AND name = ?", [environmentId, name]);
|
|
276097
|
+
return row ? this.mapToDestination(row) : null;
|
|
276098
|
+
}
|
|
276099
|
+
async upsert(destination) {
|
|
276100
|
+
const existing = await this.getByName(destination.environmentId, destination.name);
|
|
276101
|
+
const now = new Date;
|
|
276102
|
+
const configJson = JSON.stringify(destination.config);
|
|
276103
|
+
if (existing) {
|
|
276104
|
+
await this.db.run(`UPDATE storage_destinations
|
|
276105
|
+
SET type = ?, config = ?, updated_at = ?
|
|
276106
|
+
WHERE id = ?`, [destination.type, configJson, now.toISOString(), existing.id]);
|
|
276107
|
+
return { ...existing, ...destination, updatedAt: now };
|
|
276108
|
+
}
|
|
276109
|
+
const id = this.generateId();
|
|
276110
|
+
await this.db.run(`INSERT INTO storage_destinations
|
|
276111
|
+
(id, environment_id, name, type, config, created_at, updated_at)
|
|
276112
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`, [
|
|
276113
|
+
id,
|
|
276114
|
+
destination.environmentId,
|
|
276115
|
+
destination.name,
|
|
276116
|
+
destination.type,
|
|
276117
|
+
configJson,
|
|
276118
|
+
now.toISOString(),
|
|
276119
|
+
now.toISOString()
|
|
276120
|
+
]);
|
|
276121
|
+
return { id, ...destination, createdAt: now, updatedAt: now };
|
|
276122
|
+
}
|
|
276123
|
+
async deleteById(id) {
|
|
276124
|
+
await this.db.run("DELETE FROM storage_destinations WHERE id = ?", [id]);
|
|
276125
|
+
}
|
|
276126
|
+
async deleteByEnvironmentId(environmentId) {
|
|
276127
|
+
await this.db.run("DELETE FROM storage_destinations WHERE environment_id = ?", [environmentId]);
|
|
276128
|
+
}
|
|
276129
|
+
mapToDestination(row) {
|
|
276130
|
+
return {
|
|
276131
|
+
id: row.id,
|
|
276132
|
+
environmentId: row.environment_id,
|
|
276133
|
+
name: row.name,
|
|
276134
|
+
type: row.type,
|
|
276135
|
+
config: JSON.parse(row.config),
|
|
276136
|
+
createdAt: new Date(row.created_at),
|
|
276137
|
+
updatedAt: new Date(row.updated_at)
|
|
276138
|
+
};
|
|
276139
|
+
}
|
|
276140
|
+
}
|
|
276141
|
+
|
|
275888
276142
|
// src/storage/duckdb/MaterializationRepository.ts
|
|
275889
276143
|
var TERMINAL_STATUSES = new Set([
|
|
275890
276144
|
"MANIFEST_FILE_READY",
|
|
@@ -276173,12 +276427,14 @@ class DuckDBRepository {
|
|
|
276173
276427
|
environmentRepo;
|
|
276174
276428
|
packageRepo;
|
|
276175
276429
|
connectionRepo;
|
|
276430
|
+
destinationRepo;
|
|
276176
276431
|
materializationRepo;
|
|
276177
276432
|
constructor(db) {
|
|
276178
276433
|
this.db = db;
|
|
276179
276434
|
this.environmentRepo = new EnvironmentRepository(db);
|
|
276180
276435
|
this.packageRepo = new PackageRepository(db);
|
|
276181
276436
|
this.connectionRepo = new ConnectionRepository(db);
|
|
276437
|
+
this.destinationRepo = new StorageDestinationRepository(db);
|
|
276182
276438
|
this.materializationRepo = new MaterializationRepository(db);
|
|
276183
276439
|
}
|
|
276184
276440
|
async listEnvironments() {
|
|
@@ -276199,6 +276455,7 @@ class DuckDBRepository {
|
|
|
276199
276455
|
async deleteEnvironment(id) {
|
|
276200
276456
|
await this.materializationRepo.deleteByEnvironmentId(id);
|
|
276201
276457
|
await this.connectionRepo.deleteConnectionsByEnvironmentId(id);
|
|
276458
|
+
await this.destinationRepo.deleteByEnvironmentId(id);
|
|
276202
276459
|
await this.packageRepo.deletePackagesByEnvironmentId(id);
|
|
276203
276460
|
await this.environmentRepo.deleteEnvironment(id);
|
|
276204
276461
|
}
|
|
@@ -276248,6 +276505,21 @@ class DuckDBRepository {
|
|
|
276248
276505
|
async deleteConnectionsByEnvironmentId(id) {
|
|
276249
276506
|
return this.connectionRepo.deleteConnectionsByEnvironmentId(id);
|
|
276250
276507
|
}
|
|
276508
|
+
async listStorageDestinations(environmentId) {
|
|
276509
|
+
return this.destinationRepo.list(environmentId);
|
|
276510
|
+
}
|
|
276511
|
+
async getStorageDestinationByName(environmentId, name) {
|
|
276512
|
+
return this.destinationRepo.getByName(environmentId, name);
|
|
276513
|
+
}
|
|
276514
|
+
async upsertStorageDestination(destination) {
|
|
276515
|
+
return this.destinationRepo.upsert(destination);
|
|
276516
|
+
}
|
|
276517
|
+
async deleteStorageDestination(id) {
|
|
276518
|
+
return this.destinationRepo.deleteById(id);
|
|
276519
|
+
}
|
|
276520
|
+
async deleteStorageDestinationsByEnvironmentId(id) {
|
|
276521
|
+
return this.destinationRepo.deleteByEnvironmentId(id);
|
|
276522
|
+
}
|
|
276251
276523
|
async listMaterializations(environmentId, packageName, options) {
|
|
276252
276524
|
return this.materializationRepo.list(environmentId, packageName, options);
|
|
276253
276525
|
}
|
|
@@ -276323,6 +276595,19 @@ async function initializeSchema(db, force = false) {
|
|
|
276323
276595
|
UNIQUE (environment_id, name)
|
|
276324
276596
|
)
|
|
276325
276597
|
`);
|
|
276598
|
+
await db.run(`
|
|
276599
|
+
CREATE TABLE IF NOT EXISTS storage_destinations (
|
|
276600
|
+
id VARCHAR PRIMARY KEY,
|
|
276601
|
+
environment_id VARCHAR NOT NULL,
|
|
276602
|
+
name VARCHAR NOT NULL,
|
|
276603
|
+
type VARCHAR NOT NULL,
|
|
276604
|
+
config JSON NOT NULL,
|
|
276605
|
+
created_at TIMESTAMP NOT NULL,
|
|
276606
|
+
updated_at TIMESTAMP NOT NULL,
|
|
276607
|
+
FOREIGN KEY (environment_id) REFERENCES environments(id),
|
|
276608
|
+
UNIQUE (environment_id, name)
|
|
276609
|
+
)
|
|
276610
|
+
`);
|
|
276326
276611
|
await db.run(`
|
|
276327
276612
|
CREATE TABLE IF NOT EXISTS materializations (
|
|
276328
276613
|
id VARCHAR PRIMARY KEY,
|
|
@@ -276351,6 +276636,7 @@ async function initializeSchema(db, force = false) {
|
|
|
276351
276636
|
await createEntityEmbeddingsTable(db);
|
|
276352
276637
|
await db.run("CREATE INDEX IF NOT EXISTS idx_packages_environment_id ON packages(environment_id)");
|
|
276353
276638
|
await db.run("CREATE INDEX IF NOT EXISTS idx_connections_environment_id ON connections(environment_id)");
|
|
276639
|
+
await db.run("CREATE INDEX IF NOT EXISTS idx_storage_destinations_environment_id ON storage_destinations(environment_id)");
|
|
276354
276640
|
await db.run("CREATE INDEX IF NOT EXISTS idx_materializations_environment_package ON materializations(environment_id, package_name)");
|
|
276355
276641
|
await db.run("CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)");
|
|
276356
276642
|
}
|
|
@@ -276399,6 +276685,7 @@ async function dropAllTables(db) {
|
|
|
276399
276685
|
"materializations",
|
|
276400
276686
|
"packages",
|
|
276401
276687
|
"connections",
|
|
276688
|
+
"storage_destinations",
|
|
276402
276689
|
"environments",
|
|
276403
276690
|
"themes",
|
|
276404
276691
|
"entity_embeddings"
|
|
@@ -276478,10 +276765,221 @@ class StorageManager {
|
|
|
276478
276765
|
import { MalloyError as MalloyError4, Runtime as Runtime3 } from "@malloydata/malloy";
|
|
276479
276766
|
init_constants();
|
|
276480
276767
|
init_errors();
|
|
276481
|
-
import
|
|
276482
|
-
import * as
|
|
276768
|
+
import crypto4 from "crypto";
|
|
276769
|
+
import * as fs8 from "fs";
|
|
276483
276770
|
import * as path8 from "path";
|
|
276484
276771
|
import { pathToFileURL as pathToFileURL2 } from "url";
|
|
276772
|
+
|
|
276773
|
+
// src/service/authorize.ts
|
|
276774
|
+
init_errors();
|
|
276775
|
+
var AUTHORIZE_TAG = String.raw`##?\(\s*authorize\s*\)`;
|
|
276776
|
+
var AUTHORIZE_ANNOTATION_ANYWHERE = new RegExp(AUTHORIZE_TAG);
|
|
276777
|
+
var AUTHORIZE_ANNOTATION_PREFIX = new RegExp(`^${AUTHORIZE_TAG}`);
|
|
276778
|
+
function assertNoCallerAuthorizeAnnotation(callerText) {
|
|
276779
|
+
if (!AUTHORIZE_ANNOTATION_ANYWHERE.test(callerText))
|
|
276780
|
+
return;
|
|
276781
|
+
throw new BadRequestError("An `authorize` annotation is not permitted in caller-submitted Malloy " + "text. Access gates are declared by the model author on the source; a " + "request cannot introduce, replace, or relax one. To validate a gate " + "you are authoring, save it to the package's model file and reload the " + "package — model load validates every `#(authorize)` annotation it " + "declares.");
|
|
276782
|
+
}
|
|
276783
|
+
function buildAuthorizeProbe(exprs, givenDecls = []) {
|
|
276784
|
+
const selects = exprs.map((expr, i) => `__auth_${i} is (${expr})`).join(`
|
|
276785
|
+
`);
|
|
276786
|
+
const givenBlock = givenDecls.length > 0 ? `given:
|
|
276787
|
+
${givenDecls.map((g) => ` ${g.name} :: ${g.type}`).join(`
|
|
276788
|
+
`)}
|
|
276789
|
+
|
|
276790
|
+
` : "";
|
|
276791
|
+
return `${givenBlock}run: duckdb.sql("SELECT 1 AS __authorize_probe_row") -> {
|
|
276792
|
+
select:
|
|
276793
|
+
${selects}
|
|
276794
|
+
limit: 1
|
|
276795
|
+
}`;
|
|
276796
|
+
}
|
|
276797
|
+
var GIVEN_REF_PATTERN = /\$([A-Za-z_][A-Za-z0-9_]*)/g;
|
|
276798
|
+
var STRING_LITERAL_PATTERN = /'(?:\\.|[^'\\])*'/g;
|
|
276799
|
+
function referencedGivenNames(expr) {
|
|
276800
|
+
const scanned = expr.replace(STRING_LITERAL_PATTERN, "''");
|
|
276801
|
+
const names = [];
|
|
276802
|
+
const seen = new Set;
|
|
276803
|
+
for (const match of scanned.matchAll(GIVEN_REF_PATTERN)) {
|
|
276804
|
+
const name = match[1];
|
|
276805
|
+
if (!seen.has(name)) {
|
|
276806
|
+
seen.add(name);
|
|
276807
|
+
names.push(name);
|
|
276808
|
+
}
|
|
276809
|
+
}
|
|
276810
|
+
return names;
|
|
276811
|
+
}
|
|
276812
|
+
function inferGivenType(value) {
|
|
276813
|
+
if (typeof value === "string")
|
|
276814
|
+
return "string";
|
|
276815
|
+
if (typeof value === "number" || typeof value === "bigint")
|
|
276816
|
+
return "number";
|
|
276817
|
+
if (typeof value === "boolean")
|
|
276818
|
+
return "boolean";
|
|
276819
|
+
if (value instanceof Date)
|
|
276820
|
+
return "timestamp";
|
|
276821
|
+
if (Array.isArray(value)) {
|
|
276822
|
+
if (value.length === 0)
|
|
276823
|
+
return null;
|
|
276824
|
+
const elementType = inferGivenType(value[0]);
|
|
276825
|
+
return elementType ? `${elementType}[]` : null;
|
|
276826
|
+
}
|
|
276827
|
+
return null;
|
|
276828
|
+
}
|
|
276829
|
+
function bindProbeGivens(expr, givens, declaredTypes) {
|
|
276830
|
+
const decls = [];
|
|
276831
|
+
const bound = {};
|
|
276832
|
+
for (const name of referencedGivenNames(expr)) {
|
|
276833
|
+
if (!(name in givens))
|
|
276834
|
+
continue;
|
|
276835
|
+
const value = givens[name];
|
|
276836
|
+
const type = declaredTypes?.get(name) ?? inferGivenType(value);
|
|
276837
|
+
if (!type)
|
|
276838
|
+
continue;
|
|
276839
|
+
decls.push({ name, type });
|
|
276840
|
+
bound[name] = value;
|
|
276841
|
+
}
|
|
276842
|
+
return { decls, bound };
|
|
276843
|
+
}
|
|
276844
|
+
function isProbeTrue(cell) {
|
|
276845
|
+
return cell === true || cell === 1 || cell === "true";
|
|
276846
|
+
}
|
|
276847
|
+
async function runProbe(executor, probeText, givens) {
|
|
276848
|
+
const result = await executor.loadQuery(probeText).run({ rowLimit: 1, givens });
|
|
276849
|
+
const row = result?.data?.value?.[0];
|
|
276850
|
+
return !!(row && isProbeTrue(row.__auth_0));
|
|
276851
|
+
}
|
|
276852
|
+
async function evaluateAuthorize(executor, exprs, givens, declaredTypes, options) {
|
|
276853
|
+
const selfContainedFirst = options?.selfContainedFirst ?? false;
|
|
276854
|
+
const ambientPrefix = options?.ambientPrefix ?? 0;
|
|
276855
|
+
for (const [index, expr] of exprs.entries()) {
|
|
276856
|
+
if (selfContainedFirst && index >= ambientPrefix) {
|
|
276857
|
+
if (await evaluateSelfContainedFirst(executor, expr, givens, declaredTypes)) {
|
|
276858
|
+
return true;
|
|
276859
|
+
}
|
|
276860
|
+
continue;
|
|
276861
|
+
}
|
|
276862
|
+
try {
|
|
276863
|
+
if (await runProbe(executor, buildAuthorizeProbe([expr]), givens)) {
|
|
276864
|
+
return true;
|
|
276865
|
+
}
|
|
276866
|
+
continue;
|
|
276867
|
+
} catch {}
|
|
276868
|
+
try {
|
|
276869
|
+
const { decls, bound } = bindProbeGivens(expr, givens, declaredTypes);
|
|
276870
|
+
if (decls.length === 0)
|
|
276871
|
+
continue;
|
|
276872
|
+
if (await runProbe(executor, buildAuthorizeProbe([expr], decls), bound)) {
|
|
276873
|
+
return true;
|
|
276874
|
+
}
|
|
276875
|
+
} catch {
|
|
276876
|
+
continue;
|
|
276877
|
+
}
|
|
276878
|
+
}
|
|
276879
|
+
return false;
|
|
276880
|
+
}
|
|
276881
|
+
async function evaluateSelfContainedFirst(executor, expr, givens, declaredTypes) {
|
|
276882
|
+
const referenced = referencedGivenNames(expr);
|
|
276883
|
+
if (referenced.length === 0) {
|
|
276884
|
+
try {
|
|
276885
|
+
return await runProbe(executor, buildAuthorizeProbe([expr]), {});
|
|
276886
|
+
} catch {
|
|
276887
|
+
return false;
|
|
276888
|
+
}
|
|
276889
|
+
}
|
|
276890
|
+
const { decls, bound } = bindProbeGivens(expr, givens, declaredTypes);
|
|
276891
|
+
if (decls.length !== referenced.length) {
|
|
276892
|
+
return false;
|
|
276893
|
+
}
|
|
276894
|
+
try {
|
|
276895
|
+
return await runProbe(executor, buildAuthorizeProbe([expr], decls), bound);
|
|
276896
|
+
} catch {
|
|
276897
|
+
try {
|
|
276898
|
+
return await runProbe(executor, buildAuthorizeProbe([expr]), givens);
|
|
276899
|
+
} catch {
|
|
276900
|
+
return false;
|
|
276901
|
+
}
|
|
276902
|
+
}
|
|
276903
|
+
}
|
|
276904
|
+
async function validateAuthorizeProbes(compiler, sources) {
|
|
276905
|
+
for (const source of sources) {
|
|
276906
|
+
const exprs = source.authorize;
|
|
276907
|
+
if (!exprs || exprs.length === 0)
|
|
276908
|
+
continue;
|
|
276909
|
+
try {
|
|
276910
|
+
await compiler.loadQuery(buildAuthorizeProbe(exprs)).getPreparedQuery();
|
|
276911
|
+
} catch (err) {
|
|
276912
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
276913
|
+
throw new ModelCompilationError({
|
|
276914
|
+
message: `Invalid #(authorize) annotation on source "${source.name ?? "(unnamed)"}" [${exprs.join(" | ")}]: ${detail}`
|
|
276915
|
+
});
|
|
276916
|
+
}
|
|
276917
|
+
}
|
|
276918
|
+
}
|
|
276919
|
+
function parseAuthorizeAnnotation(annotation) {
|
|
276920
|
+
const trimmed2 = annotation.trim();
|
|
276921
|
+
const prefix = AUTHORIZE_ANNOTATION_PREFIX.exec(trimmed2);
|
|
276922
|
+
if (!prefix)
|
|
276923
|
+
return null;
|
|
276924
|
+
return unwrapQuotedExpression(trimmed2.slice(prefix[0].length).trim());
|
|
276925
|
+
}
|
|
276926
|
+
function collectAuthorizeExprs(annotations) {
|
|
276927
|
+
const exprs = [];
|
|
276928
|
+
for (const annotation of annotations) {
|
|
276929
|
+
const expr = parseAuthorizeAnnotation(annotation);
|
|
276930
|
+
if (expr !== null) {
|
|
276931
|
+
exprs.push(expr);
|
|
276932
|
+
}
|
|
276933
|
+
}
|
|
276934
|
+
return exprs;
|
|
276935
|
+
}
|
|
276936
|
+
function unwrapQuotedExpression(body) {
|
|
276937
|
+
if (body.length < 2 || body[0] !== '"') {
|
|
276938
|
+
throw new Error(`authorize annotation expression must be a double-quoted string, got: ${body || "(empty)"}`);
|
|
276939
|
+
}
|
|
276940
|
+
let expr = "";
|
|
276941
|
+
let i = 1;
|
|
276942
|
+
let closed = false;
|
|
276943
|
+
for (;i < body.length; i++) {
|
|
276944
|
+
const ch = body[i];
|
|
276945
|
+
if (ch === "\\" && i + 1 < body.length) {
|
|
276946
|
+
const next = body[i + 1];
|
|
276947
|
+
if (next === '"' || next === "\\") {
|
|
276948
|
+
expr += next;
|
|
276949
|
+
i++;
|
|
276950
|
+
continue;
|
|
276951
|
+
}
|
|
276952
|
+
}
|
|
276953
|
+
if (ch === '"') {
|
|
276954
|
+
closed = true;
|
|
276955
|
+
i++;
|
|
276956
|
+
break;
|
|
276957
|
+
}
|
|
276958
|
+
expr += ch;
|
|
276959
|
+
}
|
|
276960
|
+
if (!closed) {
|
|
276961
|
+
throw new Error(`authorize annotation has mismatched quotes: ${body}`);
|
|
276962
|
+
}
|
|
276963
|
+
const rest = body.slice(i).trim();
|
|
276964
|
+
if (rest.length > 0) {
|
|
276965
|
+
throw new Error(`authorize annotation has unexpected content after the expression: ${rest}`);
|
|
276966
|
+
}
|
|
276967
|
+
if (expr.trim().length === 0) {
|
|
276968
|
+
throw new Error("authorize annotation has an empty expression body");
|
|
276969
|
+
}
|
|
276970
|
+
return expr;
|
|
276971
|
+
}
|
|
276972
|
+
|
|
276973
|
+
// src/authorize_metrics.ts
|
|
276974
|
+
var guardRejectionCounter = null;
|
|
276975
|
+
function recordAuthorizeGuardRejection(field) {
|
|
276976
|
+
guardRejectionCounter ??= publisherMeter().createCounter("publisher_authorize_guard_rejected_total", {
|
|
276977
|
+
description: "Requests rejected with 400 for declaring an `#(authorize)` annotation in caller-submitted Malloy text. Label: field ('query'|'source_name'|'query_name'|'compile_source')."
|
|
276978
|
+
});
|
|
276979
|
+
guardRejectionCounter.add(1, { field });
|
|
276980
|
+
}
|
|
276981
|
+
|
|
276982
|
+
// src/service/environment.ts
|
|
276485
276983
|
init_logger();
|
|
276486
276984
|
|
|
276487
276985
|
// src/materialization_metrics.ts
|
|
@@ -276569,7 +277067,7 @@ function recordChainedStorageBuild(outcome) {
|
|
|
276569
277067
|
}
|
|
276570
277068
|
|
|
276571
277069
|
// src/utils.ts
|
|
276572
|
-
import * as
|
|
277070
|
+
import * as fs4 from "fs";
|
|
276573
277071
|
import * as path5 from "path";
|
|
276574
277072
|
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
276575
277073
|
var URL_READER = {
|
|
@@ -276578,7 +277076,7 @@ var URL_READER = {
|
|
|
276578
277076
|
if (url2.protocol == "file:") {
|
|
276579
277077
|
path6 = fileURLToPath3(url2);
|
|
276580
277078
|
}
|
|
276581
|
-
return
|
|
277079
|
+
return fs4.promises.readFile(path6, "utf8");
|
|
276582
277080
|
}
|
|
276583
277081
|
};
|
|
276584
277082
|
function ignoreDotfiles(file) {
|
|
@@ -276592,7 +277090,7 @@ function errMessage(err) {
|
|
|
276592
277090
|
init_logger();
|
|
276593
277091
|
var import_client_s32 = __toESM(require_dist_cjs75(), 1);
|
|
276594
277092
|
import { Storage } from "@google-cloud/storage";
|
|
276595
|
-
import * as
|
|
277093
|
+
import * as fs5 from "fs/promises";
|
|
276596
277094
|
import { fileURLToPath as fileURLToPath4 } from "url";
|
|
276597
277095
|
var gcsClient;
|
|
276598
277096
|
var s3Client;
|
|
@@ -276621,9 +277119,9 @@ async function readManifestBytes(uri) {
|
|
|
276621
277119
|
return res.Body.transformToString();
|
|
276622
277120
|
}
|
|
276623
277121
|
if (uri.startsWith("file://")) {
|
|
276624
|
-
return
|
|
277122
|
+
return fs5.readFile(fileURLToPath4(uri), "utf8");
|
|
276625
277123
|
}
|
|
276626
|
-
return
|
|
277124
|
+
return fs5.readFile(uri, "utf8");
|
|
276627
277125
|
}
|
|
276628
277126
|
async function fetchManifestEntries(uri) {
|
|
276629
277127
|
const raw = await readManifestBytes(uri);
|
|
@@ -276647,7 +277145,7 @@ function splitManifestEntries(entries, source) {
|
|
|
276647
277145
|
});
|
|
276648
277146
|
continue;
|
|
276649
277147
|
}
|
|
276650
|
-
if (entry.
|
|
277148
|
+
if (entry.storageDestinationName) {
|
|
276651
277149
|
storageEntries[sourceEntityId] = entry;
|
|
276652
277150
|
continue;
|
|
276653
277151
|
}
|
|
@@ -276663,7 +277161,7 @@ function splitManifestEntries(entries, source) {
|
|
|
276663
277161
|
}
|
|
276664
277162
|
|
|
276665
277163
|
// src/service/package.ts
|
|
276666
|
-
import * as
|
|
277164
|
+
import * as fs7 from "fs/promises";
|
|
276667
277165
|
import * as path7 from "path";
|
|
276668
277166
|
import"@malloydata/db-duckdb/native";
|
|
276669
277167
|
import { DuckDBConnection as DuckDBConnection3 } from "@malloydata/db-duckdb";
|
|
@@ -276741,7 +277239,7 @@ import {
|
|
|
276741
277239
|
function deriveServeBindings(entries) {
|
|
276742
277240
|
const bindings = [];
|
|
276743
277241
|
for (const entry of Object.values(entries)) {
|
|
276744
|
-
if (!entry.sourceName || !entry.
|
|
277242
|
+
if (!entry.sourceName || !entry.storageDestinationName || !entry.physicalTableName) {
|
|
276745
277243
|
continue;
|
|
276746
277244
|
}
|
|
276747
277245
|
const schema = (entry.schema ?? []).filter((c) => c.name && c.type).map((c) => ({ name: c.name, type: c.type }));
|
|
@@ -276749,9 +277247,9 @@ function deriveServeBindings(entries) {
|
|
|
276749
277247
|
continue;
|
|
276750
277248
|
bindings.push({
|
|
276751
277249
|
sourceName: entry.sourceName,
|
|
276752
|
-
|
|
277250
|
+
destinationName: entry.storageDestinationName,
|
|
276753
277251
|
virtualHandle: entry.sourceEntityId,
|
|
276754
|
-
tablePath: `${entry.
|
|
277252
|
+
tablePath: `${entry.storageDestinationName}.${entry.physicalTableName}`,
|
|
276755
277253
|
schema,
|
|
276756
277254
|
freshAsOf: entry.dataAsOf,
|
|
276757
277255
|
freshnessWindowSeconds: entry.freshnessWindowSeconds,
|
|
@@ -276830,7 +277328,7 @@ function buildServeShapeModel(sourceName, binding) {
|
|
|
276830
277328
|
type: ${shapeTypeName} is {
|
|
276831
277329
|
${fields}
|
|
276832
277330
|
}
|
|
276833
|
-
source: ${sourceName} is ${binding.
|
|
277331
|
+
source: ${sourceName} is ${binding.destinationName}.virtual('${binding.virtualHandle}')::${shapeTypeName}
|
|
276834
277332
|
`;
|
|
276835
277333
|
return { modelText, shapeTypeName };
|
|
276836
277334
|
}
|
|
@@ -276838,7 +277336,7 @@ function serveShapeFragment(binding) {
|
|
|
276838
277336
|
const shapeTypeName = `${binding.sourceName}__shape`;
|
|
276839
277337
|
const fields = binding.schema.map((c) => ` ${emitFieldName(c.name)}::${duckdbTypeToMalloy(c.type)}`).join(`,
|
|
276840
277338
|
`);
|
|
276841
|
-
let source = `source: ${binding.sourceName} is ` + `${binding.
|
|
277339
|
+
let source = `source: ${binding.sourceName} is ` + `${binding.destinationName}.virtual('${binding.virtualHandle}')::${shapeTypeName}`;
|
|
276842
277340
|
const refinements = binding.refinements ?? [];
|
|
276843
277341
|
const lines = [];
|
|
276844
277342
|
for (const r of refinements) {
|
|
@@ -277023,10 +277521,10 @@ function extractViews(fields, liftText) {
|
|
|
277023
277521
|
function buildVirtualMap(bindings) {
|
|
277024
277522
|
const map = new Map;
|
|
277025
277523
|
for (const b of bindings) {
|
|
277026
|
-
let inner = map.get(b.
|
|
277524
|
+
let inner = map.get(b.destinationName);
|
|
277027
277525
|
if (!inner) {
|
|
277028
277526
|
inner = new Map;
|
|
277029
|
-
map.set(b.
|
|
277527
|
+
map.set(b.destinationName, inner);
|
|
277030
277528
|
}
|
|
277031
277529
|
inner.set(b.virtualHandle, quoteManifestTablePath(b.tablePath, "duckdb"));
|
|
277032
277530
|
}
|
|
@@ -277057,206 +277555,6 @@ import { Annotations as Annotations3 } from "@malloydata/malloy";
|
|
|
277057
277555
|
|
|
277058
277556
|
// src/service/materialization_eligibility.ts
|
|
277059
277557
|
init_errors();
|
|
277060
|
-
|
|
277061
|
-
// src/service/authorize.ts
|
|
277062
|
-
init_errors();
|
|
277063
|
-
var SOURCE_PREFIX = "#(authorize)";
|
|
277064
|
-
var FILE_PREFIX = "##(authorize)";
|
|
277065
|
-
function buildAuthorizeProbe(exprs, givenDecls = []) {
|
|
277066
|
-
const selects = exprs.map((expr, i) => `__auth_${i} is (${expr})`).join(`
|
|
277067
|
-
`);
|
|
277068
|
-
const givenBlock = givenDecls.length > 0 ? `given:
|
|
277069
|
-
${givenDecls.map((g) => ` ${g.name} :: ${g.type}`).join(`
|
|
277070
|
-
`)}
|
|
277071
|
-
|
|
277072
|
-
` : "";
|
|
277073
|
-
return `${givenBlock}run: duckdb.sql("SELECT 1 AS __authorize_probe_row") -> {
|
|
277074
|
-
select:
|
|
277075
|
-
${selects}
|
|
277076
|
-
limit: 1
|
|
277077
|
-
}`;
|
|
277078
|
-
}
|
|
277079
|
-
var GIVEN_REF_PATTERN = /\$([A-Za-z_][A-Za-z0-9_]*)/g;
|
|
277080
|
-
var STRING_LITERAL_PATTERN = /'(?:\\.|[^'\\])*'/g;
|
|
277081
|
-
function referencedGivenNames(expr) {
|
|
277082
|
-
const scanned = expr.replace(STRING_LITERAL_PATTERN, "''");
|
|
277083
|
-
const names = [];
|
|
277084
|
-
const seen = new Set;
|
|
277085
|
-
for (const match of scanned.matchAll(GIVEN_REF_PATTERN)) {
|
|
277086
|
-
const name = match[1];
|
|
277087
|
-
if (!seen.has(name)) {
|
|
277088
|
-
seen.add(name);
|
|
277089
|
-
names.push(name);
|
|
277090
|
-
}
|
|
277091
|
-
}
|
|
277092
|
-
return names;
|
|
277093
|
-
}
|
|
277094
|
-
function inferGivenType(value) {
|
|
277095
|
-
if (typeof value === "string")
|
|
277096
|
-
return "string";
|
|
277097
|
-
if (typeof value === "number" || typeof value === "bigint")
|
|
277098
|
-
return "number";
|
|
277099
|
-
if (typeof value === "boolean")
|
|
277100
|
-
return "boolean";
|
|
277101
|
-
if (value instanceof Date)
|
|
277102
|
-
return "timestamp";
|
|
277103
|
-
if (Array.isArray(value)) {
|
|
277104
|
-
if (value.length === 0)
|
|
277105
|
-
return null;
|
|
277106
|
-
const elementType = inferGivenType(value[0]);
|
|
277107
|
-
return elementType ? `${elementType}[]` : null;
|
|
277108
|
-
}
|
|
277109
|
-
return null;
|
|
277110
|
-
}
|
|
277111
|
-
function bindProbeGivens(expr, givens, declaredTypes) {
|
|
277112
|
-
const decls = [];
|
|
277113
|
-
const bound = {};
|
|
277114
|
-
for (const name of referencedGivenNames(expr)) {
|
|
277115
|
-
if (!(name in givens))
|
|
277116
|
-
continue;
|
|
277117
|
-
const value = givens[name];
|
|
277118
|
-
const type = declaredTypes?.get(name) ?? inferGivenType(value);
|
|
277119
|
-
if (!type)
|
|
277120
|
-
continue;
|
|
277121
|
-
decls.push({ name, type });
|
|
277122
|
-
bound[name] = value;
|
|
277123
|
-
}
|
|
277124
|
-
return { decls, bound };
|
|
277125
|
-
}
|
|
277126
|
-
function isProbeTrue(cell) {
|
|
277127
|
-
return cell === true || cell === 1 || cell === "true";
|
|
277128
|
-
}
|
|
277129
|
-
async function runProbe(executor, probeText, givens) {
|
|
277130
|
-
const result = await executor.loadQuery(probeText).run({ rowLimit: 1, givens });
|
|
277131
|
-
const row = result?.data?.value?.[0];
|
|
277132
|
-
return !!(row && isProbeTrue(row.__auth_0));
|
|
277133
|
-
}
|
|
277134
|
-
async function evaluateAuthorize(executor, exprs, givens, declaredTypes, options) {
|
|
277135
|
-
const selfContainedFirst = options?.selfContainedFirst ?? false;
|
|
277136
|
-
for (const expr of exprs) {
|
|
277137
|
-
if (selfContainedFirst) {
|
|
277138
|
-
if (await evaluateSelfContainedFirst(executor, expr, givens, declaredTypes)) {
|
|
277139
|
-
return true;
|
|
277140
|
-
}
|
|
277141
|
-
continue;
|
|
277142
|
-
}
|
|
277143
|
-
try {
|
|
277144
|
-
if (await runProbe(executor, buildAuthorizeProbe([expr]), givens)) {
|
|
277145
|
-
return true;
|
|
277146
|
-
}
|
|
277147
|
-
continue;
|
|
277148
|
-
} catch {}
|
|
277149
|
-
try {
|
|
277150
|
-
const { decls, bound } = bindProbeGivens(expr, givens, declaredTypes);
|
|
277151
|
-
if (decls.length === 0)
|
|
277152
|
-
continue;
|
|
277153
|
-
if (await runProbe(executor, buildAuthorizeProbe([expr], decls), bound)) {
|
|
277154
|
-
return true;
|
|
277155
|
-
}
|
|
277156
|
-
} catch {
|
|
277157
|
-
continue;
|
|
277158
|
-
}
|
|
277159
|
-
}
|
|
277160
|
-
return false;
|
|
277161
|
-
}
|
|
277162
|
-
async function evaluateSelfContainedFirst(executor, expr, givens, declaredTypes) {
|
|
277163
|
-
const referenced = referencedGivenNames(expr);
|
|
277164
|
-
if (referenced.length === 0) {
|
|
277165
|
-
try {
|
|
277166
|
-
return await runProbe(executor, buildAuthorizeProbe([expr]), {});
|
|
277167
|
-
} catch {
|
|
277168
|
-
return false;
|
|
277169
|
-
}
|
|
277170
|
-
}
|
|
277171
|
-
const { decls, bound } = bindProbeGivens(expr, givens, declaredTypes);
|
|
277172
|
-
if (decls.length !== referenced.length) {
|
|
277173
|
-
return false;
|
|
277174
|
-
}
|
|
277175
|
-
try {
|
|
277176
|
-
return await runProbe(executor, buildAuthorizeProbe([expr], decls), bound);
|
|
277177
|
-
} catch {
|
|
277178
|
-
try {
|
|
277179
|
-
return await runProbe(executor, buildAuthorizeProbe([expr]), givens);
|
|
277180
|
-
} catch {
|
|
277181
|
-
return false;
|
|
277182
|
-
}
|
|
277183
|
-
}
|
|
277184
|
-
}
|
|
277185
|
-
async function validateAuthorizeProbes(compiler, sources) {
|
|
277186
|
-
for (const source of sources) {
|
|
277187
|
-
const exprs = source.authorize;
|
|
277188
|
-
if (!exprs || exprs.length === 0)
|
|
277189
|
-
continue;
|
|
277190
|
-
try {
|
|
277191
|
-
await compiler.loadQuery(buildAuthorizeProbe(exprs)).getPreparedQuery();
|
|
277192
|
-
} catch (err) {
|
|
277193
|
-
const detail = err instanceof Error ? err.message : String(err);
|
|
277194
|
-
throw new ModelCompilationError({
|
|
277195
|
-
message: `Invalid #(authorize) annotation on source "${source.name ?? "(unnamed)"}" [${exprs.join(" | ")}]: ${detail}`
|
|
277196
|
-
});
|
|
277197
|
-
}
|
|
277198
|
-
}
|
|
277199
|
-
}
|
|
277200
|
-
function parseAuthorizeAnnotation(annotation) {
|
|
277201
|
-
const trimmed2 = annotation.trim();
|
|
277202
|
-
let body;
|
|
277203
|
-
if (trimmed2.startsWith(FILE_PREFIX)) {
|
|
277204
|
-
body = trimmed2.slice(FILE_PREFIX.length).trim();
|
|
277205
|
-
} else if (trimmed2.startsWith(SOURCE_PREFIX)) {
|
|
277206
|
-
body = trimmed2.slice(SOURCE_PREFIX.length).trim();
|
|
277207
|
-
} else {
|
|
277208
|
-
return null;
|
|
277209
|
-
}
|
|
277210
|
-
return unwrapQuotedExpression(body);
|
|
277211
|
-
}
|
|
277212
|
-
function collectAuthorizeExprs(annotations) {
|
|
277213
|
-
const exprs = [];
|
|
277214
|
-
for (const annotation of annotations) {
|
|
277215
|
-
const expr = parseAuthorizeAnnotation(annotation);
|
|
277216
|
-
if (expr !== null) {
|
|
277217
|
-
exprs.push(expr);
|
|
277218
|
-
}
|
|
277219
|
-
}
|
|
277220
|
-
return exprs;
|
|
277221
|
-
}
|
|
277222
|
-
function unwrapQuotedExpression(body) {
|
|
277223
|
-
if (body.length < 2 || body[0] !== '"') {
|
|
277224
|
-
throw new Error(`authorize annotation expression must be a double-quoted string, got: ${body || "(empty)"}`);
|
|
277225
|
-
}
|
|
277226
|
-
let expr = "";
|
|
277227
|
-
let i = 1;
|
|
277228
|
-
let closed = false;
|
|
277229
|
-
for (;i < body.length; i++) {
|
|
277230
|
-
const ch = body[i];
|
|
277231
|
-
if (ch === "\\" && i + 1 < body.length) {
|
|
277232
|
-
const next = body[i + 1];
|
|
277233
|
-
if (next === '"' || next === "\\") {
|
|
277234
|
-
expr += next;
|
|
277235
|
-
i++;
|
|
277236
|
-
continue;
|
|
277237
|
-
}
|
|
277238
|
-
}
|
|
277239
|
-
if (ch === '"') {
|
|
277240
|
-
closed = true;
|
|
277241
|
-
i++;
|
|
277242
|
-
break;
|
|
277243
|
-
}
|
|
277244
|
-
expr += ch;
|
|
277245
|
-
}
|
|
277246
|
-
if (!closed) {
|
|
277247
|
-
throw new Error(`authorize annotation has mismatched quotes: ${body}`);
|
|
277248
|
-
}
|
|
277249
|
-
const rest = body.slice(i).trim();
|
|
277250
|
-
if (rest.length > 0) {
|
|
277251
|
-
throw new Error(`authorize annotation has unexpected content after the expression: ${rest}`);
|
|
277252
|
-
}
|
|
277253
|
-
if (expr.trim().length === 0) {
|
|
277254
|
-
throw new Error("authorize annotation has an empty expression body");
|
|
277255
|
-
}
|
|
277256
|
-
return expr;
|
|
277257
|
-
}
|
|
277258
|
-
|
|
277259
|
-
// src/service/materialization_eligibility.ts
|
|
277260
277558
|
function assertMaterializationEligible(persistSource) {
|
|
277261
277559
|
const sourceName = persistSource.name;
|
|
277262
277560
|
let unbound;
|
|
@@ -277402,9 +277700,6 @@ import {
|
|
|
277402
277700
|
API,
|
|
277403
277701
|
FixedConnectionMap,
|
|
277404
277702
|
InMemoryURLReader as InMemoryURLReader2,
|
|
277405
|
-
isBasicArray,
|
|
277406
|
-
isJoined,
|
|
277407
|
-
isRepeatedRecord,
|
|
277408
277703
|
isSourceDef as isSourceDef2,
|
|
277409
277704
|
MalloyConfig as MalloyConfig2,
|
|
277410
277705
|
MalloyError as MalloyError2,
|
|
@@ -277415,7 +277710,7 @@ import {
|
|
|
277415
277710
|
MalloySQLParser,
|
|
277416
277711
|
MalloySQLStatementType
|
|
277417
277712
|
} from "@malloydata/malloy-sql";
|
|
277418
|
-
import * as
|
|
277713
|
+
import * as fs6 from "fs/promises";
|
|
277419
277714
|
import { readFileSync } from "fs";
|
|
277420
277715
|
import { createRequire as createRequire2 } from "module";
|
|
277421
277716
|
import * as path6 from "path";
|
|
@@ -277780,12 +278075,23 @@ function assertWithinModelResponseLimits(rowCount, serializedBytes, { maxRows, m
|
|
|
277780
278075
|
throw new PayloadTooLargeError(`Query response exceeded ${maxBytes} bytes (was ${serializedBytes}). Project fewer columns, add a LIMIT, or raise PUBLISHER_MAX_RESPONSE_BYTES.`);
|
|
277781
278076
|
}
|
|
277782
278077
|
}
|
|
278078
|
+
function stringifyQueryResponse(response, rowCount, maxBytes, source) {
|
|
278079
|
+
try {
|
|
278080
|
+
return JSON.stringify(response);
|
|
278081
|
+
} catch (error) {
|
|
278082
|
+
if (!(error instanceof RangeError) || /call stack/i.test(error.message)) {
|
|
278083
|
+
throw error;
|
|
278084
|
+
}
|
|
278085
|
+
recordQueryCapExceeded("unserializable", source);
|
|
278086
|
+
throw new ResponseUnserializableError(`Query response could not be serialized: the ${rowCount}-row result is too large to turn into JSON${maxBytes > 0 ? ` (byte cap: ${maxBytes})` : ""}. Project fewer columns, add a LIMIT, or filter wide values.`);
|
|
278087
|
+
}
|
|
278088
|
+
}
|
|
277783
278089
|
|
|
277784
278090
|
// src/service/query_text.ts
|
|
277785
278091
|
function extractRunTargetSourceName(query) {
|
|
277786
278092
|
if (!query)
|
|
277787
278093
|
return;
|
|
277788
|
-
const runMatch = query.match(/run\s*:\s*(?:`([^`]+)`|(\w+))
|
|
278094
|
+
const runMatch = query.match(/run\s*:\s*(?:`([^`]+)`|(\w+))/);
|
|
277789
278095
|
const arrowMatch = query.match(/^\s*(?:`([^`]+)`|(\w+))\s*->/m);
|
|
277790
278096
|
return runMatch?.[1] ?? runMatch?.[2] ?? arrowMatch?.[1] ?? arrowMatch?.[2];
|
|
277791
278097
|
}
|
|
@@ -277806,6 +278112,7 @@ import {
|
|
|
277806
278112
|
function extractSourcesFromModelDef(modelDef, givens, onParseError) {
|
|
277807
278113
|
const filterMap = new Map;
|
|
277808
278114
|
const authorizeMap = new Map;
|
|
278115
|
+
const ownAuthorizeSources = [];
|
|
277809
278116
|
const fileLevelAuthorize = collectAuthorizeExprs((modelAnnotations(modelDef).notes ?? []).map((note) => note.text));
|
|
277810
278117
|
const sources = Object.values(modelDef.contents).filter((obj) => isSourceDef(obj)).map((sourceObj) => {
|
|
277811
278118
|
const struct = sourceObj;
|
|
@@ -277844,15 +278151,33 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
|
|
|
277844
278151
|
}
|
|
277845
278152
|
}
|
|
277846
278153
|
const ownNotes = (struct.annotations?.blockNotes ?? []).map((note) => note.text);
|
|
278154
|
+
const ownGates = collectAuthorizeExprs(ownNotes);
|
|
278155
|
+
let inheritedGates = [];
|
|
278156
|
+
if (ownGates.length === 0) {
|
|
278157
|
+
for (let cur2 = struct.annotations?.inherits;cur2; cur2 = cur2.inherits) {
|
|
278158
|
+
const exprs = collectAuthorizeExprs((cur2.blockNotes ?? []).map((note) => note.text));
|
|
278159
|
+
if (exprs.length > 0) {
|
|
278160
|
+
inheritedGates = exprs;
|
|
278161
|
+
break;
|
|
278162
|
+
}
|
|
278163
|
+
}
|
|
278164
|
+
}
|
|
277847
278165
|
const effective = [
|
|
277848
278166
|
...fileLevelAuthorize,
|
|
277849
|
-
...
|
|
278167
|
+
...ownGates.length > 0 ? ownGates : inheritedGates
|
|
277850
278168
|
];
|
|
277851
278169
|
let authorize;
|
|
277852
278170
|
if (effective.length > 0) {
|
|
277853
278171
|
authorizeMap.set(sourceName, effective);
|
|
277854
278172
|
authorize = effective;
|
|
277855
278173
|
}
|
|
278174
|
+
const ownEffective = [...fileLevelAuthorize, ...ownGates];
|
|
278175
|
+
if (ownEffective.length > 0) {
|
|
278176
|
+
ownAuthorizeSources.push({
|
|
278177
|
+
name: sourceName,
|
|
278178
|
+
authorize: ownEffective
|
|
278179
|
+
});
|
|
278180
|
+
}
|
|
277856
278181
|
const views = struct.fields.filter((field) => field.type === "turtle").filter((turtle) => turtle.pipeline.map((stage) => stage.type).every((type) => type === "reduce")).map((turtle) => ({
|
|
277857
278182
|
name: turtle.as || turtle.name,
|
|
277858
278183
|
annotations: annotationTexts(turtle.annotations)
|
|
@@ -277866,7 +278191,7 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
|
|
|
277866
278191
|
authorize
|
|
277867
278192
|
};
|
|
277868
278193
|
});
|
|
277869
|
-
return { sources, filterMap, authorizeMap };
|
|
278194
|
+
return { sources, filterMap, authorizeMap, ownAuthorizeSources };
|
|
277870
278195
|
}
|
|
277871
278196
|
function extractQueriesFromModelDef(modelDef) {
|
|
277872
278197
|
const isNamedQuery = (obj) => obj.type === "query";
|
|
@@ -277879,9 +278204,7 @@ function extractQueriesFromModelDef(modelDef) {
|
|
|
277879
278204
|
|
|
277880
278205
|
// src/service/model.ts
|
|
277881
278206
|
var MALLOY_VERSION = createRequire2(import.meta.url)("@malloydata/malloy/package.json").version;
|
|
277882
|
-
|
|
277883
|
-
return field.type === "record" || isBasicArray(field) || isRepeatedRecord(field);
|
|
277884
|
-
}
|
|
278207
|
+
var ANCESTOR_WALK_MAX_DEPTH = 32;
|
|
277885
278208
|
function quoteMalloyIdentifier(name) {
|
|
277886
278209
|
return "`" + (name ?? "").replace(/\\/g, "\\\\").replace(/`/g, "\\`") + "`";
|
|
277887
278210
|
}
|
|
@@ -277894,7 +278217,7 @@ class Model {
|
|
|
277894
278217
|
modelMaterializer;
|
|
277895
278218
|
modelDef;
|
|
277896
278219
|
modelInfo;
|
|
277897
|
-
|
|
278220
|
+
serveDestinationConfig;
|
|
277898
278221
|
serveBindings = [];
|
|
277899
278222
|
serveShapeCache;
|
|
277900
278223
|
sources;
|
|
@@ -277909,6 +278232,7 @@ class Model {
|
|
|
277909
278232
|
discoveryCurationEnabled = false;
|
|
277910
278233
|
queryBoundary = { mode: "all", exploresDeclared: false, isQueryEntryPoint: true };
|
|
277911
278234
|
freshnessResolver;
|
|
278235
|
+
entryPointGatesBySource = new Map;
|
|
277912
278236
|
meter = publisherMeter();
|
|
277913
278237
|
queryExecutionHistogram = this.meter.createHistogram("malloy_model_query_duration", {
|
|
277914
278238
|
description: "How long it takes to execute a Malloy model query",
|
|
@@ -277933,6 +278257,18 @@ class Model {
|
|
|
277933
278257
|
} catch {
|
|
277934
278258
|
this.fileLevelAuthorize = [];
|
|
277935
278259
|
}
|
|
278260
|
+
try {
|
|
278261
|
+
this.entryPointGatesBySource = this.computeEntryPointGatesBySource();
|
|
278262
|
+
} catch {
|
|
278263
|
+
this.entryPointGatesBySource = new Map;
|
|
278264
|
+
}
|
|
278265
|
+
for (const source of this.sources ?? []) {
|
|
278266
|
+
if (!source.name)
|
|
278267
|
+
continue;
|
|
278268
|
+
const exprs = this.entryPointGatesBySource.get(source.name)?.flatMap((g) => g.exprs);
|
|
278269
|
+
if (exprs && exprs.length > 0)
|
|
278270
|
+
source.authorize = exprs;
|
|
278271
|
+
}
|
|
277936
278272
|
try {
|
|
277937
278273
|
this.authorizeReferencedGivenNames = this.computeAuthorizeReferencedGivenNames();
|
|
277938
278274
|
} catch {
|
|
@@ -277977,17 +278313,24 @@ class Model {
|
|
|
277977
278313
|
}
|
|
277978
278314
|
};
|
|
277979
278315
|
addExprs(this.fileLevelAuthorize);
|
|
278316
|
+
for (const gates of this.entryPointGatesBySource.values()) {
|
|
278317
|
+
for (const { exprs } of gates)
|
|
278318
|
+
addExprs(exprs);
|
|
278319
|
+
}
|
|
278320
|
+
return names;
|
|
278321
|
+
}
|
|
278322
|
+
computeEntryPointGatesBySource() {
|
|
278323
|
+
const byName = new Map;
|
|
277980
278324
|
const modelDef = this.modelDef;
|
|
277981
278325
|
if (!modelDef)
|
|
277982
|
-
return
|
|
278326
|
+
return byName;
|
|
277983
278327
|
for (const entry of Object.values(modelDef.contents)) {
|
|
277984
278328
|
if (!isSourceDef2(entry))
|
|
277985
278329
|
continue;
|
|
277986
|
-
|
|
277987
|
-
|
|
277988
|
-
}
|
|
278330
|
+
const name = entry.as ?? entry.name;
|
|
278331
|
+
byName.set(name, this.collectEntryPointGates(entry, modelDef, new Set, true));
|
|
277989
278332
|
}
|
|
277990
|
-
return
|
|
278333
|
+
return byName;
|
|
277991
278334
|
}
|
|
277992
278335
|
hasAuthorize() {
|
|
277993
278336
|
return this.fileLevelAuthorize.length > 0 || (this.sources?.some((s) => (s.authorize?.length ?? 0) > 0) ?? false);
|
|
@@ -277999,9 +278342,16 @@ class Model {
|
|
|
277999
278342
|
return this.fileLevelAuthorize;
|
|
278000
278343
|
}
|
|
278001
278344
|
async assertAuthorized(sourceName, givens) {
|
|
278345
|
+
const gates = sourceName ? this.entryPointGatesBySource.get(sourceName) : undefined;
|
|
278346
|
+
if (gates) {
|
|
278347
|
+
for (const { label, exprs, selfContained, ambientPrefix } of gates) {
|
|
278348
|
+
await this.assertAuthorizedExprs(label, exprs, givens, selfContained, ambientPrefix);
|
|
278349
|
+
}
|
|
278350
|
+
return;
|
|
278351
|
+
}
|
|
278002
278352
|
await this.assertAuthorizedExprs(sourceName ?? "(query)", this.effectiveAuthorizeFor(sourceName), givens);
|
|
278003
278353
|
}
|
|
278004
|
-
async assertAuthorizedExprs(label, exprs, givens, selfContainedFirst = false) {
|
|
278354
|
+
async assertAuthorizedExprs(label, exprs, givens, selfContainedFirst = false, ambientPrefix = 0) {
|
|
278005
278355
|
if (exprs.length === 0)
|
|
278006
278356
|
return;
|
|
278007
278357
|
const deny = () => {
|
|
@@ -278011,7 +278361,7 @@ class Model {
|
|
|
278011
278361
|
deny();
|
|
278012
278362
|
let passed = false;
|
|
278013
278363
|
try {
|
|
278014
|
-
passed = await evaluateAuthorize(this.modelMaterializer, exprs, givens, this.givenDeclaredTypes(), { selfContainedFirst });
|
|
278364
|
+
passed = await evaluateAuthorize(this.modelMaterializer, exprs, givens, this.givenDeclaredTypes(), { selfContainedFirst, ambientPrefix });
|
|
278015
278365
|
} catch (err) {
|
|
278016
278366
|
logger.debug("Authorize probe failed; denying", {
|
|
278017
278367
|
sourceName: label,
|
|
@@ -278026,27 +278376,19 @@ class Model {
|
|
|
278026
278376
|
async assertAuthorizedForAllSources(runnable, givens) {
|
|
278027
278377
|
const ownSourceName = await this.resolveAuthorizeSourceFromRunnable(runnable);
|
|
278028
278378
|
await this.assertAuthorized(ownSourceName, givens);
|
|
278029
|
-
const { struct, modelDef, compositeResolvedSourceDef
|
|
278379
|
+
const { struct, modelDef, compositeResolvedSourceDef } = await this.resolveRunTargetStruct(runnable);
|
|
278030
278380
|
const seen = new Set;
|
|
278031
|
-
const
|
|
278032
|
-
if (modelDef) {
|
|
278033
|
-
const runTargetLabel = ownSourceName ?? "(run target)";
|
|
278034
|
-
for (const field of extendSources) {
|
|
278035
|
-
const { resolved, denyGate } = this.classifyJoinedField(field, runTargetLabel);
|
|
278036
|
-
if (denyGate) {
|
|
278037
|
-
joinedGates.push(denyGate);
|
|
278038
|
-
continue;
|
|
278039
|
-
}
|
|
278040
|
-
if (!resolved)
|
|
278041
|
-
continue;
|
|
278042
|
-
joinedGates.push(...this.collectAllReachableGates(resolved, modelDef, seen));
|
|
278043
|
-
}
|
|
278044
|
-
}
|
|
278381
|
+
const entryPointGates = this.collectEntryPointGates(struct, modelDef, seen, true);
|
|
278045
278382
|
if (compositeResolvedSourceDef && modelDef) {
|
|
278046
|
-
|
|
278383
|
+
entryPointGates.push(...this.collectEntryPointGates(compositeResolvedSourceDef, modelDef, seen, true));
|
|
278047
278384
|
}
|
|
278048
|
-
for (const {
|
|
278049
|
-
|
|
278385
|
+
for (const {
|
|
278386
|
+
label,
|
|
278387
|
+
exprs,
|
|
278388
|
+
selfContained,
|
|
278389
|
+
ambientPrefix
|
|
278390
|
+
} of entryPointGates) {
|
|
278391
|
+
await this.assertAuthorizedExprs(label, exprs, givens, selfContained, ambientPrefix);
|
|
278050
278392
|
}
|
|
278051
278393
|
}
|
|
278052
278394
|
async resolveRunTargetStruct(runnable) {
|
|
@@ -278057,112 +278399,120 @@ class Model {
|
|
|
278057
278399
|
return {
|
|
278058
278400
|
struct: undefined,
|
|
278059
278401
|
modelDef: undefined,
|
|
278060
|
-
compositeResolvedSourceDef: undefined
|
|
278061
|
-
extendSources: []
|
|
278402
|
+
compositeResolvedSourceDef: undefined
|
|
278062
278403
|
};
|
|
278063
278404
|
const structRef = prepared._query?.structRef;
|
|
278064
278405
|
const struct = typeof structRef === "string" ? modelDef.contents[structRef] : structRef;
|
|
278065
|
-
const extendSources = (prepared._query?.pipeline ?? []).flatMap((segment) => segment.extendSource ?? []);
|
|
278066
278406
|
return {
|
|
278067
278407
|
struct: struct && typeof struct === "object" ? struct : undefined,
|
|
278068
278408
|
modelDef,
|
|
278069
|
-
compositeResolvedSourceDef: prepared._query?.compositeResolvedSourceDef
|
|
278070
|
-
extendSources
|
|
278409
|
+
compositeResolvedSourceDef: prepared._query?.compositeResolvedSourceDef
|
|
278071
278410
|
};
|
|
278072
278411
|
} catch {
|
|
278073
278412
|
return {
|
|
278074
278413
|
struct: undefined,
|
|
278075
278414
|
modelDef: undefined,
|
|
278076
|
-
compositeResolvedSourceDef: undefined
|
|
278077
|
-
extendSources: []
|
|
278415
|
+
compositeResolvedSourceDef: undefined
|
|
278078
278416
|
};
|
|
278079
278417
|
}
|
|
278080
278418
|
}
|
|
278081
|
-
gateExprsForOwnAnnotations(struct) {
|
|
278419
|
+
gateExprsForOwnAnnotations(struct, modelDef) {
|
|
278082
278420
|
const ownNotes = (struct.annotations?.blockNotes ?? []).map((note) => note.text);
|
|
278083
278421
|
try {
|
|
278084
|
-
|
|
278085
|
-
|
|
278086
|
-
|
|
278087
|
-
|
|
278422
|
+
const own = collectAuthorizeExprs(ownNotes);
|
|
278423
|
+
if (own.length > 0) {
|
|
278424
|
+
return {
|
|
278425
|
+
exprs: [...this.fileLevelAuthorize, ...own],
|
|
278426
|
+
fromAncestor: false,
|
|
278427
|
+
ambientPrefix: this.fileLevelAuthorize.length
|
|
278428
|
+
};
|
|
278429
|
+
}
|
|
278430
|
+
const ancestor = this.ancestorGateExprs(struct, modelDef);
|
|
278431
|
+
return {
|
|
278432
|
+
exprs: [...this.fileLevelAuthorize, ...ancestor],
|
|
278433
|
+
fromAncestor: ancestor.length > 0,
|
|
278434
|
+
ambientPrefix: this.fileLevelAuthorize.length
|
|
278435
|
+
};
|
|
278088
278436
|
} catch {
|
|
278089
|
-
return ["false"];
|
|
278437
|
+
return { exprs: ["false"], fromAncestor: false, ambientPrefix: 0 };
|
|
278090
278438
|
}
|
|
278091
278439
|
}
|
|
278092
|
-
|
|
278093
|
-
|
|
278094
|
-
|
|
278095
|
-
|
|
278096
|
-
|
|
278097
|
-
|
|
278098
|
-
|
|
278099
|
-
|
|
278100
|
-
|
|
278101
|
-
|
|
278102
|
-
|
|
278103
|
-
|
|
278104
|
-
return
|
|
278105
|
-
|
|
278106
|
-
|
|
278107
|
-
|
|
278108
|
-
|
|
278109
|
-
|
|
278110
|
-
|
|
278440
|
+
ancestorGateExprs(struct, modelDef, seen = new Set) {
|
|
278441
|
+
let inherited = struct.annotations?.inherits;
|
|
278442
|
+
for (let depth = 0;inherited && depth < ANCESTOR_WALK_MAX_DEPTH; depth++) {
|
|
278443
|
+
const exprs2 = collectAuthorizeExprs((inherited.blockNotes ?? []).map((note) => note.text));
|
|
278444
|
+
if (exprs2.length > 0)
|
|
278445
|
+
return exprs2;
|
|
278446
|
+
inherited = inherited.inherits;
|
|
278447
|
+
}
|
|
278448
|
+
if (inherited)
|
|
278449
|
+
return ["false"];
|
|
278450
|
+
seen.add(struct);
|
|
278451
|
+
if (seen.size > ANCESTOR_WALK_MAX_DEPTH)
|
|
278452
|
+
return ["false"];
|
|
278453
|
+
const declared = this.resolveDeclaredSource(struct, modelDef);
|
|
278454
|
+
if (declared.kind === "unresolvable")
|
|
278455
|
+
return ["false"];
|
|
278456
|
+
if (declared.kind === "none" || seen.has(declared.source))
|
|
278457
|
+
return [];
|
|
278458
|
+
const exprs = collectAuthorizeExprs((declared.source.annotations?.blockNotes ?? []).map((note) => note.text));
|
|
278459
|
+
return exprs.length > 0 ? exprs : this.ancestorGateExprs(declared.source, modelDef, seen);
|
|
278460
|
+
}
|
|
278461
|
+
resolveDeclaredSource(struct, modelDef) {
|
|
278462
|
+
if (!modelDef)
|
|
278463
|
+
return { kind: "none" };
|
|
278464
|
+
let sawBrokenEntry = false;
|
|
278465
|
+
for (const id of [struct.referenceID, struct.sourceID]) {
|
|
278466
|
+
const entry = id ? modelDef.sourceRegistry?.[id]?.entry : undefined;
|
|
278467
|
+
if (!entry)
|
|
278468
|
+
continue;
|
|
278469
|
+
const declared = entry.type === "source_registry_reference" ? modelDef.contents[entry.name] : entry;
|
|
278470
|
+
if (declared === struct)
|
|
278471
|
+
continue;
|
|
278472
|
+
if (!declared || !isSourceDef2(declared)) {
|
|
278473
|
+
sawBrokenEntry = true;
|
|
278474
|
+
continue;
|
|
278475
|
+
}
|
|
278476
|
+
return { kind: "resolved", source: declared };
|
|
278111
278477
|
}
|
|
278112
|
-
return {
|
|
278478
|
+
return sawBrokenEntry ? { kind: "unresolvable" } : { kind: "none" };
|
|
278113
278479
|
}
|
|
278114
|
-
|
|
278480
|
+
collectEntryPointGates(struct, modelDef, seen = new Set, treatAsOwnGate = false) {
|
|
278115
278481
|
if (!struct || !modelDef || seen.has(struct))
|
|
278116
278482
|
return [];
|
|
278117
278483
|
seen.add(struct);
|
|
278118
278484
|
const results = [];
|
|
278119
278485
|
const label = struct.as ?? struct.name;
|
|
278120
|
-
const
|
|
278486
|
+
const {
|
|
278487
|
+
exprs: ownExprs,
|
|
278488
|
+
fromAncestor,
|
|
278489
|
+
ambientPrefix
|
|
278490
|
+
} = this.gateExprsForOwnAnnotations(struct, modelDef);
|
|
278121
278491
|
if (ownExprs.length > 0) {
|
|
278122
278492
|
results.push({
|
|
278123
278493
|
label,
|
|
278124
278494
|
exprs: ownExprs,
|
|
278125
|
-
selfContained: !treatAsOwnGate
|
|
278495
|
+
selfContained: fromAncestor || !treatAsOwnGate,
|
|
278496
|
+
ambientPrefix
|
|
278126
278497
|
});
|
|
278127
278498
|
}
|
|
278128
|
-
for (const field of struct.fields) {
|
|
278129
|
-
const { resolved, denyGate } = this.classifyJoinedField(field, label);
|
|
278130
|
-
if (denyGate) {
|
|
278131
|
-
results.push(denyGate);
|
|
278132
|
-
continue;
|
|
278133
|
-
}
|
|
278134
|
-
if (!resolved)
|
|
278135
|
-
continue;
|
|
278136
|
-
const joinedSource = resolved;
|
|
278137
|
-
results.push(...this.collectAllReachableGates(joinedSource, modelDef, seen));
|
|
278138
|
-
if (joinedSource.type === "composite") {
|
|
278139
|
-
const members = joinedSource.sources;
|
|
278140
|
-
for (const member of members) {
|
|
278141
|
-
results.push(...this.collectAllReachableGates(member, modelDef, seen));
|
|
278142
|
-
}
|
|
278143
|
-
}
|
|
278144
|
-
}
|
|
278145
278499
|
const duck = struct;
|
|
278146
278500
|
if (duck.type === "query_source") {
|
|
278147
278501
|
const ref = duck.query?.structRef;
|
|
278148
278502
|
const base = typeof ref === "string" ? modelDef.contents[ref] : ref;
|
|
278149
278503
|
if (base && isSourceDef2(base)) {
|
|
278150
|
-
results.push(...this.
|
|
278504
|
+
results.push(...this.collectEntryPointGates(base, modelDef, seen));
|
|
278505
|
+
} else {
|
|
278506
|
+
results.push({
|
|
278507
|
+
label,
|
|
278508
|
+
exprs: ["false"],
|
|
278509
|
+
selfContained: true,
|
|
278510
|
+
ambientPrefix: 0
|
|
278511
|
+
});
|
|
278151
278512
|
}
|
|
278152
278513
|
const resolved = duck.query?.compositeResolvedSourceDef;
|
|
278153
278514
|
if (resolved) {
|
|
278154
|
-
results.push(...this.
|
|
278155
|
-
}
|
|
278156
|
-
const innerJoins = (duck.query?.pipeline ?? []).flatMap((segment) => segment.extendSource ?? []);
|
|
278157
|
-
for (const field of innerJoins) {
|
|
278158
|
-
const { resolved: innerJoinSource, denyGate: innerJoinDenyGate } = this.classifyJoinedField(field, label);
|
|
278159
|
-
if (innerJoinDenyGate) {
|
|
278160
|
-
results.push(innerJoinDenyGate);
|
|
278161
|
-
continue;
|
|
278162
|
-
}
|
|
278163
|
-
if (!innerJoinSource)
|
|
278164
|
-
continue;
|
|
278165
|
-
results.push(...this.collectAllReachableGates(innerJoinSource, modelDef, seen));
|
|
278515
|
+
results.push(...this.collectEntryPointGates(resolved, modelDef, seen));
|
|
278166
278516
|
}
|
|
278167
278517
|
}
|
|
278168
278518
|
return results;
|
|
@@ -278220,7 +278570,7 @@ class Model {
|
|
|
278220
278570
|
sources = sourceResult.sources;
|
|
278221
278571
|
filterMap = sourceResult.filterMap;
|
|
278222
278572
|
queries = Model.getQueries(modelDef);
|
|
278223
|
-
await validateAuthorizeProbes(modelMaterializer,
|
|
278573
|
+
await validateAuthorizeProbes(modelMaterializer, sourceResult.ownAuthorizeSources);
|
|
278224
278574
|
const imports = modelDef.imports || [];
|
|
278225
278575
|
const importedSourceNames = new Set;
|
|
278226
278576
|
for (const importLocation of imports) {
|
|
@@ -278282,7 +278632,6 @@ class Model {
|
|
|
278282
278632
|
const modelMaterializer = runtime._loadModelFromModelDef(modelDef);
|
|
278283
278633
|
const runnableNotebookCells = data.modelType === "notebook" ? hydrateNotebookCells(runtime, data.notebookCells) : undefined;
|
|
278284
278634
|
const model = new Model(packageName, data.modelPath, dataStyles, data.modelType, modelMaterializer, modelDef, sources, queries, sourceInfos, runnableNotebookCells, undefined, filterMap, givens, modelInfo);
|
|
278285
|
-
model.setServeMalloyConfig(malloyConfig);
|
|
278286
278635
|
return model;
|
|
278287
278636
|
}
|
|
278288
278637
|
static fromCompilationError(packageName, modelPath, modelType, error) {
|
|
@@ -278481,8 +278830,12 @@ class Model {
|
|
|
278481
278830
|
throw new ModelNotFoundError(`${this.modelPath} is not a valid notebook name. Notebook files must end in .malloynb.`);
|
|
278482
278831
|
}
|
|
278483
278832
|
}
|
|
278484
|
-
|
|
278485
|
-
this.
|
|
278833
|
+
setServeDestinationConfig(provider) {
|
|
278834
|
+
this.serveDestinationConfig = provider;
|
|
278835
|
+
this.invalidateServeShapeCache();
|
|
278836
|
+
}
|
|
278837
|
+
invalidateServeShapeCache() {
|
|
278838
|
+
this.serveShapeCache = undefined;
|
|
278486
278839
|
}
|
|
278487
278840
|
setServeBindings(bindings) {
|
|
278488
278841
|
this.serveBindings = bindings;
|
|
@@ -278502,7 +278855,7 @@ class Model {
|
|
|
278502
278855
|
if (freshBindings.length === 0) {
|
|
278503
278856
|
throw new Error("no fresh storage serve bindings for this query");
|
|
278504
278857
|
}
|
|
278505
|
-
const key = freshBindings.map((b) => `${b.sourceName}@${b.
|
|
278858
|
+
const key = freshBindings.map((b) => `${b.sourceName}@${b.destinationName}/${b.virtualHandle}`).sort().join("|");
|
|
278506
278859
|
if (!this.serveShapeCache || this.serveShapeCache.key !== key) {
|
|
278507
278860
|
this.serveShapeCache = {
|
|
278508
278861
|
key,
|
|
@@ -278553,7 +278906,7 @@ class Model {
|
|
|
278553
278906
|
const url2 = `${root}shape.malloy`;
|
|
278554
278907
|
const runtime = new Runtime2({
|
|
278555
278908
|
urlReader: new InMemoryURLReader2(new Map([[url2, modelText]])),
|
|
278556
|
-
config:
|
|
278909
|
+
config: restrictMalloyConfigToConnections(this.serveDestinationConfig(), new Set(bindings.map((binding) => binding.destinationName)))
|
|
278557
278910
|
});
|
|
278558
278911
|
return runtime.loadModel(new URL(url2), {
|
|
278559
278912
|
importBaseURL: new URL(root)
|
|
@@ -278613,18 +278966,33 @@ class Model {
|
|
|
278613
278966
|
if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
|
|
278614
278967
|
throw new BadRequestError("Model has no queryable entities.");
|
|
278615
278968
|
const boundary = this.assertQueryBoundaryEarly(sourceName, queryName, query);
|
|
278616
|
-
const
|
|
278969
|
+
const surfaceName = extractRunTargetSourceName(query);
|
|
278970
|
+
const earlySource = sourceName || (queryName ? this.queries?.find((q) => q.name === queryName)?.sourceName : undefined) || (surfaceName && !this.sources?.some((s) => s.name === surfaceName) ? this.queries?.find((q) => q.name === surfaceName)?.sourceName : undefined) || surfaceName;
|
|
278617
278971
|
if (earlySource) {
|
|
278618
278972
|
await this.assertAuthorized(earlySource, givens ?? {});
|
|
278619
278973
|
}
|
|
278620
278974
|
try {
|
|
278975
|
+
for (const [field, callerText] of [
|
|
278976
|
+
["query", query],
|
|
278977
|
+
["source_name", sourceName],
|
|
278978
|
+
["query_name", queryName]
|
|
278979
|
+
]) {
|
|
278980
|
+
if (!callerText)
|
|
278981
|
+
continue;
|
|
278982
|
+
try {
|
|
278983
|
+
assertNoCallerAuthorizeAnnotation(callerText);
|
|
278984
|
+
} catch (err) {
|
|
278985
|
+
recordAuthorizeGuardRejection(field);
|
|
278986
|
+
throw err;
|
|
278987
|
+
}
|
|
278988
|
+
}
|
|
278621
278989
|
let queryString;
|
|
278622
278990
|
if (!sourceName && !queryName && query) {
|
|
278623
278991
|
queryString = `
|
|
278624
278992
|
` + query;
|
|
278625
278993
|
} else if (queryName && !query) {
|
|
278626
278994
|
queryString = `
|
|
278627
|
-
run: ${sourceName ? sourceName
|
|
278995
|
+
run: ${sourceName ? `${quoteMalloyIdentifier(sourceName)} -> ` : ""}${quoteMalloyIdentifier(queryName)}`;
|
|
278628
278996
|
} else {
|
|
278629
278997
|
const endTime = performance.now();
|
|
278630
278998
|
const executionTime2 = endTime - startTime;
|
|
@@ -278650,7 +279018,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
278650
279018
|
}
|
|
278651
279019
|
runnable = this.modelMaterializer.loadRestrictedQuery(queryString);
|
|
278652
279020
|
liveRunnable = runnable;
|
|
278653
|
-
if (getPersistStorageMode() === "on" && this.serveBindings.length > 0 && this.
|
|
279021
|
+
if (getPersistStorageMode() === "on" && this.serveBindings.length > 0 && this.serveDestinationConfig) {
|
|
278654
279022
|
try {
|
|
278655
279023
|
const shaped = await this.loadServeShapeQuery(queryString);
|
|
278656
279024
|
runnable = shaped.runnable;
|
|
@@ -278800,7 +279168,8 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
278800
279168
|
executionTime = performance.now() - startTime;
|
|
278801
279169
|
}
|
|
278802
279170
|
const wrappedResult = API.util.wrapResult(queryResults);
|
|
278803
|
-
const
|
|
279171
|
+
const serializedResult = stringifyQueryResponse(wrappedResult, queryResults.totalRows, maxBytes, "model_query");
|
|
279172
|
+
const serializedBytes = maxBytes > 0 ? Buffer.byteLength(serializedResult, "utf8") : 0;
|
|
278804
279173
|
assertWithinModelResponseLimits(queryResults.totalRows, serializedBytes, { maxRows, maxBytes }, "model_query");
|
|
278805
279174
|
this.queryExecutionHistogram.record(executionTime, {
|
|
278806
279175
|
"malloy.model.path": this.modelPath,
|
|
@@ -278815,6 +279184,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
278815
279184
|
});
|
|
278816
279185
|
return {
|
|
278817
279186
|
result: wrappedResult,
|
|
279187
|
+
serializedResult,
|
|
278818
279188
|
compactResult: queryResults.data.value,
|
|
278819
279189
|
modelInfo: this.modelInfo,
|
|
278820
279190
|
dataStyles: this.dataStyles,
|
|
@@ -278952,7 +279322,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
278952
279322
|
});
|
|
278953
279323
|
const query = (await runnableToExecute.getPreparedQuery())._query;
|
|
278954
279324
|
queryName = query.as || query.name;
|
|
278955
|
-
queryResult = result?._queryResult && this.modelInfo &&
|
|
279325
|
+
queryResult = result?._queryResult && this.modelInfo && stringifyQueryResponse(API.util.wrapResult(result), result.totalRows, cellMaxBytes, "notebook_cell");
|
|
278956
279326
|
if (result?._queryResult && queryResult) {
|
|
278957
279327
|
assertWithinModelResponseLimits(result.totalRows, Buffer.byteLength(queryResult, "utf8"), { maxRows: cellMaxRows, maxBytes: cellMaxBytes }, "notebook_cell");
|
|
278958
279328
|
}
|
|
@@ -278998,7 +279368,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
278998
279368
|
static async getModelRuntime(packagePath, modelPath, malloyConfig, options) {
|
|
278999
279369
|
const fullModelPath = path6.join(packagePath, modelPath);
|
|
279000
279370
|
try {
|
|
279001
|
-
if (!(await
|
|
279371
|
+
if (!(await fs6.stat(fullModelPath)).isFile()) {
|
|
279002
279372
|
throw new ModelNotFoundError(`${modelPath} is not a file.`);
|
|
279003
279373
|
}
|
|
279004
279374
|
} catch {
|
|
@@ -279036,8 +279406,12 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
279036
279406
|
return extractQueriesFromModelDef(modelDef);
|
|
279037
279407
|
}
|
|
279038
279408
|
static getSources(modelDef, givens) {
|
|
279039
|
-
const { sources, filterMap } = extractSourcesFromModelDef(modelDef, givens, (sourceName, err) => logger.warn(`Failed to parse filter annotations on source "${sourceName}"`, { error: err }));
|
|
279040
|
-
return {
|
|
279409
|
+
const { sources, filterMap, ownAuthorizeSources } = extractSourcesFromModelDef(modelDef, givens, (sourceName, err) => logger.warn(`Failed to parse filter annotations on source "${sourceName}"`, { error: err }));
|
|
279410
|
+
return {
|
|
279411
|
+
sources,
|
|
279412
|
+
filterMap,
|
|
279413
|
+
ownAuthorizeSources
|
|
279414
|
+
};
|
|
279041
279415
|
}
|
|
279042
279416
|
static async getModelMaterializer(runtime, importBaseURL, modelURL, modelPath) {
|
|
279043
279417
|
if (modelPath.endsWith(MODEL_FILE_SUFFIX)) {
|
|
@@ -279067,7 +279441,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
279067
279441
|
let fileContents = undefined;
|
|
279068
279442
|
let parse = undefined;
|
|
279069
279443
|
try {
|
|
279070
|
-
fileContents = await
|
|
279444
|
+
fileContents = await fs6.readFile(modelURL, "utf8");
|
|
279071
279445
|
} catch {
|
|
279072
279446
|
throw new ModelNotFoundError("Model not found: " + modelPath);
|
|
279073
279447
|
}
|
|
@@ -279160,7 +279534,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
279160
279534
|
async getFileText(packagePath) {
|
|
279161
279535
|
const fullPath = path6.join(packagePath, this.modelPath);
|
|
279162
279536
|
try {
|
|
279163
|
-
return await
|
|
279537
|
+
return await fs6.readFile(fullPath, "utf8");
|
|
279164
279538
|
} catch {
|
|
279165
279539
|
throw new ModelNotFoundError(`Model file not found: ${this.modelPath}`);
|
|
279166
279540
|
}
|
|
@@ -279681,6 +280055,7 @@ class Package {
|
|
|
279681
280055
|
models = new Map;
|
|
279682
280056
|
packagePath;
|
|
279683
280057
|
malloyConfig;
|
|
280058
|
+
serveDestinationConfig;
|
|
279684
280059
|
buildManifestEntries;
|
|
279685
280060
|
freshnessEntries;
|
|
279686
280061
|
storageServeBindings = [];
|
|
@@ -279761,11 +280136,11 @@ class Package {
|
|
|
279761
280136
|
if (!cleanupDirectoryOnFailure) {
|
|
279762
280137
|
logger.info(`Preserving existing package directory after failed load: ${packagePath}`);
|
|
279763
280138
|
} else {
|
|
279764
|
-
const stat6 = await
|
|
280139
|
+
const stat6 = await fs7.lstat(packagePath).catch(() => null);
|
|
279765
280140
|
if (stat6?.isSymbolicLink()) {
|
|
279766
280141
|
logger.info(`Skipping cleanup of symlinked package path on failure: ${packagePath}`);
|
|
279767
280142
|
} else {
|
|
279768
|
-
await
|
|
280143
|
+
await fs7.rm(packagePath, { recursive: true, force: true });
|
|
279769
280144
|
logger.info(`Cleaned up failed package directory: ${packagePath}`);
|
|
279770
280145
|
}
|
|
279771
280146
|
}
|
|
@@ -279839,7 +280214,7 @@ class Package {
|
|
|
279839
280214
|
renderTagWarnings.push({ model: sm.modelPath, ...w });
|
|
279840
280215
|
}
|
|
279841
280216
|
if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
|
|
279842
|
-
const modelSource = await
|
|
280217
|
+
const modelSource = await fs7.readFile(path7.join(packagePath, sm.modelPath), "utf-8");
|
|
279843
280218
|
assertPersistNamesQuoted(modelSource, sm.modelPath);
|
|
279844
280219
|
}
|
|
279845
280220
|
models.set(sm.modelPath, model);
|
|
@@ -279940,7 +280315,7 @@ class Package {
|
|
|
279940
280315
|
if (this.storageServeBindings.length > 0) {
|
|
279941
280316
|
metadata.storageServeBindings = this.storageServeBindings.map((b) => ({
|
|
279942
280317
|
sourceName: b.sourceName,
|
|
279943
|
-
|
|
280318
|
+
storageDestinationName: b.destinationName,
|
|
279944
280319
|
tablePath: b.tablePath
|
|
279945
280320
|
}));
|
|
279946
280321
|
}
|
|
@@ -280016,6 +280391,22 @@ class Package {
|
|
|
280016
280391
|
model.setServeBindings(this.storageServeBindings);
|
|
280017
280392
|
}
|
|
280018
280393
|
}
|
|
280394
|
+
setServeDestinationConfig(provider) {
|
|
280395
|
+
this.serveDestinationConfig = provider;
|
|
280396
|
+
this.pushServeDestinationConfigToModels();
|
|
280397
|
+
}
|
|
280398
|
+
pushServeDestinationConfigToModels() {
|
|
280399
|
+
if (!this.serveDestinationConfig)
|
|
280400
|
+
return;
|
|
280401
|
+
for (const model of this.models.values()) {
|
|
280402
|
+
model.setServeDestinationConfig(this.serveDestinationConfig);
|
|
280403
|
+
}
|
|
280404
|
+
}
|
|
280405
|
+
invalidateServeShapes() {
|
|
280406
|
+
for (const model of this.models.values()) {
|
|
280407
|
+
model.invalidateServeShapeCache();
|
|
280408
|
+
}
|
|
280409
|
+
}
|
|
280019
280410
|
getInvalidExplores(exploresOverride) {
|
|
280020
280411
|
const declared = exploresOverride ?? this.packageMetadata.explores;
|
|
280021
280412
|
if (!declared || declared.length === 0)
|
|
@@ -280258,6 +280649,7 @@ class Package {
|
|
|
280258
280649
|
}
|
|
280259
280650
|
this.models = nextModels;
|
|
280260
280651
|
this.pushStorageServeBindingsToModels();
|
|
280652
|
+
this.pushServeDestinationConfigToModels();
|
|
280261
280653
|
this.renderTagWarnings = renderTagWarnings;
|
|
280262
280654
|
this.manifestWarnings = outcome.packageMetadata.manifestWarnings ?? [];
|
|
280263
280655
|
this.packageMetadata.explores = outcome.packageMetadata.explores;
|
|
@@ -280354,7 +280746,7 @@ class Package {
|
|
|
280354
280746
|
static async validatePackageManifestExistsOrThrowError(packagePath) {
|
|
280355
280747
|
const packageConfigPath = safeJoinUnderRoot(packagePath, PACKAGE_MANIFEST_NAME);
|
|
280356
280748
|
try {
|
|
280357
|
-
await
|
|
280749
|
+
await fs7.stat(packageConfigPath);
|
|
280358
280750
|
} catch {
|
|
280359
280751
|
logger.error(`Can't find ${packageConfigPath}`);
|
|
280360
280752
|
throw new PackageNotFoundError(`Package manifest for ${packagePath} does not exist.`);
|
|
@@ -280435,6 +280827,11 @@ var STAGING_DIR_NAME = ".staging";
|
|
|
280435
280827
|
var RETIRED_DIR_NAME = ".retired";
|
|
280436
280828
|
var MANIFEST_FETCH_TIMEOUT_MS = 15000;
|
|
280437
280829
|
var RETIRED_CONNECTION_DRAIN_MS = 30000;
|
|
280830
|
+
var DESTINATION_READ_FIELDS = new Set([
|
|
280831
|
+
"name",
|
|
280832
|
+
"type",
|
|
280833
|
+
"resource"
|
|
280834
|
+
]);
|
|
280438
280835
|
var queryAdmissionRejectionsCounter = null;
|
|
280439
280836
|
var packageAdmissionRejectionsCounter = null;
|
|
280440
280837
|
function getQueryAdmissionRejectionsCounter() {
|
|
@@ -280464,6 +280861,9 @@ class Environment {
|
|
|
280464
280861
|
connectionMutex = new Mutex;
|
|
280465
280862
|
retiredConnectionGenerations = new Set;
|
|
280466
280863
|
apiConnections;
|
|
280864
|
+
destinations;
|
|
280865
|
+
destinationMalloyConfig;
|
|
280866
|
+
destinationsAuthoritative = true;
|
|
280467
280867
|
environmentPath;
|
|
280468
280868
|
environmentName;
|
|
280469
280869
|
storageBindingResolver;
|
|
@@ -280475,12 +280875,14 @@ class Environment {
|
|
|
280475
280875
|
getEnvironmentName() {
|
|
280476
280876
|
return this.environmentName;
|
|
280477
280877
|
}
|
|
280478
|
-
constructor(environmentName, environmentPath, malloyConfig, apiConnections) {
|
|
280878
|
+
constructor(environmentName, environmentPath, malloyConfig, apiConnections, storageDestinations = []) {
|
|
280479
280879
|
assertSafeEnvironmentPath(environmentPath);
|
|
280480
280880
|
this.environmentName = environmentName;
|
|
280481
280881
|
this.environmentPath = environmentPath;
|
|
280482
280882
|
this.malloyConfig = malloyConfig;
|
|
280483
280883
|
this.apiConnections = apiConnections;
|
|
280884
|
+
this.destinations = [];
|
|
280885
|
+
this.setStorageDestinations(storageDestinations);
|
|
280484
280886
|
this.metadata = {
|
|
280485
280887
|
resource: `${API_PREFIX}/environments/${this.environmentName}`,
|
|
280486
280888
|
name: this.environmentName,
|
|
@@ -280493,7 +280895,7 @@ class Environment {
|
|
|
280493
280895
|
return;
|
|
280494
280896
|
const readmePath = path8.join(this.environmentPath, "README.md");
|
|
280495
280897
|
try {
|
|
280496
|
-
await
|
|
280898
|
+
await fs8.promises.writeFile(readmePath, readme, "utf-8");
|
|
280497
280899
|
logger.info(`Updated README.md for environment ${this.environmentName}`);
|
|
280498
280900
|
} catch (err) {
|
|
280499
280901
|
logger.error(`Failed to write README.md`, { error: err });
|
|
@@ -280501,6 +280903,11 @@ class Environment {
|
|
|
280501
280903
|
}
|
|
280502
280904
|
}
|
|
280503
280905
|
async update(payload) {
|
|
280906
|
+
if (payload.storageDestinations !== undefined) {
|
|
280907
|
+
this.setStorageDestinations(payload.storageDestinations, {
|
|
280908
|
+
rejectInvalid: true
|
|
280909
|
+
});
|
|
280910
|
+
}
|
|
280504
280911
|
if (payload.readme !== undefined) {
|
|
280505
280912
|
this.metadata.readme = payload.readme;
|
|
280506
280913
|
await this.writeEnvironmentReadme(payload.readme);
|
|
@@ -280520,9 +280927,9 @@ class Environment {
|
|
|
280520
280927
|
}
|
|
280521
280928
|
return this;
|
|
280522
280929
|
}
|
|
280523
|
-
static async create(environmentName, environmentPath, connections) {
|
|
280930
|
+
static async create(environmentName, environmentPath, connections, storageDestinations = []) {
|
|
280524
280931
|
assertSafeEnvironmentPath(environmentPath);
|
|
280525
|
-
if (!(await
|
|
280932
|
+
if (!(await fs8.promises.stat(environmentPath))?.isDirectory()) {
|
|
280526
280933
|
throw new EnvironmentNotFoundError(`Environment path ${environmentPath} not found`);
|
|
280527
280934
|
}
|
|
280528
280935
|
logger.info(`Creating environment with connection configuration`);
|
|
@@ -280533,14 +280940,14 @@ class Environment {
|
|
|
280533
280940
|
type: c.type
|
|
280534
280941
|
}))
|
|
280535
280942
|
});
|
|
280536
|
-
const environment = new Environment(environmentName, environmentPath, malloyConfig, malloyConfig.apiConnections);
|
|
280943
|
+
const environment = new Environment(environmentName, environmentPath, malloyConfig, malloyConfig.apiConnections, storageDestinations);
|
|
280537
280944
|
await Environment.sweepStaleInstallDirs(environmentPath);
|
|
280538
280945
|
return environment;
|
|
280539
280946
|
}
|
|
280540
280947
|
async reloadEnvironmentMetadata() {
|
|
280541
280948
|
let readme = "";
|
|
280542
280949
|
try {
|
|
280543
|
-
readme = (await
|
|
280950
|
+
readme = (await fs8.promises.readFile(safeJoinUnderRoot(this.environmentPath, README_NAME))).toString();
|
|
280544
280951
|
} catch {}
|
|
280545
280952
|
this.metadata = {
|
|
280546
280953
|
...this.metadata,
|
|
@@ -280553,6 +280960,12 @@ class Environment {
|
|
|
280553
280960
|
async compileSource(packageName, modelName, source, includeSql = false, givens) {
|
|
280554
280961
|
assertSafePackageName(packageName);
|
|
280555
280962
|
assertSafeRelativeModelPath(modelName);
|
|
280963
|
+
try {
|
|
280964
|
+
assertNoCallerAuthorizeAnnotation(source);
|
|
280965
|
+
} catch (err) {
|
|
280966
|
+
recordAuthorizeGuardRejection("compile_source");
|
|
280967
|
+
throw err;
|
|
280968
|
+
}
|
|
280556
280969
|
if (modelName.endsWith(NOTEBOOK_FILE_SUFFIX)) {
|
|
280557
280970
|
throw new BadRequestError(`Cannot compile against a notebook ("${modelName}"). ` + `/compile takes a .malloy model path for namespace context.`);
|
|
280558
280971
|
}
|
|
@@ -280563,7 +280976,7 @@ class Environment {
|
|
|
280563
280976
|
const virtualUri = virtualUrl.toString();
|
|
280564
280977
|
let modelContent = "";
|
|
280565
280978
|
try {
|
|
280566
|
-
modelContent = await
|
|
280979
|
+
modelContent = await fs8.promises.readFile(modelPath, "utf8");
|
|
280567
280980
|
} catch {}
|
|
280568
280981
|
const fullSource = modelContent ? `${modelContent}
|
|
280569
280982
|
${source}` : source;
|
|
@@ -280633,6 +281046,76 @@ ${source}` : source;
|
|
|
280633
281046
|
}
|
|
280634
281047
|
return connection;
|
|
280635
281048
|
}
|
|
281049
|
+
setStorageDestinations(storageDestinations, { rejectInvalid = false } = {}) {
|
|
281050
|
+
const previous = this.destinations;
|
|
281051
|
+
const requested = Array.isArray(storageDestinations) ? storageDestinations.map((destination) => this.resolveDestinationReference(destination)) : storageDestinations;
|
|
281052
|
+
this.destinations = rejectInvalid ? processStorageDestinationsOrThrow(requested) : processStorageDestinations(requested);
|
|
281053
|
+
this.destinationsAuthoritative = true;
|
|
281054
|
+
if (this.destinationMalloyConfig && storageDestinationsEqual(previous, this.destinations)) {
|
|
281055
|
+
return;
|
|
281056
|
+
}
|
|
281057
|
+
this.rebuildDestinationMalloyConfig();
|
|
281058
|
+
if (previous.length > 0 || this.destinations.length > 0) {
|
|
281059
|
+
logger.info(`Environment ${this.environmentName} has ${this.destinations.length} storage destination(s)`, { destinations: this.destinations.map((d) => d.name) });
|
|
281060
|
+
}
|
|
281061
|
+
}
|
|
281062
|
+
resolveDestinationReference(destination) {
|
|
281063
|
+
if (!destination || typeof destination !== "object") {
|
|
281064
|
+
return destination;
|
|
281065
|
+
}
|
|
281066
|
+
const carriesOnlyReportedFields = Object.keys(destination).every((field) => DESTINATION_READ_FIELDS.has(field));
|
|
281067
|
+
if (!carriesOnlyReportedFields) {
|
|
281068
|
+
return destination;
|
|
281069
|
+
}
|
|
281070
|
+
return this.destinations.find((stored) => stored.name === destination.name) ?? destination;
|
|
281071
|
+
}
|
|
281072
|
+
attachDestinationServeConfig(_package) {
|
|
281073
|
+
_package.setServeDestinationConfig(() => this.getStorageDestinationMalloyConfig());
|
|
281074
|
+
}
|
|
281075
|
+
rebuildDestinationMalloyConfig() {
|
|
281076
|
+
const previous = this.destinationMalloyConfig;
|
|
281077
|
+
try {
|
|
281078
|
+
const destinationRoot = storageDestinationRoot(this.environmentPath);
|
|
281079
|
+
if (this.destinations.length > 0) {
|
|
281080
|
+
fs8.mkdirSync(destinationRoot, { recursive: true });
|
|
281081
|
+
}
|
|
281082
|
+
this.destinationMalloyConfig = buildEnvironmentMalloyConfig(this.destinations, destinationRoot);
|
|
281083
|
+
} catch (error) {
|
|
281084
|
+
logger.error(`Failed to assemble storage destinations for environment ${this.environmentName}; serving without them`, { error });
|
|
281085
|
+
this.destinationMalloyConfig = buildEnvironmentMalloyConfig([], storageDestinationRoot(this.environmentPath));
|
|
281086
|
+
}
|
|
281087
|
+
if (previous && previous !== this.destinationMalloyConfig) {
|
|
281088
|
+
this.retireConnectionGeneration(`environment ${this.environmentName} destinations`, () => previous.releaseConnections());
|
|
281089
|
+
this.invalidateServeShapes();
|
|
281090
|
+
}
|
|
281091
|
+
}
|
|
281092
|
+
invalidateServeShapes() {
|
|
281093
|
+
for (const _package of this.packages.values()) {
|
|
281094
|
+
_package.invalidateServeShapes();
|
|
281095
|
+
}
|
|
281096
|
+
}
|
|
281097
|
+
getStorageDestinationMalloyConfig() {
|
|
281098
|
+
return this.destinationMalloyConfig.malloyConfig;
|
|
281099
|
+
}
|
|
281100
|
+
markStorageDestinationsUnknown() {
|
|
281101
|
+
this.destinationsAuthoritative = false;
|
|
281102
|
+
}
|
|
281103
|
+
hasAuthoritativeStorageDestinations() {
|
|
281104
|
+
return this.destinationsAuthoritative;
|
|
281105
|
+
}
|
|
281106
|
+
listStorageDestinations() {
|
|
281107
|
+
return this.destinations;
|
|
281108
|
+
}
|
|
281109
|
+
hasStorageDestination(destinationName) {
|
|
281110
|
+
return this.destinations.some((destination) => destination.name === destinationName);
|
|
281111
|
+
}
|
|
281112
|
+
getStorageDestination(destinationName) {
|
|
281113
|
+
const destination = this.destinations.find((destination2) => destination2.name === destinationName);
|
|
281114
|
+
if (!destination) {
|
|
281115
|
+
throw new DestinationNotFoundError(`Storage destination ${destinationName} not found`);
|
|
281116
|
+
}
|
|
281117
|
+
return destination;
|
|
281118
|
+
}
|
|
280636
281119
|
async getMalloyConnection(connectionName) {
|
|
280637
281120
|
return this.malloyConfig.malloyConfig.connections.lookupConnection(connectionName);
|
|
280638
281121
|
}
|
|
@@ -280714,10 +281197,10 @@ ${source}` : source;
|
|
|
280714
281197
|
return this.getOrCreatePackageMutex(packageName).runExclusive(fn);
|
|
280715
281198
|
}
|
|
280716
281199
|
allocateStagingPath(packageName) {
|
|
280717
|
-
return safeJoinUnderRoot(this.environmentPath, STAGING_DIR_NAME, `${packageName}-${
|
|
281200
|
+
return safeJoinUnderRoot(this.environmentPath, STAGING_DIR_NAME, `${packageName}-${crypto4.randomUUID()}`);
|
|
280718
281201
|
}
|
|
280719
281202
|
allocateRetiredPath(packageName) {
|
|
280720
|
-
return safeJoinUnderRoot(this.environmentPath, RETIRED_DIR_NAME, `${packageName}-${
|
|
281203
|
+
return safeJoinUnderRoot(this.environmentPath, RETIRED_DIR_NAME, `${packageName}-${crypto4.randomUUID()}`);
|
|
280721
281204
|
}
|
|
280722
281205
|
static async sweepStaleInstallDirs(environmentPath) {
|
|
280723
281206
|
assertSafeEnvironmentPath(environmentPath);
|
|
@@ -280728,7 +281211,7 @@ ${source}` : source;
|
|
|
280728
281211
|
if (path8.basename(dir) !== dirName)
|
|
280729
281212
|
continue;
|
|
280730
281213
|
try {
|
|
280731
|
-
await
|
|
281214
|
+
await fs8.promises.rm(dir, { recursive: true, force: true });
|
|
280732
281215
|
} catch (err) {
|
|
280733
281216
|
logger.warn(`Failed to sweep stale ${dirName} dir at ${dir}`, {
|
|
280734
281217
|
error: err
|
|
@@ -280804,6 +281287,7 @@ ${source}` : source;
|
|
|
280804
281287
|
logger.debug(`Loading package ${packageName}...`);
|
|
280805
281288
|
const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
280806
281289
|
const _package = await Package.create(this.environmentName, packageName, packagePath, () => this.malloyConfig.malloyConfig);
|
|
281290
|
+
this.attachDestinationServeConfig(_package);
|
|
280807
281291
|
await this.bindManifestIfConfigured(_package);
|
|
280808
281292
|
await this.rebindServeBindingsFromLocalStore(_package);
|
|
280809
281293
|
if (existingPackage !== undefined && reload) {
|
|
@@ -280828,7 +281312,7 @@ ${source}` : source;
|
|
|
280828
281312
|
async addPackage(packageName, options = {}) {
|
|
280829
281313
|
assertSafePackageName(packageName);
|
|
280830
281314
|
const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
280831
|
-
if (!await
|
|
281315
|
+
if (!await fs8.promises.access(packagePath).then(() => true).catch(() => false) || !(await fs8.promises.stat(packagePath))?.isDirectory()) {
|
|
280832
281316
|
throw new PackageNotFoundError(`Package ${packageName} not found`);
|
|
280833
281317
|
}
|
|
280834
281318
|
this.assertCanAdmitNewPackage(packageName, "add a new package", options.allowAdmission === true);
|
|
@@ -280845,7 +281329,9 @@ ${source}` : source;
|
|
|
280845
281329
|
}
|
|
280846
281330
|
this.setPackageStatus(packageName, "loading" /* LOADING */);
|
|
280847
281331
|
try {
|
|
280848
|
-
|
|
281332
|
+
const addedPackage = await Package.create(this.environmentName, packageName, packagePath, () => this.malloyConfig.malloyConfig);
|
|
281333
|
+
this.attachDestinationServeConfig(addedPackage);
|
|
281334
|
+
this.packages.set(packageName, addedPackage);
|
|
280849
281335
|
} catch (error) {
|
|
280850
281336
|
logger.error("Error adding package", { error });
|
|
280851
281337
|
this.deletePackageStatus(packageName);
|
|
@@ -280858,7 +281344,7 @@ ${source}` : source;
|
|
|
280858
281344
|
async installPackage(packageName, downloader, validate) {
|
|
280859
281345
|
assertSafePackageName(packageName);
|
|
280860
281346
|
const stagingPath = this.allocateStagingPath(packageName);
|
|
280861
|
-
await
|
|
281347
|
+
await fs8.promises.mkdir(path8.dirname(stagingPath), { recursive: true });
|
|
280862
281348
|
logger.debug("install.phase1.download.started", {
|
|
280863
281349
|
environmentName: this.environmentName,
|
|
280864
281350
|
packageName,
|
|
@@ -280868,7 +281354,7 @@ ${source}` : source;
|
|
|
280868
281354
|
try {
|
|
280869
281355
|
await downloader(stagingPath);
|
|
280870
281356
|
} catch (err) {
|
|
280871
|
-
await
|
|
281357
|
+
await fs8.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
|
|
280872
281358
|
throw err;
|
|
280873
281359
|
}
|
|
280874
281360
|
logger.debug("install.phase1.download.completed", {
|
|
@@ -280884,13 +281370,13 @@ ${source}` : source;
|
|
|
280884
281370
|
const canonicalPath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
280885
281371
|
let retiredPath;
|
|
280886
281372
|
const oldPackage = this.packages.get(packageName);
|
|
280887
|
-
const oldExistsOnDisk = await
|
|
281373
|
+
const oldExistsOnDisk = await fs8.promises.access(canonicalPath).then(() => true).catch(() => false);
|
|
280888
281374
|
if (oldExistsOnDisk) {
|
|
280889
281375
|
retiredPath = this.allocateRetiredPath(packageName);
|
|
280890
|
-
await
|
|
281376
|
+
await fs8.promises.mkdir(path8.dirname(retiredPath), {
|
|
280891
281377
|
recursive: true
|
|
280892
281378
|
});
|
|
280893
|
-
await
|
|
281379
|
+
await fs8.promises.rename(canonicalPath, retiredPath);
|
|
280894
281380
|
logger.debug("install.phase2.retired_old", {
|
|
280895
281381
|
environmentName: this.environmentName,
|
|
280896
281382
|
packageName,
|
|
@@ -280899,9 +281385,10 @@ ${source}` : source;
|
|
|
280899
281385
|
}
|
|
280900
281386
|
let newPackage;
|
|
280901
281387
|
try {
|
|
280902
|
-
await
|
|
281388
|
+
await fs8.promises.rename(stagingPath, canonicalPath);
|
|
280903
281389
|
this.setPackageStatus(packageName, "loading" /* LOADING */);
|
|
280904
281390
|
newPackage = await Package.create(this.environmentName, packageName, canonicalPath, () => this.malloyConfig.malloyConfig, true);
|
|
281391
|
+
this.attachDestinationServeConfig(newPackage);
|
|
280905
281392
|
const validationMsg = validate?.(newPackage);
|
|
280906
281393
|
if (validationMsg) {
|
|
280907
281394
|
throw new BadRequestError(validationMsg);
|
|
@@ -280912,11 +281399,11 @@ ${source}` : source;
|
|
|
280912
281399
|
canonicalPath
|
|
280913
281400
|
});
|
|
280914
281401
|
} catch (err) {
|
|
280915
|
-
await
|
|
281402
|
+
await fs8.promises.rm(canonicalPath, { recursive: true, force: true }).catch(() => {});
|
|
280916
281403
|
let restored = false;
|
|
280917
281404
|
if (retiredPath) {
|
|
280918
281405
|
try {
|
|
280919
|
-
await
|
|
281406
|
+
await fs8.promises.rename(retiredPath, canonicalPath);
|
|
280920
281407
|
restored = true;
|
|
280921
281408
|
} catch (restoreErr) {
|
|
280922
281409
|
logger.error("Failed to restore retired package after install rollback", {
|
|
@@ -280926,7 +281413,7 @@ ${source}` : source;
|
|
|
280926
281413
|
});
|
|
280927
281414
|
}
|
|
280928
281415
|
}
|
|
280929
|
-
await
|
|
281416
|
+
await fs8.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
|
|
280930
281417
|
if (oldPackage && restored) {
|
|
280931
281418
|
this.setPackageStatus(packageName, "serving" /* SERVING */);
|
|
280932
281419
|
} else {
|
|
@@ -280960,7 +281447,7 @@ ${source}` : source;
|
|
|
280960
281447
|
packageName,
|
|
280961
281448
|
retiredPath: pathToClean
|
|
280962
281449
|
});
|
|
280963
|
-
|
|
281450
|
+
fs8.promises.rm(pathToClean, { recursive: true, force: true }).catch((err) => {
|
|
280964
281451
|
logger.warn(`Failed to clean up retired package directory ${pathToClean}`, { error: err });
|
|
280965
281452
|
});
|
|
280966
281453
|
});
|
|
@@ -281093,7 +281580,7 @@ ${source}` : source;
|
|
|
281093
281580
|
try {
|
|
281094
281581
|
let existingManifest = {};
|
|
281095
281582
|
try {
|
|
281096
|
-
const content = await
|
|
281583
|
+
const content = await fs8.promises.readFile(manifestPath, "utf-8");
|
|
281097
281584
|
existingManifest = JSON.parse(content);
|
|
281098
281585
|
} catch (_err) {
|
|
281099
281586
|
logger.warn(`Could not read manifest for ${packageName}`);
|
|
@@ -281113,7 +281600,7 @@ ${source}` : source;
|
|
|
281113
281600
|
...resolvedScope !== undefined ? { scope: resolvedScope } : {},
|
|
281114
281601
|
...materializationBlock !== undefined ? { materialization: materializationBlock } : {}
|
|
281115
281602
|
};
|
|
281116
|
-
await
|
|
281603
|
+
await fs8.promises.writeFile(manifestPath, JSON.stringify(updatedManifest, null, 2), "utf-8");
|
|
281117
281604
|
logger.info(`Updated publisher.json for ${packageName}`);
|
|
281118
281605
|
} catch (error) {
|
|
281119
281606
|
logger.error(`Failed to update publisher.json`, { error });
|
|
@@ -281236,10 +281723,10 @@ ${source}` : source;
|
|
|
281236
281723
|
const retiredPath = this.allocateRetiredPath(packageName);
|
|
281237
281724
|
let renamed = false;
|
|
281238
281725
|
try {
|
|
281239
|
-
await
|
|
281726
|
+
await fs8.promises.mkdir(path8.dirname(retiredPath), {
|
|
281240
281727
|
recursive: true
|
|
281241
281728
|
});
|
|
281242
|
-
await
|
|
281729
|
+
await fs8.promises.rename(canonicalPath, retiredPath);
|
|
281243
281730
|
renamed = true;
|
|
281244
281731
|
} catch (err) {
|
|
281245
281732
|
logger.error("Error renaming package directory to retired during unload", {
|
|
@@ -281252,7 +281739,7 @@ ${source}` : source;
|
|
|
281252
281739
|
this.packageStatuses.delete(packageName);
|
|
281253
281740
|
if (renamed) {
|
|
281254
281741
|
setImmediate(() => {
|
|
281255
|
-
|
|
281742
|
+
fs8.promises.rm(retiredPath, { recursive: true, force: true }).catch((err) => {
|
|
281256
281743
|
logger.warn(`Failed to clean up retired package directory ${retiredPath}`, { error: err });
|
|
281257
281744
|
});
|
|
281258
281745
|
});
|
|
@@ -281312,6 +281799,13 @@ ${source}` : source;
|
|
|
281312
281799
|
} catch (error) {
|
|
281313
281800
|
logger.error(`Error closing connections for environment ${this.environmentName}`, { error });
|
|
281314
281801
|
}
|
|
281802
|
+
try {
|
|
281803
|
+
await this.destinationMalloyConfig.releaseConnections();
|
|
281804
|
+
} catch (error) {
|
|
281805
|
+
logger.error(`Error closing storage destinations for environment ${this.environmentName}`, { error: error instanceof Error ? error.message : String(error) });
|
|
281806
|
+
}
|
|
281807
|
+
this.destinations = [];
|
|
281808
|
+
this.destinationsAuthoritative = false;
|
|
281315
281809
|
await this.releaseAllRetiredConnectionGenerations();
|
|
281316
281810
|
this.apiConnections = [];
|
|
281317
281811
|
logger.info(`Closed all connections for environment ${this.environmentName}`);
|
|
@@ -281320,13 +281814,17 @@ ${source}` : source;
|
|
|
281320
281814
|
return {
|
|
281321
281815
|
...this.metadata,
|
|
281322
281816
|
connections: this.listApiConnections(),
|
|
281817
|
+
storageDestinations: this.destinations.map(({ name, type }) => ({
|
|
281818
|
+
name,
|
|
281819
|
+
type
|
|
281820
|
+
})),
|
|
281323
281821
|
packages: await this.listPackages()
|
|
281324
281822
|
};
|
|
281325
281823
|
}
|
|
281326
281824
|
async deleteDuckDBConnection(connectionName) {
|
|
281327
281825
|
const duckdbPath = path8.join(this.environmentPath, `${connectionName}.duckdb`);
|
|
281328
281826
|
try {
|
|
281329
|
-
await
|
|
281827
|
+
await fs8.promises.rm(duckdbPath, { force: true });
|
|
281330
281828
|
logger.info(`Removed DuckDB connection file ${connectionName} from environment ${this.environmentName}`);
|
|
281331
281829
|
} catch (error) {
|
|
281332
281830
|
logger.error(`Failed to remove DuckDB connection file ${connectionName} from environment ${this.environmentName}`, { error });
|
|
@@ -281387,11 +281885,11 @@ function validateEnvironmentAzureUrls(environment) {
|
|
|
281387
281885
|
}
|
|
281388
281886
|
async function clearMountTarget(targetPath) {
|
|
281389
281887
|
try {
|
|
281390
|
-
const stats = await
|
|
281888
|
+
const stats = await fs9.promises.lstat(targetPath);
|
|
281391
281889
|
if (stats.isDirectory() && !stats.isSymbolicLink()) {
|
|
281392
|
-
await
|
|
281890
|
+
await fs9.promises.rm(targetPath, { recursive: true, force: true });
|
|
281393
281891
|
} else {
|
|
281394
|
-
await
|
|
281892
|
+
await fs9.promises.unlink(targetPath);
|
|
281395
281893
|
}
|
|
281396
281894
|
} catch {}
|
|
281397
281895
|
}
|
|
@@ -281609,6 +282107,9 @@ class EnvironmentStore {
|
|
|
281609
282107
|
name: environment.name,
|
|
281610
282108
|
resource: `${API_PREFIX}/environments/${environment.name}`,
|
|
281611
282109
|
connections: environment.connections,
|
|
282110
|
+
...environment.storageDestinations.length ? {
|
|
282111
|
+
storageDestinations: environment.storageDestinations
|
|
282112
|
+
} : {},
|
|
281612
282113
|
packages: environment.packages
|
|
281613
282114
|
}, true);
|
|
281614
282115
|
} catch (error) {
|
|
@@ -281640,16 +282141,17 @@ class EnvironmentStore {
|
|
|
281640
282141
|
if (existingEnvironments.length > 0) {
|
|
281641
282142
|
await Promise.all(existingEnvironments.map(async (dbEnvironment) => {
|
|
281642
282143
|
try {
|
|
281643
|
-
const environmentExists = await
|
|
282144
|
+
const environmentExists = await fs9.promises.access(dbEnvironment.path).then(() => true).catch(() => false);
|
|
282145
|
+
const environmentConfig = environmentManifest.environments.find((p) => p.name === dbEnvironment.name);
|
|
281644
282146
|
if (!environmentExists) {
|
|
281645
|
-
const environmentConfig = environmentManifest.environments.find((p) => p.name === dbEnvironment.name);
|
|
281646
282147
|
if (environmentConfig) {
|
|
281647
282148
|
const environmentInstance2 = await this.addEnvironment({
|
|
281648
282149
|
name: environmentConfig.name,
|
|
281649
282150
|
resource: `${API_PREFIX}/environments/${environmentConfig.name}`,
|
|
281650
282151
|
connections: environmentConfig.connections,
|
|
282152
|
+
storageDestinations: environmentConfig.storageDestinations,
|
|
281651
282153
|
packages: environmentConfig.packages
|
|
281652
|
-
}, true);
|
|
282154
|
+
}, true, { storageDestinationsAreFallback: true });
|
|
281653
282155
|
await repository.updateEnvironment(dbEnvironment.id, {
|
|
281654
282156
|
path: environmentInstance2.metadata.location
|
|
281655
282157
|
});
|
|
@@ -281660,12 +282162,30 @@ class EnvironmentStore {
|
|
|
281660
282162
|
}
|
|
281661
282163
|
}
|
|
281662
282164
|
const connections = await repository.listConnections(dbEnvironment.id);
|
|
282165
|
+
let destinations = environmentConfig?.storageDestinations;
|
|
282166
|
+
let destinationsRead = true;
|
|
282167
|
+
try {
|
|
282168
|
+
const stored = await repository.listStorageDestinations(dbEnvironment.id);
|
|
282169
|
+
if (stored.length) {
|
|
282170
|
+
destinations = stored.map((row) => ({
|
|
282171
|
+
name: row.name,
|
|
282172
|
+
type: row.type,
|
|
282173
|
+
...row.config
|
|
282174
|
+
}));
|
|
282175
|
+
}
|
|
282176
|
+
} catch (error) {
|
|
282177
|
+
destinationsRead = false;
|
|
282178
|
+
logger.error(`Error reading storage destinations for "${dbEnvironment.name}"; continuing without the stored ones`, { error });
|
|
282179
|
+
}
|
|
281663
282180
|
const environmentInstance = await Environment.create(dbEnvironment.name, dbEnvironment.path, connections.map((conn) => ({
|
|
281664
282181
|
name: conn.name,
|
|
281665
282182
|
type: conn.type,
|
|
281666
282183
|
resource: `${API_PREFIX}/connections/${conn.name}`,
|
|
281667
282184
|
...conn.config
|
|
281668
|
-
})));
|
|
282185
|
+
})), destinations);
|
|
282186
|
+
if (!destinationsRead) {
|
|
282187
|
+
environmentInstance.markStorageDestinationsUnknown();
|
|
282188
|
+
}
|
|
281669
282189
|
environmentInstance.setMemoryGovernor(this.memoryGovernor);
|
|
281670
282190
|
const envId = dbEnvironment.id;
|
|
281671
282191
|
environmentInstance.setStorageBindingResolver(async (packageName) => {
|
|
@@ -281732,6 +282252,7 @@ class EnvironmentStore {
|
|
|
281732
282252
|
const repository = this.storageManager.getRepository();
|
|
281733
282253
|
const dbEnvironment = await this.addEnvironmentMetadata(environment, repository);
|
|
281734
282254
|
await this.addConnections(environment, dbEnvironment.id, repository);
|
|
282255
|
+
await this.syncStorageDestinations(environment, dbEnvironment.id, repository);
|
|
281735
282256
|
await this.addPackages(environment, dbEnvironment.id, repository);
|
|
281736
282257
|
logger.info(`Synced environment "${environmentName}" to database`);
|
|
281737
282258
|
}
|
|
@@ -281832,6 +282353,38 @@ class EnvironmentStore {
|
|
|
281832
282353
|
logger.error(`Error syncing connections for "${environmentName}":`, error);
|
|
281833
282354
|
}
|
|
281834
282355
|
}
|
|
282356
|
+
async syncStorageDestinations(environment, environmentId, repository) {
|
|
282357
|
+
try {
|
|
282358
|
+
const destinations = environment.listStorageDestinations();
|
|
282359
|
+
const kept = new Set;
|
|
282360
|
+
for (const destination of destinations) {
|
|
282361
|
+
if (!destination.name || !destination.type) {
|
|
282362
|
+
continue;
|
|
282363
|
+
}
|
|
282364
|
+
kept.add(destination.name);
|
|
282365
|
+
await repository.upsertStorageDestination({
|
|
282366
|
+
environmentId,
|
|
282367
|
+
name: destination.name,
|
|
282368
|
+
type: destination.type,
|
|
282369
|
+
config: destination
|
|
282370
|
+
});
|
|
282371
|
+
}
|
|
282372
|
+
if (!environment.hasAuthoritativeStorageDestinations()) {
|
|
282373
|
+
logger.warn(`Not reconciling stored storage destinations for "${environment.metadata?.name}": this environment's destinations were never read, so the in-memory list is not the desired state`);
|
|
282374
|
+
return;
|
|
282375
|
+
}
|
|
282376
|
+
const stored = await repository.listStorageDestinations(environmentId);
|
|
282377
|
+
for (const row of stored) {
|
|
282378
|
+
if (!kept.has(row.name)) {
|
|
282379
|
+
await repository.deleteStorageDestination(row.id);
|
|
282380
|
+
logger.info(`Removed storage destination ${row.name} from environment ${environment.metadata?.name}`);
|
|
282381
|
+
}
|
|
282382
|
+
}
|
|
282383
|
+
} catch (err) {
|
|
282384
|
+
const error = err;
|
|
282385
|
+
logger.error(`Error syncing storage destinations for "${environment.metadata?.name}":`, error);
|
|
282386
|
+
}
|
|
282387
|
+
}
|
|
281835
282388
|
async addConnection(conn, environmentId, repository) {
|
|
281836
282389
|
if (!conn.name) {
|
|
281837
282390
|
logger.warn("Skipping connection with undefined name");
|
|
@@ -281940,12 +282493,12 @@ class EnvironmentStore {
|
|
|
281940
282493
|
}
|
|
281941
282494
|
async cleanupAndCreatePublisherPath() {
|
|
281942
282495
|
const reInit = process.env.INITIALIZE_STORAGE === "true";
|
|
281943
|
-
await
|
|
282496
|
+
await fs9.promises.mkdir(this.serverRootPath, { recursive: true });
|
|
281944
282497
|
if (reInit) {
|
|
281945
282498
|
const uploadDocsPath2 = path9.join(this.serverRootPath, PUBLISHER_DATA_DIR);
|
|
281946
282499
|
logger.info(`Reinitialization mode: Cleaning up upload documents path ${uploadDocsPath2}`);
|
|
281947
282500
|
try {
|
|
281948
|
-
await
|
|
282501
|
+
await fs9.promises.rm(uploadDocsPath2, {
|
|
281949
282502
|
recursive: true,
|
|
281950
282503
|
force: true
|
|
281951
282504
|
});
|
|
@@ -281960,7 +282513,7 @@ class EnvironmentStore {
|
|
|
281960
282513
|
logger.info(`Using existing publisher path`);
|
|
281961
282514
|
}
|
|
281962
282515
|
const uploadDocsPath = path9.join(this.serverRootPath, PUBLISHER_DATA_DIR);
|
|
281963
|
-
await
|
|
282516
|
+
await fs9.promises.mkdir(uploadDocsPath, { recursive: true });
|
|
281964
282517
|
}
|
|
281965
282518
|
async listEnvironments(skipInitializationCheck = false) {
|
|
281966
282519
|
if (!skipInitializationCheck) {
|
|
@@ -282055,11 +282608,14 @@ class EnvironmentStore {
|
|
|
282055
282608
|
return await this.addEnvironment({
|
|
282056
282609
|
name: environmentName,
|
|
282057
282610
|
resource: `${API_PREFIX}/environments/${environmentName}`,
|
|
282058
|
-
connections: environmentConfig?.connections || []
|
|
282611
|
+
connections: environmentConfig?.connections || [],
|
|
282612
|
+
...environmentConfig?.storageDestinations?.length ? {
|
|
282613
|
+
storageDestinations: environmentConfig.storageDestinations
|
|
282614
|
+
} : {}
|
|
282059
282615
|
});
|
|
282060
282616
|
});
|
|
282061
282617
|
}
|
|
282062
|
-
async addEnvironment(environment, skipInitialization = false) {
|
|
282618
|
+
async addEnvironment(environment, skipInitialization = false, { storageDestinationsAreFallback = false } = {}) {
|
|
282063
282619
|
if (!skipInitialization) {
|
|
282064
282620
|
await this.finishedInitialization;
|
|
282065
282621
|
}
|
|
@@ -282075,6 +282631,9 @@ class EnvironmentStore {
|
|
|
282075
282631
|
if (existingEnvironment) {
|
|
282076
282632
|
const updatedEnvironment = await existingEnvironment.update(environment);
|
|
282077
282633
|
this.environments.set(environmentName, updatedEnvironment);
|
|
282634
|
+
if (storageDestinationsAreFallback) {
|
|
282635
|
+
updatedEnvironment.markStorageDestinationsUnknown();
|
|
282636
|
+
}
|
|
282078
282637
|
await this.addEnvironmentToDatabase(updatedEnvironment);
|
|
282079
282638
|
return updatedEnvironment;
|
|
282080
282639
|
}
|
|
@@ -282094,7 +282653,7 @@ class EnvironmentStore {
|
|
|
282094
282653
|
} else {
|
|
282095
282654
|
absoluteEnvironmentPath = await this.scaffoldEnvironment(environment);
|
|
282096
282655
|
}
|
|
282097
|
-
const newEnvironment = await Environment.create(environmentName, absoluteEnvironmentPath, environment.connections || []);
|
|
282656
|
+
const newEnvironment = await Environment.create(environmentName, absoluteEnvironmentPath, environment.connections || [], environment.storageDestinations || []);
|
|
282098
282657
|
newEnvironment.setMemoryGovernor(this.memoryGovernor);
|
|
282099
282658
|
if (!newEnvironment.metadata)
|
|
282100
282659
|
newEnvironment.metadata = {};
|
|
@@ -282109,6 +282668,9 @@ class EnvironmentStore {
|
|
|
282109
282668
|
newEnvironment.setPackageStatus(_package.name, "serving" /* SERVING */);
|
|
282110
282669
|
}
|
|
282111
282670
|
});
|
|
282671
|
+
if (storageDestinationsAreFallback) {
|
|
282672
|
+
newEnvironment.markStorageDestinationsUnknown();
|
|
282673
|
+
}
|
|
282112
282674
|
await this.addEnvironmentToDatabase(newEnvironment);
|
|
282113
282675
|
return newEnvironment;
|
|
282114
282676
|
}
|
|
@@ -282117,11 +282679,11 @@ class EnvironmentStore {
|
|
|
282117
282679
|
const startedAt = Date.now();
|
|
282118
282680
|
logger.info(`Detected zip file at "${absoluteEnvironmentPath}". Unzipping...`);
|
|
282119
282681
|
const unzippedEnvironmentPath = absoluteEnvironmentPath.replace(".zip", "");
|
|
282120
|
-
await
|
|
282682
|
+
await fs9.promises.rm(unzippedEnvironmentPath, {
|
|
282121
282683
|
recursive: true,
|
|
282122
282684
|
force: true
|
|
282123
282685
|
});
|
|
282124
|
-
await
|
|
282686
|
+
await fs9.promises.mkdir(unzippedEnvironmentPath, { recursive: true });
|
|
282125
282687
|
let entryCount = 0;
|
|
282126
282688
|
let totalUncompressedBytes = 0;
|
|
282127
282689
|
await import_extract_zip.default(absoluteEnvironmentPath, {
|
|
@@ -282171,7 +282733,7 @@ class EnvironmentStore {
|
|
|
282171
282733
|
await this.deleteEnvironmentFromDatabase(environmentName);
|
|
282172
282734
|
if (environmentPath) {
|
|
282173
282735
|
try {
|
|
282174
|
-
await
|
|
282736
|
+
await fs9.promises.rm(environmentPath, {
|
|
282175
282737
|
recursive: true,
|
|
282176
282738
|
force: true
|
|
282177
282739
|
});
|
|
@@ -282193,7 +282755,7 @@ class EnvironmentStore {
|
|
|
282193
282755
|
return { frozenConfig: false, environments: [] };
|
|
282194
282756
|
} else {
|
|
282195
282757
|
try {
|
|
282196
|
-
const entries = await
|
|
282758
|
+
const entries = await fs9.promises.readdir(serverRootPath, {
|
|
282197
282759
|
withFileTypes: true
|
|
282198
282760
|
});
|
|
282199
282761
|
const environments = [];
|
|
@@ -282207,7 +282769,8 @@ class EnvironmentStore {
|
|
|
282207
282769
|
location: `./${entry.name}`
|
|
282208
282770
|
}
|
|
282209
282771
|
],
|
|
282210
|
-
connections: []
|
|
282772
|
+
connections: [],
|
|
282773
|
+
storageDestinations: []
|
|
282211
282774
|
});
|
|
282212
282775
|
}
|
|
282213
282776
|
}
|
|
@@ -282225,9 +282788,9 @@ class EnvironmentStore {
|
|
|
282225
282788
|
assertSafePackageName(environment.name);
|
|
282226
282789
|
const environmentName = environment.name;
|
|
282227
282790
|
const absoluteEnvironmentPath = safeJoinUnderRoot(this.serverRootPath, PUBLISHER_DATA_DIR, environmentName);
|
|
282228
|
-
await
|
|
282791
|
+
await fs9.promises.mkdir(absoluteEnvironmentPath, { recursive: true });
|
|
282229
282792
|
if (environment.readme) {
|
|
282230
|
-
await
|
|
282793
|
+
await fs9.promises.writeFile(safeJoinUnderRoot(absoluteEnvironmentPath, "README.md"), environment.readme);
|
|
282231
282794
|
}
|
|
282232
282795
|
return absoluteEnvironmentPath;
|
|
282233
282796
|
}
|
|
@@ -282257,7 +282820,7 @@ class EnvironmentStore {
|
|
|
282257
282820
|
assertSafePackageName(environmentName);
|
|
282258
282821
|
const absoluteTargetPath = safeJoinUnderRoot(this.serverRootPath, PUBLISHER_DATA_DIR, environmentName);
|
|
282259
282822
|
const mountErrors = new Map;
|
|
282260
|
-
await
|
|
282823
|
+
await fs9.promises.mkdir(absoluteTargetPath, { recursive: true });
|
|
282261
282824
|
if (!packages || packages.length === 0) {
|
|
282262
282825
|
throw new PackageNotFoundError(`No packages found for environment ${environmentName}`);
|
|
282263
282826
|
}
|
|
@@ -282287,9 +282850,9 @@ class EnvironmentStore {
|
|
|
282287
282850
|
const totalPackages = packages.length;
|
|
282288
282851
|
let mountedCount = 0;
|
|
282289
282852
|
for (const [groupedLocation, packagesForLocation] of locationGroups) {
|
|
282290
|
-
const locationHash =
|
|
282853
|
+
const locationHash = crypto5.createHash("sha256").update(groupedLocation).digest("hex").substring(0, 16);
|
|
282291
282854
|
const tempDownloadPath = safeJoinUnderRoot(absoluteTargetPath, `.temp_${locationHash}`);
|
|
282292
|
-
await
|
|
282855
|
+
await fs9.promises.mkdir(tempDownloadPath, { recursive: true });
|
|
282293
282856
|
logger.info(`Created temporary directory: ${tempDownloadPath}`);
|
|
282294
282857
|
let downloaded = true;
|
|
282295
282858
|
try {
|
|
@@ -282324,7 +282887,7 @@ class EnvironmentStore {
|
|
|
282324
282887
|
sourcePath = safeJoinUnderRoot(tempDownloadPath, groupedLocation);
|
|
282325
282888
|
}
|
|
282326
282889
|
}
|
|
282327
|
-
const sourceExists = await
|
|
282890
|
+
const sourceExists = await fs9.promises.access(sourcePath).then(() => true).catch(() => false);
|
|
282328
282891
|
if (sourceExists) {
|
|
282329
282892
|
const isInPlace = this.inPlaceEnvs.has(environmentName) && this.isLocalPath(_package.location);
|
|
282330
282893
|
if (isInPlace) {
|
|
@@ -282332,16 +282895,16 @@ class EnvironmentStore {
|
|
|
282332
282895
|
const absoluteSourcePath = path9.resolve(sourcePath);
|
|
282333
282896
|
const linkType = process.platform === "win32" ? "junction" : "dir";
|
|
282334
282897
|
try {
|
|
282335
|
-
await
|
|
282898
|
+
await fs9.promises.symlink(absoluteSourcePath, absolutePackagePath, linkType);
|
|
282336
282899
|
logger.info(`In-place mount (watch mode): linked package "${packageDir}" -> "${absoluteSourcePath}" (${++mountedCount}/${totalPackages})`);
|
|
282337
282900
|
} catch (linkError) {
|
|
282338
282901
|
const code = linkError?.code ?? String(linkError);
|
|
282339
282902
|
logger.warn(`In-place mount failed for package "${packageDir}" (${code}); falling back to a copy. Source-edit live reload is disabled for this package.`);
|
|
282340
282903
|
await clearMountTarget(absolutePackagePath);
|
|
282341
|
-
await
|
|
282904
|
+
await fs9.promises.mkdir(absolutePackagePath, {
|
|
282342
282905
|
recursive: true
|
|
282343
282906
|
});
|
|
282344
|
-
await
|
|
282907
|
+
await fs9.promises.cp(sourcePath, absolutePackagePath, {
|
|
282345
282908
|
recursive: true
|
|
282346
282909
|
});
|
|
282347
282910
|
logger.info(`Copied package "${packageDir}" (${++mountedCount}/${totalPackages})`);
|
|
@@ -282351,20 +282914,20 @@ class EnvironmentStore {
|
|
|
282351
282914
|
logger.warn(`Watch mode: package "${packageDir}" has remote location "${_package.location}" — falling back to copy. Source-edit live reload won't work for this package; clone the source locally and use a local-dir location to enable it.`);
|
|
282352
282915
|
}
|
|
282353
282916
|
await clearMountTarget(absolutePackagePath);
|
|
282354
|
-
await
|
|
282917
|
+
await fs9.promises.mkdir(absolutePackagePath, {
|
|
282355
282918
|
recursive: true
|
|
282356
282919
|
});
|
|
282357
|
-
await
|
|
282920
|
+
await fs9.promises.cp(sourcePath, absolutePackagePath, {
|
|
282358
282921
|
recursive: true
|
|
282359
282922
|
});
|
|
282360
282923
|
logger.info(`Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"} (${++mountedCount}/${totalPackages})`);
|
|
282361
282924
|
}
|
|
282362
282925
|
} else {
|
|
282363
282926
|
await clearMountTarget(absolutePackagePath);
|
|
282364
|
-
await
|
|
282927
|
+
await fs9.promises.mkdir(absolutePackagePath, {
|
|
282365
282928
|
recursive: true
|
|
282366
282929
|
});
|
|
282367
|
-
await
|
|
282930
|
+
await fs9.promises.cp(tempDownloadPath, absolutePackagePath, {
|
|
282368
282931
|
recursive: true
|
|
282369
282932
|
});
|
|
282370
282933
|
logger.info(`Copied entire download as package "${packageDir}" (${++mountedCount}/${totalPackages})`);
|
|
@@ -282377,7 +282940,7 @@ class EnvironmentStore {
|
|
|
282377
282940
|
}
|
|
282378
282941
|
}
|
|
282379
282942
|
try {
|
|
282380
|
-
await
|
|
282943
|
+
await fs9.promises.rm(tempDownloadPath, {
|
|
282381
282944
|
recursive: true,
|
|
282382
282945
|
force: true
|
|
282383
282946
|
});
|
|
@@ -282448,14 +283011,14 @@ class EnvironmentStore {
|
|
|
282448
283011
|
if (environmentPath.endsWith(".zip")) {
|
|
282449
283012
|
environmentPath = await this.unzipEnvironment(environmentPath);
|
|
282450
283013
|
}
|
|
282451
|
-
const environmentDirExists = (await
|
|
283014
|
+
const environmentDirExists = (await fs9.promises.stat(environmentPath))?.isDirectory() ?? false;
|
|
282452
283015
|
if (environmentDirExists) {
|
|
282453
|
-
await
|
|
283016
|
+
await fs9.promises.rm(absoluteTargetPath, {
|
|
282454
283017
|
recursive: true,
|
|
282455
283018
|
force: true
|
|
282456
283019
|
});
|
|
282457
|
-
await
|
|
282458
|
-
await
|
|
283020
|
+
await fs9.promises.mkdir(absoluteTargetPath, { recursive: true });
|
|
283021
|
+
await fs9.promises.cp(environmentPath, absoluteTargetPath, {
|
|
282459
283022
|
recursive: true
|
|
282460
283023
|
});
|
|
282461
283024
|
} else {
|
|
@@ -282474,11 +283037,11 @@ class EnvironmentStore {
|
|
|
282474
283037
|
throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${gcsPath}`);
|
|
282475
283038
|
}
|
|
282476
283039
|
if (!isCompressedFile) {
|
|
282477
|
-
await
|
|
283040
|
+
await fs9.promises.rm(absoluteDirPath, {
|
|
282478
283041
|
recursive: true,
|
|
282479
283042
|
force: true
|
|
282480
283043
|
});
|
|
282481
|
-
await
|
|
283044
|
+
await fs9.promises.mkdir(absoluteDirPath, { recursive: true });
|
|
282482
283045
|
} else {
|
|
282483
283046
|
absoluteDirPath = `${absoluteDirPath}.zip`;
|
|
282484
283047
|
}
|
|
@@ -282488,10 +283051,10 @@ class EnvironmentStore {
|
|
|
282488
283051
|
if (file.name.endsWith("/")) {
|
|
282489
283052
|
return;
|
|
282490
283053
|
}
|
|
282491
|
-
await
|
|
283054
|
+
await fs9.promises.mkdir(path9.dirname(absoluteFilePath), {
|
|
282492
283055
|
recursive: true
|
|
282493
283056
|
});
|
|
282494
|
-
return
|
|
283057
|
+
return fs9.promises.writeFile(absoluteFilePath, await file.download());
|
|
282495
283058
|
}));
|
|
282496
283059
|
if (isCompressedFile) {
|
|
282497
283060
|
await this.unzipEnvironment(absoluteDirPath);
|
|
@@ -282505,7 +283068,7 @@ class EnvironmentStore {
|
|
|
282505
283068
|
const prefix = prefixParts.join("/");
|
|
282506
283069
|
if (isCompressedFile) {
|
|
282507
283070
|
const zipFilePath = `${absoluteDirPath}.zip`;
|
|
282508
|
-
await
|
|
283071
|
+
await fs9.promises.mkdir(path9.dirname(zipFilePath), {
|
|
282509
283072
|
recursive: true
|
|
282510
283073
|
});
|
|
282511
283074
|
const command = new import_client_s33.GetObjectCommand({
|
|
@@ -282516,7 +283079,7 @@ class EnvironmentStore {
|
|
|
282516
283079
|
if (!item.Body) {
|
|
282517
283080
|
throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${s3Path}`);
|
|
282518
283081
|
}
|
|
282519
|
-
const file =
|
|
283082
|
+
const file = fs9.createWriteStream(zipFilePath);
|
|
282520
283083
|
item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
|
|
282521
283084
|
await new Promise((resolve5, reject) => {
|
|
282522
283085
|
file.on("error", reject);
|
|
@@ -282530,8 +283093,8 @@ class EnvironmentStore {
|
|
|
282530
283093
|
Bucket: bucketName,
|
|
282531
283094
|
Prefix: prefix
|
|
282532
283095
|
});
|
|
282533
|
-
await
|
|
282534
|
-
await
|
|
283096
|
+
await fs9.promises.rm(absoluteDirPath, { recursive: true, force: true });
|
|
283097
|
+
await fs9.promises.mkdir(absoluteDirPath, { recursive: true });
|
|
282535
283098
|
if (!objects.Contents || objects.Contents.length === 0) {
|
|
282536
283099
|
throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${s3Path}`);
|
|
282537
283100
|
}
|
|
@@ -282545,7 +283108,7 @@ class EnvironmentStore {
|
|
|
282545
283108
|
return;
|
|
282546
283109
|
}
|
|
282547
283110
|
const absoluteFilePath = safeJoinUnderRoot(absoluteDirPath, relativeFilePath);
|
|
282548
|
-
await
|
|
283111
|
+
await fs9.promises.mkdir(path9.dirname(absoluteFilePath), {
|
|
282549
283112
|
recursive: true
|
|
282550
283113
|
});
|
|
282551
283114
|
const command = new import_client_s33.GetObjectCommand({
|
|
@@ -282556,7 +283119,7 @@ class EnvironmentStore {
|
|
|
282556
283119
|
if (!item.Body) {
|
|
282557
283120
|
return;
|
|
282558
283121
|
}
|
|
282559
|
-
const file =
|
|
283122
|
+
const file = fs9.createWriteStream(absoluteFilePath);
|
|
282560
283123
|
item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
|
|
282561
283124
|
await new Promise((resolve5, reject) => {
|
|
282562
283125
|
file.on("error", reject);
|
|
@@ -282573,11 +283136,11 @@ class EnvironmentStore {
|
|
|
282573
283136
|
}
|
|
282574
283137
|
const { owner, repoName, packagePath } = githubInfo;
|
|
282575
283138
|
const cleanPackagePath = (packagePath?.replace("/tree/main", "") || "").replace(/^\/+/, "");
|
|
282576
|
-
await
|
|
283139
|
+
await fs9.promises.rm(absoluteDirPath, {
|
|
282577
283140
|
recursive: true,
|
|
282578
283141
|
force: true
|
|
282579
283142
|
});
|
|
282580
|
-
await
|
|
283143
|
+
await fs9.promises.mkdir(absoluteDirPath, { recursive: true });
|
|
282581
283144
|
const repoUrl = `https://github.com/${owner}/${repoName}`;
|
|
282582
283145
|
const reporter = new CloneProgressReporter(cloneProgressLabel(`${owner}/${repoName}`, progressContext));
|
|
282583
283146
|
await new Promise((resolve5, reject) => {
|
|
@@ -282603,24 +283166,24 @@ class EnvironmentStore {
|
|
|
282603
283166
|
return;
|
|
282604
283167
|
}
|
|
282605
283168
|
const packageFullPath = safeJoinUnderRoot(absoluteDirPath, cleanPackagePath);
|
|
282606
|
-
const packageExists = await
|
|
283169
|
+
const packageExists = await fs9.promises.access(packageFullPath).then(() => true).catch(() => false);
|
|
282607
283170
|
if (!packageExists) {
|
|
282608
283171
|
throw new Error(`Package path "${cleanPackagePath}" does not exist in the cloned repository.`);
|
|
282609
283172
|
}
|
|
282610
|
-
const dirContents = await
|
|
283173
|
+
const dirContents = await fs9.promises.readdir(absoluteDirPath);
|
|
282611
283174
|
for (const entry of dirContents) {
|
|
282612
283175
|
if (entry !== cleanPackagePath.replace(/^\/+/, "").split("/")[0]) {
|
|
282613
|
-
await
|
|
283176
|
+
await fs9.promises.rm(safeJoinUnderRoot(absoluteDirPath, entry), {
|
|
282614
283177
|
recursive: true,
|
|
282615
283178
|
force: true
|
|
282616
283179
|
});
|
|
282617
283180
|
}
|
|
282618
283181
|
}
|
|
282619
|
-
const packageContents = await
|
|
283182
|
+
const packageContents = await fs9.promises.readdir(packageFullPath);
|
|
282620
283183
|
for (const entry of packageContents) {
|
|
282621
|
-
await
|
|
283184
|
+
await fs9.promises.rename(safeJoinUnderRoot(packageFullPath, entry), safeJoinUnderRoot(absoluteDirPath, entry));
|
|
282622
283185
|
}
|
|
282623
|
-
await
|
|
283186
|
+
await fs9.promises.rm(packageFullPath, { recursive: true, force: true });
|
|
282624
283187
|
}
|
|
282625
283188
|
extractErrorDataFromError(error) {
|
|
282626
283189
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
@@ -285613,6 +286176,15 @@ function classifyToolError(operation, identifier, error) {
|
|
|
285613
286176
|
suggestions: [...BACK_PRESSURE_SUGGESTIONS]
|
|
285614
286177
|
};
|
|
285615
286178
|
}
|
|
286179
|
+
if (error instanceof ResponseUnserializableError) {
|
|
286180
|
+
return {
|
|
286181
|
+
message: error.message,
|
|
286182
|
+
suggestions: [
|
|
286183
|
+
"This is not transient. The same query will fail the same way, so change the query rather than retrying it.",
|
|
286184
|
+
"Raising the byte cap will not help, because the response cannot be serialized at any cap. Shrink it instead: project fewer columns, add a LIMIT, or filter out the wide values."
|
|
286185
|
+
]
|
|
286186
|
+
};
|
|
286187
|
+
}
|
|
285616
286188
|
if (error instanceof QueryTimeoutError || error instanceof PayloadTooLargeError) {
|
|
285617
286189
|
return {
|
|
285618
286190
|
message: error.message,
|
|
@@ -285739,7 +286311,7 @@ var COMPILE_DESCRIPTION = `Compile-check Malloy source against a model and retur
|
|
|
285739
286311
|
|
|
285740
286312
|
## Parameters
|
|
285741
286313
|
- environmentName, packageName, modelPath (required): the model whose namespace the source compiles against. The source is appended to that model, so its imports, sources, and queries are in scope, and modelPath is real context, not a label.
|
|
285742
|
-
- source (required): the Malloy text to validate.
|
|
286314
|
+
- source (required): the Malloy text to validate. An \`#(authorize)\` annotation in it is rejected with a 400 — gates come only from package files; save and reload to validate one.
|
|
285743
286315
|
- includeSql (optional): also return the generated SQL when the source ends in a runnable query. The query is still not executed and no data is scanned.
|
|
285744
286316
|
|
|
285745
286317
|
## Checking part of a source
|
|
@@ -285982,31 +286554,31 @@ var executeQueryShape = {
|
|
|
285982
286554
|
packageName: exports_external.string().describe("Package containing the model. Call malloy_getContext with just environmentName to list its packages."),
|
|
285983
286555
|
modelPath: exports_external.string().describe("Path to the .malloy model file"),
|
|
285984
286556
|
query: exports_external.string().optional().describe("Ad-hoc Malloy query code"),
|
|
285985
|
-
sourceName: exports_external.string().optional().describe("Source name for a view"),
|
|
285986
|
-
queryName: exports_external.string().optional().describe("Named query or view"),
|
|
286557
|
+
sourceName: exports_external.string().optional().describe("Source name for a view. A NAME, not Malloy code: one name exactly as malloy_getContext returned it, sent bare (the server quotes it, so a hyphen or a reserved word is fine — do not add backticks yourself). Anything richer, such as a parameterized source or an inline extension, goes in query."),
|
|
286558
|
+
queryName: exports_external.string().optional().describe("Named query or view. A NAME, not Malloy code, on the same terms as sourceName: one view name as malloy_getContext returned it. A dotted path (carriers.by_name), a refinement (by_carrier + { limit: 10 }), or anything containing a newline goes in query instead."),
|
|
285987
286559
|
filterParams: exports_external.record(exports_external.union([exports_external.string(), exports_external.array(exports_external.string())])).optional().describe("Filter parameter values keyed by filter name. Used with sources that declare #(filter) annotations."),
|
|
285988
286560
|
givens: exports_external.record(exports_external.unknown()).optional().describe("Per-query given values that override model defaults. Keys are given names declared in the model's given: block.")
|
|
285989
286561
|
};
|
|
285990
286562
|
var EXECUTE_QUERY_DESCRIPTION = `Run a Malloy query against a model and return the rows. Takes either ad-hoc Malloy in query, or a named view/query via queryName (with sourceName for a view).
|
|
285991
286563
|
|
|
285992
286564
|
## Contract rules
|
|
285993
|
-
- Check _limit_hit before reporting any total, count, or "top N". True means the server's default
|
|
286565
|
+
- Check _limit_hit before reporting any total, count, or "top N". True means the server's default cap cut the result off and more rows exist, so what came back is a partial set, not the answer.
|
|
285994
286566
|
- Never sum or count the returned rows to state a total when _limit_hit or _rows_truncated is set. Aggregate in the query instead.
|
|
285995
286567
|
- _returned_rows: 0 with _rows_truncated set means one row was too large to send, NOT that nothing matched. Do not report it as an empty result.
|
|
285996
|
-
- Use source, view, and field names exactly as malloy_getContext returned them.
|
|
286568
|
+
- Use source, view, and field names exactly as malloy_getContext returned them. sourceName/queryName take one NAME each, never Malloy code — they are quoted for you, so send even a hyphenated name bare, and put anything richer (a dotted path, a refinement, a second statement) in query.
|
|
285997
286569
|
|
|
285998
286570
|
## Response
|
|
285999
|
-
A JSON object, the same shape Credible's execute_query
|
|
286000
|
-
- rows: flat objects keyed by column name
|
|
286571
|
+
A JSON object, the same shape Credible's execute_query and an in-package data app receive:
|
|
286572
|
+
- rows: flat objects keyed by column name.
|
|
286001
286573
|
- _meta: the Malloy metadata flat rows drop (schema with field types and render tags, annotations, connection_name, query_timezone).
|
|
286002
286574
|
- _query_row_limit: the cap pushed into the SQL, from the query's own limit: or the server default.
|
|
286003
286575
|
- _limit_source: "query" when the cap came from the query's own limit:/top:, "server_default" otherwise.
|
|
286004
|
-
- _limit_hit: the row count equals that cap AND the cap was the server default.
|
|
286576
|
+
- _limit_hit: the row count equals that cap AND the cap was the server default, so a query carrying its own limit:/top: never sets it and exactly that many rows is a complete answer.
|
|
286005
286577
|
- _rows_truncated / _total_rows / _returned_rows: present only when the payload cap dropped rows.
|
|
286006
286578
|
- _query_id: this query's id in the warehouse's own query history. Present only where enabled.
|
|
286007
286579
|
- warning, renderLogErrors: present only when they apply.
|
|
286008
286580
|
|
|
286009
|
-
|
|
286581
|
+
Values above 2^53 are returned as JSON strings so their digits survive.`;
|
|
286010
286582
|
function registerExecuteQueryTool(mcpServer, environmentStore) {
|
|
286011
286583
|
mcpServer.tool("malloy_executeQuery", EXECUTE_QUERY_DESCRIPTION, executeQueryShape, async (params) => {
|
|
286012
286584
|
const {
|
|
@@ -288349,7 +288921,7 @@ source: orders is duckdb.table('orders.parquet') extend {
|
|
|
288349
288921
|
\`\`\`
|
|
288350
288922
|
|
|
288351
288923
|
- **Source-level** \`#(authorize) "<expr>"\` gates that one source. **File-level** \`##(authorize) "<expr>"\` applies to every source in the file. Multiple gates combine as an OR, access is granted if any one is true, so a permissive file-level gate is a **model-wide override**, not an added restriction.
|
|
288352
|
-
- **
|
|
288924
|
+
- **Entry point only: not joined, but inherited through \`extend\`.** The gate applies to the source a query enters through. A gate on a source reached only via \`join_*\` **never fires**, at any depth, so anything ungated that joins a locked base hands the base's rows to every caller. A source that \`extend\`s a locked base and declares no gate of its own **does** carry the base's gate; declaring its own replaces it. Pair a locked base (\`#(authorize) "false"\`) with curated extension sources, using access modifiers (\`include { public: …, private: * }\`), so an extension re-exposes only a curated column surface, and keep sensitive sources out of ungated joins.
|
|
288353
288925
|
- The expression may reference only givens and literals, never a column of the gated source; the check runs against a synthetic probe row, not your data.
|
|
288354
288926
|
|
|
288355
288927
|
> **Trust caveat.** Givens are **caller-asserted**, anyone who can reach the query API can claim a favorable given, e.g. \`{"ROLE":"admin"}\`. \`#(authorize)\` is only a real boundary when it sits behind a trusted tier that sets givens from its own verified context, never directly from an untrusted caller. It is not, on its own, end-user authentication.
|
|
@@ -289705,7 +290277,7 @@ function initializeMcpServer(environmentStore) {
|
|
|
289705
290277
|
}
|
|
289706
290278
|
|
|
289707
290279
|
// src/mcp_config.ts
|
|
289708
|
-
import * as
|
|
290280
|
+
import * as fs10 from "fs";
|
|
289709
290281
|
import * as os3 from "os";
|
|
289710
290282
|
import * as path11 from "path";
|
|
289711
290283
|
init_logger();
|
|
@@ -289734,7 +290306,7 @@ function mcpEndpoint(host, port) {
|
|
|
289734
290306
|
function findGitWorkTreeRoot(dir) {
|
|
289735
290307
|
let current = path11.resolve(dir);
|
|
289736
290308
|
for (;; ) {
|
|
289737
|
-
if (
|
|
290309
|
+
if (fs10.existsSync(path11.join(current, ".git")))
|
|
289738
290310
|
return current;
|
|
289739
290311
|
const parent = path11.dirname(current);
|
|
289740
290312
|
if (parent === current)
|
|
@@ -289753,7 +290325,7 @@ function ensureMcpConfig(options) {
|
|
|
289753
290325
|
try {
|
|
289754
290326
|
return {
|
|
289755
290327
|
path: file,
|
|
289756
|
-
isSymlink:
|
|
290328
|
+
isSymlink: fs10.lstatSync(file).isSymbolicLink()
|
|
289757
290329
|
};
|
|
289758
290330
|
} catch {
|
|
289759
290331
|
return;
|
|
@@ -289772,7 +290344,7 @@ function ensureMcpConfig(options) {
|
|
|
289772
290344
|
}
|
|
289773
290345
|
const realish = (p) => {
|
|
289774
290346
|
try {
|
|
289775
|
-
return
|
|
290347
|
+
return fs10.realpathSync(p);
|
|
289776
290348
|
} catch {
|
|
289777
290349
|
return path11.resolve(p);
|
|
289778
290350
|
}
|
|
@@ -289790,7 +290362,7 @@ function ensureMcpConfig(options) {
|
|
|
289790
290362
|
action: "skipped-git",
|
|
289791
290363
|
dir,
|
|
289792
290364
|
gitRoot,
|
|
289793
|
-
rootConfig:
|
|
290365
|
+
rootConfig: fs10.existsSync(rootCandidate) ? rootCandidate : undefined,
|
|
289794
290366
|
endpoint,
|
|
289795
290367
|
staleConfig
|
|
289796
290368
|
};
|
|
@@ -289799,7 +290371,7 @@ function ensureMcpConfig(options) {
|
|
|
289799
290371
|
return { action: "exists", file, endpoint };
|
|
289800
290372
|
const body = JSON.stringify({ mcpServers: { malloy: malloyServer(endpoint) } }, null, 2) + `
|
|
289801
290373
|
`;
|
|
289802
|
-
|
|
290374
|
+
fs10.writeFileSync(file, body, { encoding: "utf8", flag: "wx" });
|
|
289803
290375
|
return { action: "created", file };
|
|
289804
290376
|
} catch (error) {
|
|
289805
290377
|
const code = error?.code;
|
|
@@ -289910,7 +290482,8 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
289910
290482
|
});
|
|
289911
290483
|
app.post(`${LEGACY_API_PREFIX}/projects`, async (req, res) => {
|
|
289912
290484
|
try {
|
|
289913
|
-
logger.info("Adding project", { body: req.body });
|
|
290485
|
+
logger.info("Adding project", { body: redactSensitive(req.body) });
|
|
290486
|
+
processStorageDestinationsOrThrow(req.body?.storageDestinations ?? []);
|
|
289914
290487
|
const environment = await environmentStore.addEnvironment(req.body);
|
|
289915
290488
|
res.status(200).json(await environment.serialize());
|
|
289916
290489
|
} catch (error) {
|
|
@@ -290523,7 +291096,7 @@ import {
|
|
|
290523
291096
|
InMemoryURLReader as InMemoryURLReader3,
|
|
290524
291097
|
Runtime as Runtime4
|
|
290525
291098
|
} from "@malloydata/malloy";
|
|
290526
|
-
import { mkdtempSync, rmSync } from "node:fs";
|
|
291099
|
+
import { mkdirSync as mkdirSync2, mkdtempSync, rmSync } from "node:fs";
|
|
290527
291100
|
import os4 from "node:os";
|
|
290528
291101
|
import path12 from "node:path";
|
|
290529
291102
|
var sharedGateSession;
|
|
@@ -290616,7 +291189,7 @@ async function buildSourceIntoStorage(params) {
|
|
|
290616
291189
|
const target = quoteManifestTablePath(`${destinationName}.${physicalTableName}`, "duckdb");
|
|
290617
291190
|
const passthrough = wrapPassthrough(sourceType, federated.handle, buildSQL);
|
|
290618
291191
|
const schema = await createTableAndDescribe(session, target, passthrough);
|
|
290619
|
-
return {
|
|
291192
|
+
return { storageDestinationName: destinationName, schema };
|
|
290620
291193
|
} finally {
|
|
290621
291194
|
await dispose();
|
|
290622
291195
|
}
|
|
@@ -290668,7 +291241,7 @@ async function buildDownstreamIntoStorage(params) {
|
|
|
290668
291241
|
const sql = projectToPublicColumns(downstream, downstream.getSQL({ virtualMap }));
|
|
290669
291242
|
const target = quoteManifestTablePath(`${destinationName}.${physicalTableName}`, "duckdb");
|
|
290670
291243
|
const schema = await createTableAndDescribe(session, target, sql);
|
|
290671
|
-
return {
|
|
291244
|
+
return { storageDestinationName: destinationName, schema };
|
|
290672
291245
|
} finally {
|
|
290673
291246
|
await dispose();
|
|
290674
291247
|
}
|
|
@@ -290677,7 +291250,7 @@ async function assertStorageServeShapeCompiles(params) {
|
|
|
290677
291250
|
const { destinationName, sourceName, virtualHandle, physicalTableName } = params;
|
|
290678
291251
|
const binding = {
|
|
290679
291252
|
sourceName,
|
|
290680
|
-
|
|
291253
|
+
destinationName,
|
|
290681
291254
|
virtualHandle,
|
|
290682
291255
|
tablePath: `${destinationName}.${physicalTableName}`,
|
|
290683
291256
|
schema: params.schema.filter((c) => c.name && c.type).map((c) => ({ name: c.name, type: c.type }))
|
|
@@ -290713,7 +291286,9 @@ async function attachDestinationReadWrite(session, destinationName, destinationC
|
|
|
290713
291286
|
await attachDuckLakeReadWrite(session, destinationName, cfg);
|
|
290714
291287
|
return;
|
|
290715
291288
|
}
|
|
290716
|
-
const
|
|
291289
|
+
const destinationRoot = storageDestinationRoot(environmentPath);
|
|
291290
|
+
mkdirSync2(destinationRoot, { recursive: true });
|
|
291291
|
+
const dbPath = path12.join(destinationRoot, `${destinationName}.duckdb`);
|
|
290717
291292
|
await session.runSQL(`ATTACH '${escapeSQL(dbPath)}' AS ${quoteIdentifier(destinationName, "duckdb")}`);
|
|
290718
291293
|
}
|
|
290719
291294
|
function assertSupportedDestination(destinationName, destinationConnection) {
|
|
@@ -290802,7 +291377,7 @@ function manifestExcludingStorage(manifest, builtEntries) {
|
|
|
290802
291377
|
reduced.strict = manifest.strict;
|
|
290803
291378
|
const quoted = manifest.buildManifest.entries;
|
|
290804
291379
|
for (const [id, entry] of Object.entries(builtEntries)) {
|
|
290805
|
-
if (!entry.
|
|
291380
|
+
if (!entry.storageDestinationName && quoted[id]) {
|
|
290806
291381
|
reduced.update(id, { tableName: quoted[id].tableName });
|
|
290807
291382
|
}
|
|
290808
291383
|
}
|
|
@@ -291050,7 +291625,7 @@ class MaterializationService {
|
|
|
291050
291625
|
continue;
|
|
291051
291626
|
seen.add(sourceEntityId);
|
|
291052
291627
|
const prior = priorEntries[sourceEntityId];
|
|
291053
|
-
if (prior && prior.physicalTableName && (prior.
|
|
291628
|
+
if (prior && prior.physicalTableName && (prior.storageDestinationName ?? undefined) === destination) {
|
|
291054
291629
|
carried[sourceEntityId] = prior;
|
|
291055
291630
|
continue;
|
|
291056
291631
|
}
|
|
@@ -291159,7 +291734,7 @@ class MaterializationService {
|
|
|
291159
291734
|
async autoLoadManifest(environment, packageName, entries) {
|
|
291160
291735
|
const manifestEntries = {};
|
|
291161
291736
|
for (const [sourceEntityId, entry] of Object.entries(entries)) {
|
|
291162
|
-
if (entry.physicalTableName && !entry.
|
|
291737
|
+
if (entry.physicalTableName && !entry.storageDestinationName) {
|
|
291163
291738
|
manifestEntries[sourceEntityId] = {
|
|
291164
291739
|
tableName: entry.physicalTableName,
|
|
291165
291740
|
connectionName: entry.connectionName
|
|
@@ -291254,7 +291829,7 @@ class MaterializationService {
|
|
|
291254
291829
|
}
|
|
291255
291830
|
const entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest, environment, entries, buildMetadata);
|
|
291256
291831
|
entries[sourceEntityId] = entry;
|
|
291257
|
-
if (entry.
|
|
291832
|
+
if (entry.storageDestinationName)
|
|
291258
291833
|
builtThisRun.push(entry);
|
|
291259
291834
|
}
|
|
291260
291835
|
}
|
|
@@ -291266,6 +291841,14 @@ class MaterializationService {
|
|
|
291266
291841
|
}
|
|
291267
291842
|
return entries;
|
|
291268
291843
|
}
|
|
291844
|
+
destinationForCleanup(environment, destinationName) {
|
|
291845
|
+
try {
|
|
291846
|
+
return environment.getStorageDestination(destinationName);
|
|
291847
|
+
} catch (error) {
|
|
291848
|
+
logger.warn("Skipping a table drop: its storage destination is no longer configured", { destinationName, error: errMessage(error) });
|
|
291849
|
+
return;
|
|
291850
|
+
}
|
|
291851
|
+
}
|
|
291269
291852
|
async reclaimStorageTablesFromFailedRun(builtThisRun, environment, owner) {
|
|
291270
291853
|
if (builtThisRun.length === 0)
|
|
291271
291854
|
return;
|
|
@@ -291277,14 +291860,14 @@ class MaterializationService {
|
|
|
291277
291860
|
if (other.status !== "MANIFEST_FILE_READY")
|
|
291278
291861
|
continue;
|
|
291279
291862
|
for (const e of Object.values(other.manifest?.entries ?? {})) {
|
|
291280
|
-
const dest = e.
|
|
291863
|
+
const dest = e.storageDestinationName ?? e.connectionName;
|
|
291281
291864
|
if (dest && e.physicalTableName) {
|
|
291282
291865
|
stillReferenced.add(tableKey(dest, e.physicalTableName));
|
|
291283
291866
|
}
|
|
291284
291867
|
}
|
|
291285
291868
|
}
|
|
291286
291869
|
for (const entry of builtThisRun) {
|
|
291287
|
-
const dest = entry.
|
|
291870
|
+
const dest = entry.storageDestinationName;
|
|
291288
291871
|
const table = entry.physicalTableName;
|
|
291289
291872
|
if (!dest || !table)
|
|
291290
291873
|
continue;
|
|
@@ -291292,10 +291875,13 @@ class MaterializationService {
|
|
|
291292
291875
|
logger.info("Keeping a table from a failed run: a live manifest still serves it", { destinationName: dest, physicalTableName: table });
|
|
291293
291876
|
continue;
|
|
291294
291877
|
}
|
|
291878
|
+
const destinationConnection = this.destinationForCleanup(environment, dest);
|
|
291879
|
+
if (!destinationConnection)
|
|
291880
|
+
continue;
|
|
291295
291881
|
try {
|
|
291296
291882
|
await dropStorageTable({
|
|
291297
291883
|
destinationName: dest,
|
|
291298
|
-
destinationConnection
|
|
291884
|
+
destinationConnection,
|
|
291299
291885
|
physicalTableName: table,
|
|
291300
291886
|
environmentPath: environment.getEnvironmentPath()
|
|
291301
291887
|
});
|
|
@@ -291309,7 +291895,7 @@ class MaterializationService {
|
|
|
291309
291895
|
logger.warn("Failed to reclaim a table stranded by a failed build", {
|
|
291310
291896
|
destinationName: dest,
|
|
291311
291897
|
physicalTableName: table,
|
|
291312
|
-
error: redactConnectionSecrets(errMessage(dropErr),
|
|
291898
|
+
error: redactConnectionSecrets(errMessage(dropErr), destinationConnection)
|
|
291313
291899
|
});
|
|
291314
291900
|
}
|
|
291315
291901
|
}
|
|
@@ -291415,7 +292001,7 @@ class MaterializationService {
|
|
|
291415
292001
|
const physicalTableName = instruction.physicalTableName;
|
|
291416
292002
|
const destinationName = instruction.destination;
|
|
291417
292003
|
const sourceConnection = environment.getApiConnection(persistSource.connectionName);
|
|
291418
|
-
const destinationConnection = environment.
|
|
292004
|
+
const destinationConnection = environment.getStorageDestination(destinationName);
|
|
291419
292005
|
const startTime = performance.now();
|
|
291420
292006
|
let result;
|
|
291421
292007
|
if (dependsOnStorageUpstream) {
|
|
@@ -291494,7 +292080,7 @@ class MaterializationService {
|
|
|
291494
292080
|
recordSourceBuildDuration(durationMs, "storage");
|
|
291495
292081
|
logger.info(`Built materialized source ${persistSource.name} into storage`, {
|
|
291496
292082
|
physicalTableName,
|
|
291497
|
-
|
|
292083
|
+
storageDestinationName: result.storageDestinationName,
|
|
291498
292084
|
columns: result.schema.length,
|
|
291499
292085
|
durationMs
|
|
291500
292086
|
});
|
|
@@ -291504,14 +292090,14 @@ class MaterializationService {
|
|
|
291504
292090
|
materializedTableId: instruction.materializedTableId,
|
|
291505
292091
|
physicalTableName,
|
|
291506
292092
|
connectionName: persistSource.connectionName,
|
|
291507
|
-
|
|
292093
|
+
storageDestinationName: result.storageDestinationName,
|
|
291508
292094
|
schema: result.schema,
|
|
291509
292095
|
realization: instruction.realization,
|
|
291510
292096
|
rowCount: null
|
|
291511
292097
|
};
|
|
291512
292098
|
}
|
|
291513
292099
|
async buildDownstreamViaParents(persistSource, destinationName, destinationConnection, builtEntries, environment, physicalTableName) {
|
|
291514
|
-
const upstreams = deriveServeBindings(builtEntries).filter((b) => b.
|
|
292100
|
+
const upstreams = deriveServeBindings(builtEntries).filter((b) => b.destinationName === destinationName);
|
|
291515
292101
|
if (upstreams.length === 0) {
|
|
291516
292102
|
throw new MaterializationEligibilityError({
|
|
291517
292103
|
message: "no materialized upstream is available in this destination to build on"
|
|
@@ -291618,7 +292204,7 @@ class MaterializationService {
|
|
|
291618
292204
|
if (other.status !== "MANIFEST_FILE_READY")
|
|
291619
292205
|
continue;
|
|
291620
292206
|
for (const e of Object.values(other.manifest?.entries ?? {})) {
|
|
291621
|
-
const dest = e.
|
|
292207
|
+
const dest = e.storageDestinationName ?? e.connectionName;
|
|
291622
292208
|
if (dest && e.physicalTableName) {
|
|
291623
292209
|
stillReferenced.add(tableKey(dest, e.physicalTableName));
|
|
291624
292210
|
}
|
|
@@ -291634,7 +292220,7 @@ class MaterializationService {
|
|
|
291634
292220
|
});
|
|
291635
292221
|
continue;
|
|
291636
292222
|
}
|
|
291637
|
-
const destForKey = entry.
|
|
292223
|
+
const destForKey = entry.storageDestinationName ?? connectionName;
|
|
291638
292224
|
if (stillReferenced.has(tableKey(destForKey, physicalTableName))) {
|
|
291639
292225
|
logger.info("Skipping drop: table still referenced by another materialization", {
|
|
291640
292226
|
materializationId: m.id,
|
|
@@ -291643,11 +292229,13 @@ class MaterializationService {
|
|
|
291643
292229
|
});
|
|
291644
292230
|
continue;
|
|
291645
292231
|
}
|
|
291646
|
-
if (entry.
|
|
291647
|
-
const destinationConnection =
|
|
292232
|
+
if (entry.storageDestinationName) {
|
|
292233
|
+
const destinationConnection = this.destinationForCleanup(environment, entry.storageDestinationName);
|
|
292234
|
+
if (!destinationConnection)
|
|
292235
|
+
continue;
|
|
291648
292236
|
try {
|
|
291649
292237
|
await dropStorageTable({
|
|
291650
|
-
destinationName: entry.
|
|
292238
|
+
destinationName: entry.storageDestinationName,
|
|
291651
292239
|
destinationConnection,
|
|
291652
292240
|
physicalTableName,
|
|
291653
292241
|
environmentPath: environment.getEnvironmentPath()
|
|
@@ -291656,14 +292244,14 @@ class MaterializationService {
|
|
|
291656
292244
|
logger.info("Dropped materialized storage table on delete", {
|
|
291657
292245
|
materializationId: m.id,
|
|
291658
292246
|
physicalTableName,
|
|
291659
|
-
|
|
292247
|
+
storageDestinationName: entry.storageDestinationName
|
|
291660
292248
|
});
|
|
291661
292249
|
} catch (err) {
|
|
291662
292250
|
recordDropTables("failure", "storage");
|
|
291663
292251
|
logger.warn("Failed to drop a storage-materialized table", {
|
|
291664
292252
|
materializationId: m.id,
|
|
291665
292253
|
physicalTableName,
|
|
291666
|
-
|
|
292254
|
+
storageDestinationName: entry.storageDestinationName,
|
|
291667
292255
|
error: redactConnectionSecrets(errMessage(err), destinationConnection)
|
|
291668
292256
|
});
|
|
291669
292257
|
}
|
|
@@ -291961,7 +292549,82 @@ function listSanitizerDrops(raw, sanitized) {
|
|
|
291961
292549
|
return drops;
|
|
291962
292550
|
}
|
|
291963
292551
|
|
|
292552
|
+
// src/spa_fallback.ts
|
|
292553
|
+
var ASSET_EXTENSIONS = new Set([
|
|
292554
|
+
"avif",
|
|
292555
|
+
"css",
|
|
292556
|
+
"csv",
|
|
292557
|
+
"gif",
|
|
292558
|
+
"htm",
|
|
292559
|
+
"html",
|
|
292560
|
+
"ico",
|
|
292561
|
+
"jpeg",
|
|
292562
|
+
"jpg",
|
|
292563
|
+
"js",
|
|
292564
|
+
"json",
|
|
292565
|
+
"map",
|
|
292566
|
+
"mjs",
|
|
292567
|
+
"parquet",
|
|
292568
|
+
"pdf",
|
|
292569
|
+
"png",
|
|
292570
|
+
"svg",
|
|
292571
|
+
"ttf",
|
|
292572
|
+
"txt",
|
|
292573
|
+
"webp",
|
|
292574
|
+
"woff",
|
|
292575
|
+
"woff2",
|
|
292576
|
+
"xlsx"
|
|
292577
|
+
]);
|
|
292578
|
+
var SPA_OWNED_SEGMENTS = new Set(["pages", "workbook"]);
|
|
292579
|
+
function extensionOf(segment) {
|
|
292580
|
+
const dot = segment.lastIndexOf(".");
|
|
292581
|
+
if (dot <= 0 || dot === segment.length - 1)
|
|
292582
|
+
return "";
|
|
292583
|
+
return segment.slice(dot + 1).toLowerCase();
|
|
292584
|
+
}
|
|
292585
|
+
function classifySpaFallback(requestPath, apiPrefix) {
|
|
292586
|
+
if (requestPath === apiPrefix || requestPath.startsWith(`${apiPrefix}/`)) {
|
|
292587
|
+
return { kind: "apiNotFound", path: requestPath };
|
|
292588
|
+
}
|
|
292589
|
+
const segments = requestPath.split("/").filter((s) => s.length > 0);
|
|
292590
|
+
const last2 = segments[segments.length - 1] ?? "";
|
|
292591
|
+
if (!ASSET_EXTENSIONS.has(extensionOf(last2)))
|
|
292592
|
+
return { kind: "spa" };
|
|
292593
|
+
if (segments.length >= 3 && SPA_OWNED_SEGMENTS.has(segments[2])) {
|
|
292594
|
+
return { kind: "spa" };
|
|
292595
|
+
}
|
|
292596
|
+
if (segments.length >= 3) {
|
|
292597
|
+
const alreadyStaticForm = segments[0] === "environments" && segments[2] === "packages";
|
|
292598
|
+
const traverses = segments.some((segment) => {
|
|
292599
|
+
let decoded;
|
|
292600
|
+
try {
|
|
292601
|
+
decoded = decodeURIComponent(segment);
|
|
292602
|
+
} catch {
|
|
292603
|
+
return true;
|
|
292604
|
+
}
|
|
292605
|
+
return decoded === "." || decoded === ".." || decoded.includes("\\");
|
|
292606
|
+
});
|
|
292607
|
+
if (!alreadyStaticForm && !traverses) {
|
|
292608
|
+
const [environmentName, packageName, ...rest] = segments;
|
|
292609
|
+
if (rest[0] === "public" && rest.length > 1)
|
|
292610
|
+
rest.shift();
|
|
292611
|
+
return {
|
|
292612
|
+
kind: "redirect",
|
|
292613
|
+
location: `/environments/${environmentName}/packages/${packageName}/${rest.join("/")}`,
|
|
292614
|
+
environmentName,
|
|
292615
|
+
packageName
|
|
292616
|
+
};
|
|
292617
|
+
}
|
|
292618
|
+
}
|
|
292619
|
+
return {
|
|
292620
|
+
kind: "assetNotFound",
|
|
292621
|
+
path: requestPath,
|
|
292622
|
+
appRouteCandidate: segments.length === 1 ? { environmentName: segments[0] } : segments.length === 2 ? { environmentName: segments[0], packageName: segments[1] } : null
|
|
292623
|
+
};
|
|
292624
|
+
}
|
|
292625
|
+
|
|
291964
292626
|
// src/server.ts
|
|
292627
|
+
assertSupportedNodeVersion();
|
|
291965
292628
|
function parseArgs() {
|
|
291966
292629
|
const args = process.argv.slice(2);
|
|
291967
292630
|
let sawServerRoot = false;
|
|
@@ -292151,6 +292814,17 @@ app.get("/sdk/publisher.js", (_req, res) => {
|
|
|
292151
292814
|
}
|
|
292152
292815
|
});
|
|
292153
292816
|
});
|
|
292817
|
+
var PACKAGE_FILE_NOT_FOUND_HTML = `<!doctype html><meta charset="utf-8">
|
|
292818
|
+
<title>Not found</title>
|
|
292819
|
+
<style>body{font:14px/1.4 -apple-system,system-ui,sans-serif;margin:40px;max-width:720px;color:#222}code{background:#f4f4f5;padding:1px 4px;border-radius:3px}</style>
|
|
292820
|
+
<h1>Not found</h1>
|
|
292821
|
+
<p>This package does not serve that file. Only files inside the package's
|
|
292822
|
+
<code>public/</code> directory are web-served, at
|
|
292823
|
+
<code>/environments/<env>/packages/<pkg>/<file></code>, where <code><file></code> is
|
|
292824
|
+
relative to <code>public/</code> and does not include it.</p>
|
|
292825
|
+
<p>Models and notebooks are not served here; they open in the web UI at
|
|
292826
|
+
<code>/<env>/<pkg>/<file>.malloy</code>. <a href="/">Publisher home</a> lists what
|
|
292827
|
+
this server has.</p>`;
|
|
292154
292828
|
async function serveFromPackage(req, res) {
|
|
292155
292829
|
const subPathRaw = req.params["0"] ?? "";
|
|
292156
292830
|
try {
|
|
@@ -292170,7 +292844,7 @@ async function serveFromPackage(req, res) {
|
|
|
292170
292844
|
realFullPath = await fsp.realpath(fullPath);
|
|
292171
292845
|
} catch {
|
|
292172
292846
|
if (!res.headersSent) {
|
|
292173
|
-
res.status(404).
|
|
292847
|
+
res.status(404).type("text/html").send(PACKAGE_FILE_NOT_FOUND_HTML);
|
|
292174
292848
|
}
|
|
292175
292849
|
return;
|
|
292176
292850
|
}
|
|
@@ -292189,7 +292863,7 @@ async function serveFromPackage(req, res) {
|
|
|
292189
292863
|
res.sendFile(realFullPath, (err) => {
|
|
292190
292864
|
if (err) {
|
|
292191
292865
|
if (!res.headersSent) {
|
|
292192
|
-
res.status(404).
|
|
292866
|
+
res.status(404).type("text/html").send(PACKAGE_FILE_NOT_FOUND_HTML);
|
|
292193
292867
|
}
|
|
292194
292868
|
}
|
|
292195
292869
|
});
|
|
@@ -292200,21 +292874,17 @@ async function serveFromPackage(req, res) {
|
|
|
292200
292874
|
}
|
|
292201
292875
|
}
|
|
292202
292876
|
}
|
|
292877
|
+
function withRequestQuery(req, target) {
|
|
292878
|
+
const hash = req.originalUrl.indexOf("#");
|
|
292879
|
+
const target_ = hash === -1 ? req.originalUrl : req.originalUrl.slice(0, hash);
|
|
292880
|
+
const marker = target_.indexOf("?");
|
|
292881
|
+
return marker === -1 ? target : target + target_.slice(marker);
|
|
292882
|
+
}
|
|
292203
292883
|
app.get("/environments/:environmentName/packages/:packageName", (req, res, next) => {
|
|
292204
292884
|
if (req.path.endsWith("/"))
|
|
292205
292885
|
return next();
|
|
292206
292886
|
const canonical = `/environments/${encodeURIComponent(req.params.environmentName)}/packages/${encodeURIComponent(req.params.packageName)}/`;
|
|
292207
|
-
|
|
292208
|
-
for (const [key, value] of Object.entries(req.query)) {
|
|
292209
|
-
if (Array.isArray(value)) {
|
|
292210
|
-
for (const v of value)
|
|
292211
|
-
query.append(key, String(v));
|
|
292212
|
-
} else if (value !== undefined) {
|
|
292213
|
-
query.append(key, String(value));
|
|
292214
|
-
}
|
|
292215
|
-
}
|
|
292216
|
-
const qs = query.toString();
|
|
292217
|
-
res.redirect(308, qs ? `${canonical}?${qs}` : canonical);
|
|
292887
|
+
res.redirect(308, withRequestQuery(req, canonical));
|
|
292218
292888
|
});
|
|
292219
292889
|
app.get("/environments/:environmentName/packages/:packageName/*", serveFromPackage);
|
|
292220
292890
|
var PAGES_DEPTH_CAP = 3;
|
|
@@ -292229,11 +292899,11 @@ function stripNonTagText(input) {
|
|
|
292229
292899
|
return current;
|
|
292230
292900
|
}
|
|
292231
292901
|
async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
292232
|
-
const
|
|
292902
|
+
const fs11 = await import("fs/promises");
|
|
292233
292903
|
const out = [];
|
|
292234
292904
|
let realPublicRoot;
|
|
292235
292905
|
try {
|
|
292236
|
-
realPublicRoot = await
|
|
292906
|
+
realPublicRoot = await fs11.realpath(publicRoot);
|
|
292237
292907
|
} catch {
|
|
292238
292908
|
return out;
|
|
292239
292909
|
}
|
|
@@ -292242,7 +292912,7 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
|
292242
292912
|
return;
|
|
292243
292913
|
let entries;
|
|
292244
292914
|
try {
|
|
292245
|
-
entries = await
|
|
292915
|
+
entries = await fs11.readdir(dir, { withFileTypes: true });
|
|
292246
292916
|
} catch {
|
|
292247
292917
|
return;
|
|
292248
292918
|
}
|
|
@@ -292252,7 +292922,7 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
|
292252
292922
|
const full = path13.join(dir, entry.name);
|
|
292253
292923
|
let realFull;
|
|
292254
292924
|
try {
|
|
292255
|
-
realFull = await
|
|
292925
|
+
realFull = await fs11.realpath(full);
|
|
292256
292926
|
} catch {
|
|
292257
292927
|
continue;
|
|
292258
292928
|
}
|
|
@@ -292266,7 +292936,7 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
|
292266
292936
|
let title = rel;
|
|
292267
292937
|
let fit;
|
|
292268
292938
|
try {
|
|
292269
|
-
const fh = await
|
|
292939
|
+
const fh = await fs11.open(full, "r");
|
|
292270
292940
|
try {
|
|
292271
292941
|
const buf = Buffer.alloc(4096);
|
|
292272
292942
|
const { bytesRead } = await fh.read(buf, 0, 4096, 0);
|
|
@@ -292461,7 +293131,8 @@ app.get(`${API_PREFIX2}/environments`, async (_req, res) => {
|
|
|
292461
293131
|
});
|
|
292462
293132
|
app.post(`${API_PREFIX2}/environments`, async (req, res) => {
|
|
292463
293133
|
try {
|
|
292464
|
-
logger.info("Adding environment", { body: req.body });
|
|
293134
|
+
logger.info("Adding environment", { body: redactSensitive(req.body) });
|
|
293135
|
+
processStorageDestinationsOrThrow(req.body?.storageDestinations ?? []);
|
|
292465
293136
|
const environment = await environmentStore.addEnvironment(req.body);
|
|
292466
293137
|
res.status(200).json(await environment.serialize());
|
|
292467
293138
|
} catch (error) {
|
|
@@ -292954,7 +293625,62 @@ registerLegacyRoutes(app, {
|
|
|
292954
293625
|
});
|
|
292955
293626
|
if (!isDevelopment) {
|
|
292956
293627
|
const SPA_INDEX = path13.resolve(ROOT, "index.html");
|
|
293628
|
+
const escapeHtml = (value) => value.replace(/[<>&]/g, (c) => ({ "<": "<", ">": ">", "&": "&" })[c] ?? c);
|
|
293629
|
+
const decodeSegment = (segment) => {
|
|
293630
|
+
if (segment === undefined)
|
|
293631
|
+
return "";
|
|
293632
|
+
try {
|
|
293633
|
+
return decodeURIComponent(segment);
|
|
293634
|
+
} catch {
|
|
293635
|
+
return segment;
|
|
293636
|
+
}
|
|
293637
|
+
};
|
|
292957
293638
|
app.get("*", (req, res) => {
|
|
293639
|
+
let fallback = classifySpaFallback(req.path, API_PREFIX2);
|
|
293640
|
+
const loadedEnvironment = (name) => environmentStore.getLoadedEnvironments().find((environment) => environment.getEnvironmentName() === name);
|
|
293641
|
+
if (fallback.kind === "redirect") {
|
|
293642
|
+
const environment = loadedEnvironment(decodeSegment(fallback.environmentName));
|
|
293643
|
+
const packageName = decodeSegment(fallback.packageName);
|
|
293644
|
+
const known = environment?.getLoadedPackages().some((pkg) => pkg.getPackageName() === packageName);
|
|
293645
|
+
if (!known) {
|
|
293646
|
+
fallback = {
|
|
293647
|
+
kind: "assetNotFound",
|
|
293648
|
+
path: req.path,
|
|
293649
|
+
appRouteCandidate: null
|
|
293650
|
+
};
|
|
293651
|
+
}
|
|
293652
|
+
}
|
|
293653
|
+
if (fallback.kind === "assetNotFound" && fallback.appRouteCandidate) {
|
|
293654
|
+
const { environmentName, packageName } = fallback.appRouteCandidate;
|
|
293655
|
+
const environment = loadedEnvironment(decodeSegment(environmentName));
|
|
293656
|
+
const isAppRoute = environment !== undefined && (packageName === undefined || environment.getLoadedPackages().some((pkg) => pkg.getPackageName() === decodeSegment(packageName)));
|
|
293657
|
+
if (isAppRoute)
|
|
293658
|
+
fallback = { kind: "spa" };
|
|
293659
|
+
}
|
|
293660
|
+
if (fallback.kind === "redirect") {
|
|
293661
|
+
res.redirect(302, withRequestQuery(req, fallback.location));
|
|
293662
|
+
return;
|
|
293663
|
+
}
|
|
293664
|
+
if (fallback.kind === "apiNotFound") {
|
|
293665
|
+
res.status(404).json({
|
|
293666
|
+
code: 404,
|
|
293667
|
+
message: `Unknown API endpoint: ${fallback.path}. See /api-doc.yaml for the endpoints this server serves.`
|
|
293668
|
+
});
|
|
293669
|
+
return;
|
|
293670
|
+
}
|
|
293671
|
+
if (fallback.kind === "assetNotFound") {
|
|
293672
|
+
res.status(404).type("text/html").send(`<!doctype html><meta charset="utf-8">
|
|
293673
|
+
<title>Not found</title>
|
|
293674
|
+
<style>body{font:14px/1.4 -apple-system,system-ui,sans-serif;margin:40px;max-width:720px;color:#222}code{background:#f4f4f5;padding:1px 4px;border-radius:3px}</style>
|
|
293675
|
+
<h1>Not found</h1>
|
|
293676
|
+
<p>Nothing is served at <code>${escapeHtml(fallback.path)}</code>.</p>
|
|
293677
|
+
<p>A file inside a package is served from that package's <code>public/</code> directory at
|
|
293678
|
+
<code>/environments/<env>/packages/<pkg>/<file></code>, where <code><file></code> is relative to
|
|
293679
|
+
<code>public/</code> and does not include it. Models and notebooks open in the web UI at
|
|
293680
|
+
<code>/<env>/<pkg>/<file>.malloy</code> and <code>.malloynb</code>.</p>
|
|
293681
|
+
<p><a href="/">Publisher home</a> lists the environments and packages this server has.</p>`);
|
|
293682
|
+
return;
|
|
293683
|
+
}
|
|
292958
293684
|
res.sendFile(SPA_INDEX, (err) => {
|
|
292959
293685
|
if (!err)
|
|
292960
293686
|
return;
|