@malloy-publisher/server 0.0.235 → 0.0.237
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +83 -27
- package/dist/app/assets/{EnvironmentPage-BsAnavYN.js → EnvironmentPage-DaP6CpgK.js} +1 -1
- package/dist/app/assets/{HomePage-CADE138j.js → HomePage-C9yNw3iu.js} +1 -1
- package/dist/app/assets/{LightMode-Cfh7KzN8.js → LightMode-Be4-G-dX.js} +1 -1
- package/dist/app/assets/{MainPage-CO3pRlnV.js → MainPage-J_rU2SgI.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-p9YjkRXZ.js → MaterializationsPage-DHgPI4MO.js} +1 -1
- package/dist/app/assets/ModelPage-BzXfjTu5.js +1 -0
- package/dist/app/assets/{PackagePage-e4kN75YR.js → PackagePage-ByoSTLJ4.js} +1 -1
- package/dist/app/assets/RouteError-CrSeKN2r.js +1 -0
- package/dist/app/assets/{ThemeEditorPage-CciagFTq.js → ThemeEditorPage-Btxnmhtk.js} +1 -1
- package/dist/app/assets/{WorkbookPage-DNWmkCXa.js → WorkbookPage-D6JLPnWO.js} +1 -1
- package/dist/app/assets/{core-Rj_4rRnA.es-BZyvITuO.js → core-BSRgt1mG.es-Bc_D2Lq4.js} +1 -1
- package/dist/app/assets/{index-CH2AcDzc.js → index-BRijWj6K.js} +15 -15
- package/dist/app/assets/{index-DQa463gC.js → index-BX4D94xw.js} +1 -1
- package/dist/app/assets/{index-VBbcc8s6.js → index-Cvy0xtb7.js} +1 -1
- package/dist/app/assets/{index-DQpV7MyA.js → index-DMAhIJNC.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/package_load_worker.mjs +59 -20
- package/dist/server.mjs +1342 -616
- package/package.json +8 -2
- package/.eslintrc.json +0 -44
- package/.prettierignore +0 -1
- package/.prettierrc +0 -4
- package/build.ts +0 -97
- package/dist/app/assets/ModelPage-C1OSTv-x.js +0 -1
- package/dist/app/assets/RouteError-CzbfOkng.js +0 -1
- package/eslint.config.mjs +0 -76
- package/k6-tests/README.md +0 -144
- package/k6-tests/bun.lock +0 -797
- package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
- package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
- package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
- package/k6-tests/load-test/load-test-crud.ts +0 -142
- package/k6-tests/load-test/load-test.ts +0 -784
- package/k6-tests/package.json +0 -27
- package/k6-tests/scripts/run-with-otel.sh +0 -50
- package/k6-tests/smoke-test/smoke-test.ts +0 -158
- package/k6-tests/tsconfig.json +0 -29
- package/k6-tests/utils/client_factory.ts +0 -128
- package/k6-tests/utils/common.ts +0 -489
- package/k6-tests/utils/logger.ts +0 -62
- package/k6-tests/utils/types.d.ts +0 -35
- package/malloy_mcp.dxt +0 -0
- package/publisher.config.example.bigquery.json +0 -33
- package/publisher.config.example.duckdb.json +0 -23
- package/publisher.config.json +0 -23
- package/scripts/bake-duckdb-extensions.js +0 -110
- package/src/config.spec.ts +0 -1535
- package/src/config.theme.spec.ts +0 -178
- package/src/config.ts +0 -1142
- package/src/constants.ts +0 -105
- package/src/controller/compile.controller.ts +0 -40
- package/src/controller/connection.controller.spec.ts +0 -849
- package/src/controller/connection.controller.ts +0 -901
- package/src/controller/database.controller.ts +0 -24
- package/src/controller/materialization.controller.spec.ts +0 -293
- package/src/controller/materialization.controller.ts +0 -319
- package/src/controller/model.controller.ts +0 -164
- package/src/controller/package.controller.spec.ts +0 -273
- package/src/controller/package.controller.ts +0 -306
- package/src/controller/query.controller.ts +0 -152
- package/src/controller/theme.controller.ts +0 -83
- package/src/controller/watch-mode.controller.ts +0 -243
- package/src/data_styles.ts +0 -71
- package/src/default-publisher.config.json +0 -23
- package/src/ducklake_version.spec.ts +0 -163
- package/src/ducklake_version.ts +0 -153
- package/src/errors.spec.ts +0 -119
- package/src/errors.ts +0 -237
- package/src/filter_deprecation.spec.ts +0 -64
- package/src/filter_deprecation.ts +0 -42
- package/src/health.spec.ts +0 -123
- package/src/health.ts +0 -233
- package/src/heap_check.spec.ts +0 -144
- package/src/heap_check.ts +0 -144
- package/src/instrumentation.ts +0 -214
- package/src/json_utils.spec.ts +0 -51
- package/src/json_utils.ts +0 -33
- package/src/logger.spec.ts +0 -210
- package/src/logger.ts +0 -251
- package/src/malloy_pin_prereqs.spec.ts +0 -25
- package/src/materialization_metrics.spec.ts +0 -169
- package/src/materialization_metrics.ts +0 -346
- package/src/mcp/error_messages.spec.ts +0 -35
- package/src/mcp/error_messages.ts +0 -183
- package/src/mcp/handler_utils.spec.ts +0 -108
- package/src/mcp/handler_utils.ts +0 -251
- package/src/mcp/mcp_constants.ts +0 -6
- package/src/mcp/query_envelope.spec.ts +0 -229
- package/src/mcp/query_envelope.ts +0 -240
- package/src/mcp/server.protocol.spec.ts +0 -250
- package/src/mcp/server.ts +0 -87
- package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
- package/src/mcp/skills/build_skills_bundle.ts +0 -176
- package/src/mcp/skills/skills_bundle.json +0 -1
- package/src/mcp/skills/skills_bundle.spec.ts +0 -171
- package/src/mcp/tool_response.spec.ts +0 -108
- package/src/mcp/tool_response.ts +0 -138
- package/src/mcp/tools/compile_tool.spec.ts +0 -315
- package/src/mcp/tools/compile_tool.ts +0 -208
- package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
- package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
- package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
- package/src/mcp/tools/docs_search_tool.ts +0 -128
- package/src/mcp/tools/embedding_index.spec.ts +0 -1236
- package/src/mcp/tools/embedding_index.ts +0 -808
- package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
- package/src/mcp/tools/execute_query_tool.ts +0 -316
- package/src/mcp/tools/get_context_eval.ts +0 -275
- package/src/mcp/tools/get_context_tool.spec.ts +0 -592
- package/src/mcp/tools/get_context_tool.ts +0 -585
- package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
- package/src/mcp/tools/reload_package_tool.ts +0 -132
- package/src/mcp_config.spec.ts +0 -919
- package/src/mcp_config.ts +0 -425
- package/src/oom_guards.integration.spec.ts +0 -269
- package/src/package_load/package_load_pool.spec.ts +0 -252
- package/src/package_load/package_load_pool.ts +0 -931
- package/src/package_load/package_load_worker.ts +0 -1034
- package/src/package_load/protocol.ts +0 -364
- package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
- package/src/package_load/rpc_wait_accountant.ts +0 -76
- package/src/package_load_metrics.spec.ts +0 -114
- package/src/package_load_metrics.ts +0 -127
- package/src/path_safety.spec.ts +0 -158
- package/src/path_safety.ts +0 -146
- package/src/pg_helpers.spec.ts +0 -223
- package/src/pg_helpers.ts +0 -52
- package/src/query_cap_metrics.spec.ts +0 -89
- package/src/query_cap_metrics.ts +0 -116
- package/src/query_concurrency.spec.ts +0 -247
- package/src/query_concurrency.ts +0 -237
- package/src/query_metadata_metrics.ts +0 -49
- package/src/query_param_utils.ts +0 -29
- package/src/query_timeout.spec.ts +0 -224
- package/src/query_timeout.ts +0 -179
- package/src/runtime/publisher.js +0 -328
- package/src/server-old.ts +0 -988
- package/src/server.ts +0 -2059
- package/src/service/annotations.spec.ts +0 -118
- package/src/service/annotations.ts +0 -91
- package/src/service/authorize.spec.ts +0 -154
- package/src/service/authorize.ts +0 -498
- package/src/service/authorize_integration.spec.ts +0 -2041
- package/src/service/build_plan.spec.ts +0 -685
- package/src/service/build_plan.ts +0 -797
- package/src/service/compile_authorize.spec.ts +0 -85
- package/src/service/compile_fragment_techniques.spec.ts +0 -156
- package/src/service/connection.spec.ts +0 -2360
- package/src/service/connection.ts +0 -2187
- package/src/service/connection_config.spec.ts +0 -734
- package/src/service/connection_config.ts +0 -899
- package/src/service/connection_federation.spec.ts +0 -184
- package/src/service/connection_fingerprint.spec.ts +0 -102
- package/src/service/connection_service.spec.ts +0 -711
- package/src/service/connection_service.ts +0 -291
- package/src/service/cron_evaluator.spec.ts +0 -79
- package/src/service/cron_evaluator.ts +0 -105
- package/src/service/db_utils.spec.ts +0 -1085
- package/src/service/db_utils.ts +0 -1502
- package/src/service/duckdb_instance_isolation.spec.ts +0 -137
- package/src/service/ducklake_lazy_attach.spec.ts +0 -110
- package/src/service/embedding_provider.spec.ts +0 -329
- package/src/service/embedding_provider.ts +0 -236
- package/src/service/environment.ts +0 -2191
- package/src/service/environment_admission.spec.ts +0 -344
- package/src/service/environment_compile.spec.ts +0 -197
- package/src/service/environment_store.spec.ts +0 -2274
- package/src/service/environment_store.ts +0 -2434
- package/src/service/environment_store_anchoring.spec.ts +0 -107
- package/src/service/environment_store_clone.spec.ts +0 -350
- package/src/service/explore_visibility.spec.ts +0 -434
- package/src/service/exports_probe.spec.ts +0 -107
- package/src/service/extension_fetch_policy.spec.ts +0 -256
- package/src/service/filter.spec.ts +0 -458
- package/src/service/filter.ts +0 -341
- package/src/service/filter_bypass.spec.ts +0 -418
- package/src/service/filter_integration.spec.ts +0 -1004
- package/src/service/freshness.spec.ts +0 -183
- package/src/service/freshness.ts +0 -112
- package/src/service/gcs_s3_utils.ts +0 -427
- package/src/service/given.ts +0 -105
- package/src/service/givens_integration.spec.ts +0 -219
- package/src/service/manifest_loader.spec.ts +0 -189
- package/src/service/manifest_loader.ts +0 -156
- package/src/service/materialization_build_session.spec.ts +0 -435
- package/src/service/materialization_build_session.ts +0 -681
- package/src/service/materialization_config_validation.spec.ts +0 -99
- package/src/service/materialization_config_validation.ts +0 -120
- package/src/service/materialization_eligibility.spec.ts +0 -158
- package/src/service/materialization_eligibility.ts +0 -305
- package/src/service/materialization_schedule_surface.spec.ts +0 -256
- package/src/service/materialization_scheduler.spec.ts +0 -289
- package/src/service/materialization_scheduler.ts +0 -304
- package/src/service/materialization_serve_transform.spec.ts +0 -1003
- package/src/service/materialization_serve_transform.ts +0 -779
- package/src/service/materialization_service.spec.ts +0 -2373
- package/src/service/materialization_service.ts +0 -2342
- package/src/service/materialization_test_fixtures.ts +0 -249
- package/src/service/model.spec.ts +0 -1157
- package/src/service/model.ts +0 -3370
- package/src/service/model_limits.spec.ts +0 -209
- package/src/service/model_limits.ts +0 -131
- package/src/service/model_storage_serve.spec.ts +0 -193
- package/src/service/model_storage_serve_joins.spec.ts +0 -193
- package/src/service/package.spec.ts +0 -612
- package/src/service/package.ts +0 -1799
- package/src/service/package_manifest.spec.ts +0 -230
- package/src/service/package_manifest.ts +0 -241
- package/src/service/package_memory_governor.spec.ts +0 -173
- package/src/service/package_memory_governor.ts +0 -233
- package/src/service/package_quote_bound_tables.spec.ts +0 -171
- package/src/service/package_race.spec.ts +0 -208
- package/src/service/package_reload_safety.spec.ts +0 -193
- package/src/service/package_rollback.spec.ts +0 -190
- package/src/service/package_worker_path.spec.ts +0 -645
- package/src/service/path_injection.spec.ts +0 -39
- package/src/service/persist_annotation_validation.spec.ts +0 -89
- package/src/service/persist_annotation_validation.ts +0 -59
- package/src/service/persistence_policy.spec.ts +0 -374
- package/src/service/proxy.spec.ts +0 -414
- package/src/service/proxy.ts +0 -248
- package/src/service/query_boundary.spec.ts +0 -470
- package/src/service/query_metadata.spec.ts +0 -408
- package/src/service/query_metadata.ts +0 -492
- package/src/service/query_metadata_identity.spec.ts +0 -149
- package/src/service/query_text.spec.ts +0 -79
- package/src/service/query_text.ts +0 -43
- package/src/service/quoting.spec.ts +0 -135
- package/src/service/quoting.ts +0 -78
- package/src/service/resolve_environment.ts +0 -15
- package/src/service/restricted_mode.spec.ts +0 -299
- package/src/service/source_extraction.ts +0 -225
- package/src/service/theme_store.ts +0 -199
- package/src/storage/DatabaseInterface.ts +0 -198
- package/src/storage/StorageManager.ts +0 -113
- package/src/storage/duckdb/ConnectionRepository.ts +0 -143
- package/src/storage/duckdb/DuckDBConnection.ts +0 -123
- package/src/storage/duckdb/DuckDBRepository.ts +0 -208
- package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
- package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
- package/src/storage/duckdb/MaterializationRepository.ts +0 -336
- package/src/storage/duckdb/PackageRepository.ts +0 -135
- package/src/storage/duckdb/schema.ts +0 -229
- package/src/stream_helpers.spec.ts +0 -280
- package/src/stream_helpers.ts +0 -162
- package/src/telemetry.ts +0 -20
- package/src/test_helpers/metrics_harness.ts +0 -166
- package/src/theme_key_parity.spec.ts +0 -57
- package/src/utils.ts +0 -29
- package/tests/fixtures/authorize-compile/model.malloy +0 -9
- package/tests/fixtures/authorize-compile/publisher.json +0 -4
- package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
- package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
- package/tests/fixtures/html-pages-test/data.csv +0 -3
- package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
- package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
- package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
- package/tests/fixtures/html-pages-test/public/data.json +0 -1
- package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
- package/tests/fixtures/html-pages-test/public/index.html +0 -9
- package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
- package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
- package/tests/fixtures/html-pages-test/public/slides.html +0 -12
- package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
- package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
- package/tests/fixtures/html-pages-test/publisher.json +0 -5
- package/tests/fixtures/html-pages-test/report.malloy +0 -1
- package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
- package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
- package/tests/fixtures/persist-multi-level/publisher.json +0 -5
- package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
- package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
- package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
- package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
- package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
- package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
- package/tests/fixtures/persist-test/data/orders.csv +0 -5
- package/tests/fixtures/persist-test/persist_test.malloy +0 -11
- package/tests/fixtures/persist-test/publisher.json +0 -5
- package/tests/fixtures/publisher.config.json +0 -15
- package/tests/fixtures/query-givens/data/orders.csv +0 -7
- package/tests/fixtures/query-givens/model.malloy +0 -34
- package/tests/fixtures/query-givens/publisher.json +0 -5
- package/tests/fixtures/xlsx/database.xlsx +0 -0
- package/tests/harness/mcp_test_setup.ts +0 -332
- package/tests/harness/rest_e2e.ts +0 -68
- package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
- package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
- package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
- package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
- package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
- package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
- package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
- package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
- package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
- package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
- package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
- package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
- package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
- package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
- package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
- package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
- package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
- package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
- package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
- package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
- package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
- package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
- package/tests/unit/controller/theme.controller.spec.ts +0 -141
- package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
- package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
- package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
- package/tests/unit/duckdb/repositories.test.ts +0 -208
- package/tests/unit/ducklake/ducklake.test.ts +0 -952
- package/tests/unit/service/theme_store.spec.ts +0 -139
- package/tsconfig.json +0 -32
|
@@ -1,127 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Per-load phase timing for package loads.
|
|
3
|
-
*
|
|
4
|
-
* `malloy_package_load_duration` already reports the *total* time to load a
|
|
5
|
-
* package, but an operator watching a slow load can't tell whether it's the
|
|
6
|
-
* Malloy compile or the connection schema fetches that dominate — the two are
|
|
7
|
-
* tuned very differently (CPU/worker sizing vs. warehouse round-trips). These
|
|
8
|
-
* histograms split the load into those phases so a single dashboard answers
|
|
9
|
-
* "where did the time go?". The worker measures the phases and reports them on
|
|
10
|
-
* its result; see `package_load/package_load_worker.ts` and
|
|
11
|
-
* `package_load/rpc_wait_accountant.ts`.
|
|
12
|
-
*
|
|
13
|
-
* Labels: `status` only. Deliberately NOT labelled by package name — unlike
|
|
14
|
-
* `malloy_package_load_duration`, which carries `malloy_package_name` — to keep
|
|
15
|
-
* cardinality bounded on deployments with many packages. Per-package latency,
|
|
16
|
-
* when needed, is better answered by exemplars/traces than a label explosion.
|
|
17
|
-
*
|
|
18
|
-
* Lazy init for the same reason as `query_cap_metrics.ts` / `query_timeout.ts`:
|
|
19
|
-
* instruments created before `setGlobalMeterProvider` bind to a NoOp meter
|
|
20
|
-
* (https://github.com/open-telemetry/opentelemetry-js/issues/3505).
|
|
21
|
-
*/
|
|
22
|
-
|
|
23
|
-
import { type Histogram } from "@opentelemetry/api";
|
|
24
|
-
import { publisherMeter } from "./telemetry";
|
|
25
|
-
|
|
26
|
-
/**
|
|
27
|
-
* Explicit bucket boundaries (ms) for the load-duration histograms. OTel's
|
|
28
|
-
* default boundaries top out at 10 000 ms, so any load slower than 10 s falls
|
|
29
|
-
* in the `+Inf` bucket and `histogram_quantile` can't resolve the tail — which
|
|
30
|
-
* is exactly the tail (large/slow packages) worth seeing. These extend to
|
|
31
|
-
* 5 minutes while keeping sub-10 ms resolution for the fast common case.
|
|
32
|
-
* Shared with the total `malloy_package_load_duration` histogram so all
|
|
33
|
-
* load-timing histograms bucket consistently.
|
|
34
|
-
*/
|
|
35
|
-
export const LOAD_DURATION_BUCKETS_MS = [
|
|
36
|
-
1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000,
|
|
37
|
-
120_000, 300_000,
|
|
38
|
-
];
|
|
39
|
-
|
|
40
|
-
/** Bucket boundaries for the per-load schema-fetch count. */
|
|
41
|
-
const SCHEMA_FETCH_COUNT_BUCKETS = [0, 1, 2, 5, 10, 25, 50, 100, 250, 500];
|
|
42
|
-
|
|
43
|
-
/**
|
|
44
|
-
* Terminal status of a package load, mirroring the `status` label on the
|
|
45
|
-
* existing `malloy_package_load_duration` histogram so the phase metrics slice
|
|
46
|
-
* the same way. In practice the phase histograms only ever carry `success`,
|
|
47
|
-
* `compilation_error`, or `error` — a `pool_unavailable` failure happens before
|
|
48
|
-
* the worker returns any timings, so there is nothing to record for it.
|
|
49
|
-
*/
|
|
50
|
-
export type PackageLoadStatus =
|
|
51
|
-
| "success"
|
|
52
|
-
| "compilation_error"
|
|
53
|
-
| "pool_unavailable"
|
|
54
|
-
| "error";
|
|
55
|
-
|
|
56
|
-
/** Phase timings a single package load reports (subset of the total). */
|
|
57
|
-
export interface PackageLoadPhaseTimings {
|
|
58
|
-
/** Ceiling on Malloy compile CPU (compile region minus schema-fetch wait). */
|
|
59
|
-
compileDurationMs: number;
|
|
60
|
-
/** Wall-clock awaiting proxied connection schema fetches. */
|
|
61
|
-
schemaFetchDurationMs: number;
|
|
62
|
-
/** Number of proxied connection schema fetches the load drove. */
|
|
63
|
-
schemaFetchCount: number;
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
let compileDuration: Histogram | null = null;
|
|
67
|
-
let schemaFetchDuration: Histogram | null = null;
|
|
68
|
-
let schemaFetches: Histogram | null = null;
|
|
69
|
-
|
|
70
|
-
function ensureInstruments(): void {
|
|
71
|
-
if (compileDuration && schemaFetchDuration && schemaFetches) return;
|
|
72
|
-
const meter = publisherMeter();
|
|
73
|
-
compileDuration ??= meter.createHistogram(
|
|
74
|
-
"malloy_package_load_compile_duration",
|
|
75
|
-
{
|
|
76
|
-
description:
|
|
77
|
-
"Per-load Malloy compile time (compile region minus proxied schema-fetch wait; a ceiling on compile CPU). Label: status.",
|
|
78
|
-
unit: "ms",
|
|
79
|
-
advice: { explicitBucketBoundaries: LOAD_DURATION_BUCKETS_MS },
|
|
80
|
-
},
|
|
81
|
-
);
|
|
82
|
-
schemaFetchDuration ??= meter.createHistogram(
|
|
83
|
-
"malloy_package_load_schema_fetch_duration",
|
|
84
|
-
{
|
|
85
|
-
description:
|
|
86
|
-
"Per-load wall-clock awaiting proxied connection schema fetches. Label: status.",
|
|
87
|
-
unit: "ms",
|
|
88
|
-
advice: { explicitBucketBoundaries: LOAD_DURATION_BUCKETS_MS },
|
|
89
|
-
},
|
|
90
|
-
);
|
|
91
|
-
schemaFetches ??= meter.createHistogram(
|
|
92
|
-
"malloy_package_load_schema_fetches",
|
|
93
|
-
{
|
|
94
|
-
description:
|
|
95
|
-
"Per-load count of proxied connection schema fetches. Label: status.",
|
|
96
|
-
advice: { explicitBucketBoundaries: SCHEMA_FETCH_COUNT_BUCKETS },
|
|
97
|
-
},
|
|
98
|
-
);
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
/**
|
|
102
|
-
* Record one load's phase breakdown. Call once the worker has returned timings,
|
|
103
|
-
* with the load's terminal `status` (so the phases can be sliced by outcome the
|
|
104
|
-
* same way the total duration is). The terms are meaningless for a load that
|
|
105
|
-
* never produced a worker result (e.g. a pool failure), so don't call it there.
|
|
106
|
-
*/
|
|
107
|
-
export function recordPackageLoadPhases(
|
|
108
|
-
timings: PackageLoadPhaseTimings,
|
|
109
|
-
status: PackageLoadStatus,
|
|
110
|
-
): void {
|
|
111
|
-
ensureInstruments();
|
|
112
|
-
const attrs = { status };
|
|
113
|
-
compileDuration?.record(timings.compileDurationMs, attrs);
|
|
114
|
-
schemaFetchDuration?.record(timings.schemaFetchDurationMs, attrs);
|
|
115
|
-
schemaFetches?.record(timings.schemaFetchCount, attrs);
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
/**
|
|
119
|
-
* Visible for tests. Drops cached instruments so a fresh `MeterProvider`
|
|
120
|
-
* (installed via `startMetricsHarness`) captures future emissions. Do NOT call
|
|
121
|
-
* from production code.
|
|
122
|
-
*/
|
|
123
|
-
export function resetPackageLoadMetricsForTesting(): void {
|
|
124
|
-
compileDuration = null;
|
|
125
|
-
schemaFetchDuration = null;
|
|
126
|
-
schemaFetches = null;
|
|
127
|
-
}
|
package/src/path_safety.spec.ts
DELETED
|
@@ -1,158 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from "bun:test";
|
|
2
|
-
import * as path from "path";
|
|
3
|
-
|
|
4
|
-
import { BadRequestError } from "./errors";
|
|
5
|
-
import {
|
|
6
|
-
assertSafeEnvironmentPath,
|
|
7
|
-
assertSafePackageName,
|
|
8
|
-
assertSafeRelativeModelPath,
|
|
9
|
-
safeJoinUnderRoot,
|
|
10
|
-
} from "./path_safety";
|
|
11
|
-
|
|
12
|
-
describe("assertSafePackageName", () => {
|
|
13
|
-
it.each([
|
|
14
|
-
"pkg",
|
|
15
|
-
"test_package",
|
|
16
|
-
"test-package",
|
|
17
|
-
"TestPackage1",
|
|
18
|
-
"test.package.name",
|
|
19
|
-
"a",
|
|
20
|
-
"x".repeat(255),
|
|
21
|
-
])("accepts %p", (name) => {
|
|
22
|
-
expect(() => assertSafePackageName(name)).not.toThrow();
|
|
23
|
-
});
|
|
24
|
-
|
|
25
|
-
it.each([
|
|
26
|
-
["empty", ""],
|
|
27
|
-
["dot", "."],
|
|
28
|
-
["dot-dot", ".."],
|
|
29
|
-
["leading dot", ".staging"],
|
|
30
|
-
["forward slash", "foo/bar"],
|
|
31
|
-
["backslash", "foo\\bar"],
|
|
32
|
-
["null byte", "foo\0bar"],
|
|
33
|
-
["traversal", "../etc/passwd"],
|
|
34
|
-
["abs", "/etc/passwd"],
|
|
35
|
-
["space", "my pkg"],
|
|
36
|
-
["unicode", "pkg\u202E"],
|
|
37
|
-
["too long", "x".repeat(256)],
|
|
38
|
-
])("rejects %s (%p)", (_label, name) => {
|
|
39
|
-
expect(() => assertSafePackageName(name)).toThrow(BadRequestError);
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
it.each([
|
|
43
|
-
["number", 42],
|
|
44
|
-
["null", null],
|
|
45
|
-
["undefined", undefined],
|
|
46
|
-
["object", { name: "pkg" }],
|
|
47
|
-
])("rejects non-string %s (%p)", (_label, value) => {
|
|
48
|
-
expect(() => assertSafePackageName(value)).toThrow(BadRequestError);
|
|
49
|
-
});
|
|
50
|
-
});
|
|
51
|
-
|
|
52
|
-
describe("assertSafeRelativeModelPath", () => {
|
|
53
|
-
it.each([
|
|
54
|
-
"model.malloy",
|
|
55
|
-
"models/foo.malloy",
|
|
56
|
-
"a/b/c/d.malloynb",
|
|
57
|
-
"deep/nested/file_name-1.malloy",
|
|
58
|
-
])("accepts %p", (modelPath) => {
|
|
59
|
-
expect(() => assertSafeRelativeModelPath(modelPath)).not.toThrow();
|
|
60
|
-
});
|
|
61
|
-
|
|
62
|
-
it.each([
|
|
63
|
-
["empty", ""],
|
|
64
|
-
["leading slash (absolute)", "/etc/passwd"],
|
|
65
|
-
["traversal", "../etc/passwd"],
|
|
66
|
-
["embedded traversal", "models/../../../etc/passwd"],
|
|
67
|
-
["embedded dot segment", "models/./foo.malloy"],
|
|
68
|
-
["double slash", "models//foo.malloy"],
|
|
69
|
-
["trailing slash", "models/foo/"],
|
|
70
|
-
["backslash", "models\\foo.malloy"],
|
|
71
|
-
["null byte", "models/foo\0.malloy"],
|
|
72
|
-
["dotfile segment", ".staging/foo.malloy"],
|
|
73
|
-
["dotfile leaf", "models/.hidden.malloy"],
|
|
74
|
-
])("rejects %s (%p)", (_label, modelPath) => {
|
|
75
|
-
expect(() => assertSafeRelativeModelPath(modelPath)).toThrow(
|
|
76
|
-
BadRequestError,
|
|
77
|
-
);
|
|
78
|
-
});
|
|
79
|
-
|
|
80
|
-
it("rejects non-string inputs", () => {
|
|
81
|
-
expect(() => assertSafeRelativeModelPath(undefined)).toThrow(
|
|
82
|
-
BadRequestError,
|
|
83
|
-
);
|
|
84
|
-
expect(() => assertSafeRelativeModelPath(123)).toThrow(BadRequestError);
|
|
85
|
-
});
|
|
86
|
-
});
|
|
87
|
-
|
|
88
|
-
describe("assertSafeEnvironmentPath", () => {
|
|
89
|
-
it.each([
|
|
90
|
-
"/etc/publisher",
|
|
91
|
-
"/var/lib/publisher/env1",
|
|
92
|
-
"/Users/me/data",
|
|
93
|
-
"/a",
|
|
94
|
-
"C:\\Users\\me\\publisher",
|
|
95
|
-
"C:/Users/me/publisher",
|
|
96
|
-
])("accepts %p", (p) => {
|
|
97
|
-
expect(() => assertSafeEnvironmentPath(p)).not.toThrow();
|
|
98
|
-
});
|
|
99
|
-
|
|
100
|
-
it.each([
|
|
101
|
-
["empty", ""],
|
|
102
|
-
["relative", "publisher/data"],
|
|
103
|
-
["traversal in middle", "/var/lib/../../etc/passwd"],
|
|
104
|
-
["traversal at end", "/var/lib/publisher/.."],
|
|
105
|
-
["null byte", "/var/lib/publisher\0"],
|
|
106
|
-
["bare dot-dot", ".."],
|
|
107
|
-
["bare dot", "."],
|
|
108
|
-
["too long", "/" + "a".repeat(5000)],
|
|
109
|
-
])("rejects %s (%p)", (_label, p) => {
|
|
110
|
-
expect(() => assertSafeEnvironmentPath(p)).toThrow(BadRequestError);
|
|
111
|
-
});
|
|
112
|
-
|
|
113
|
-
it("rejects non-string inputs", () => {
|
|
114
|
-
expect(() => assertSafeEnvironmentPath(undefined)).toThrow(
|
|
115
|
-
BadRequestError,
|
|
116
|
-
);
|
|
117
|
-
expect(() => assertSafeEnvironmentPath(null)).toThrow(BadRequestError);
|
|
118
|
-
expect(() => assertSafeEnvironmentPath(42)).toThrow(BadRequestError);
|
|
119
|
-
});
|
|
120
|
-
});
|
|
121
|
-
|
|
122
|
-
describe("safeJoinUnderRoot", () => {
|
|
123
|
-
const root = "/tmp/test-root";
|
|
124
|
-
|
|
125
|
-
it("returns the resolved root when joined with no segments", () => {
|
|
126
|
-
expect(safeJoinUnderRoot(root)).toBe(path.resolve(root));
|
|
127
|
-
});
|
|
128
|
-
|
|
129
|
-
it("joins safe segments into a path under root", () => {
|
|
130
|
-
expect(safeJoinUnderRoot(root, "pkg", "model.malloy")).toBe(
|
|
131
|
-
path.resolve(root, "pkg", "model.malloy"),
|
|
132
|
-
);
|
|
133
|
-
});
|
|
134
|
-
|
|
135
|
-
it("throws when traversal escapes the root", () => {
|
|
136
|
-
expect(() => safeJoinUnderRoot(root, "..")).toThrow(BadRequestError);
|
|
137
|
-
expect(() => safeJoinUnderRoot(root, "..", "etc", "passwd")).toThrow(
|
|
138
|
-
BadRequestError,
|
|
139
|
-
);
|
|
140
|
-
expect(() => safeJoinUnderRoot(root, "pkg", "..", "..", "etc")).toThrow(
|
|
141
|
-
BadRequestError,
|
|
142
|
-
);
|
|
143
|
-
});
|
|
144
|
-
|
|
145
|
-
it("throws when an absolute segment overrides the root", () => {
|
|
146
|
-
expect(() => safeJoinUnderRoot(root, "/etc/passwd")).toThrow(
|
|
147
|
-
BadRequestError,
|
|
148
|
-
);
|
|
149
|
-
});
|
|
150
|
-
|
|
151
|
-
it("does NOT match a sibling directory with the same prefix", () => {
|
|
152
|
-
// path.resolve("/tmp/test-root", "../test-root-bad") -> "/tmp/test-root-bad"
|
|
153
|
-
// which starts with "/tmp/test-root" textually but is NOT a child.
|
|
154
|
-
expect(() => safeJoinUnderRoot(root, "..", "test-root-bad")).toThrow(
|
|
155
|
-
BadRequestError,
|
|
156
|
-
);
|
|
157
|
-
});
|
|
158
|
-
});
|
package/src/path_safety.ts
DELETED
|
@@ -1,146 +0,0 @@
|
|
|
1
|
-
import * as path from "path";
|
|
2
|
-
|
|
3
|
-
import { BadRequestError } from "./errors";
|
|
4
|
-
|
|
5
|
-
/**
|
|
6
|
-
* Path-safety helpers used by `Environment` (and any other service that
|
|
7
|
-
* builds an on-disk path from request data) to defend against directory
|
|
8
|
-
* traversal. The intent is two-fold:
|
|
9
|
-
*
|
|
10
|
-
* 1. **Source-side allowlist**: `assertSafePackageName` /
|
|
11
|
-
* `assertSafeRelativeModelPath` reject hostile inputs (`..`, leading
|
|
12
|
-
* `/`, `\`, NUL, dotfiles) at the entry of every public service
|
|
13
|
-
* method before any path-construction happens. These throw
|
|
14
|
-
* `BadRequestError` so the controller layer's error mapper returns
|
|
15
|
-
* HTTP 400.
|
|
16
|
-
*
|
|
17
|
-
* 2. **Sink-side containment**: `safeJoinUnderRoot` joins, resolves,
|
|
18
|
-
* and verifies the result is strictly within the supplied root.
|
|
19
|
-
* Even if a future caller forgets the source-side check, the sink
|
|
20
|
-
* refuses to hand back an escaping path. This is the standard
|
|
21
|
-
* "resolve-and-contain" pattern that CodeQL's `js/path-injection`
|
|
22
|
-
* query recognises as a sanitizer.
|
|
23
|
-
*/
|
|
24
|
-
|
|
25
|
-
// Single path segment: ASCII letters, digits, `-`, `_`, `.`. No leading
|
|
26
|
-
// `.` so internal sibling dirs (`.staging`, `.retired`) and editor /
|
|
27
|
-
// VCS dirs can't be addressed by name from outside.
|
|
28
|
-
const SAFE_NAME_RE = /^(?!\.\.?$)(?!\.)[A-Za-z0-9._-]{1,255}$/;
|
|
29
|
-
|
|
30
|
-
const MAX_MODEL_PATH_LEN = 1024;
|
|
31
|
-
|
|
32
|
-
// An environment path is server-controlled (config / disk-derived), but
|
|
33
|
-
// CodeQL conservatively treats it as tainted because Express handlers on
|
|
34
|
-
// the same class touch user input. The combined regex test +
|
|
35
|
-
// `..` / NUL / length check at the constructor gate is the sanitizer
|
|
36
|
-
// barrier the `js/path-injection` query recognises. Printable ASCII
|
|
37
|
-
// only; absolute POSIX-or-Windows path; no `..`, no NUL.
|
|
38
|
-
const SAFE_ENVIRONMENT_PATH_RE = /^(?:\/|[A-Za-z]:[\\/])[\x20-\x7E]*$/;
|
|
39
|
-
const MAX_ENVIRONMENT_PATH_LEN = 4096;
|
|
40
|
-
|
|
41
|
-
/**
|
|
42
|
-
* Reject anything that isn't a plausible single-segment package name.
|
|
43
|
-
* The allowlist is deliberately conservative — every existing test and
|
|
44
|
-
* production package name we've seen fits within it, and tightening
|
|
45
|
-
* here costs nothing.
|
|
46
|
-
*/
|
|
47
|
-
export function assertSafePackageName(
|
|
48
|
-
packageName: unknown,
|
|
49
|
-
): asserts packageName is string {
|
|
50
|
-
if (typeof packageName !== "string" || !SAFE_NAME_RE.test(packageName)) {
|
|
51
|
-
throw new BadRequestError(
|
|
52
|
-
`Invalid package name: must be 1-255 characters of letters, digits, "-", "_", or "." and must not start with "."`,
|
|
53
|
-
);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
/**
|
|
58
|
-
* Reject anything that isn't a plausible *relative* path to a model
|
|
59
|
-
* file inside a package directory. Forward slashes are allowed (models
|
|
60
|
-
* live in subdirectories like `models/foo.malloy`); backslashes,
|
|
61
|
-
* absolute paths, NUL bytes, and `..` / `.` segments are not.
|
|
62
|
-
*/
|
|
63
|
-
export function assertSafeRelativeModelPath(
|
|
64
|
-
modelPath: unknown,
|
|
65
|
-
): asserts modelPath is string {
|
|
66
|
-
if (
|
|
67
|
-
typeof modelPath !== "string" ||
|
|
68
|
-
modelPath.length === 0 ||
|
|
69
|
-
modelPath.length > MAX_MODEL_PATH_LEN ||
|
|
70
|
-
modelPath.includes("\0") ||
|
|
71
|
-
modelPath.includes("\\") ||
|
|
72
|
-
path.isAbsolute(modelPath) ||
|
|
73
|
-
modelPath.startsWith("/")
|
|
74
|
-
) {
|
|
75
|
-
throw new BadRequestError(`Invalid model path`);
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
const segments = modelPath.split("/");
|
|
79
|
-
for (const segment of segments) {
|
|
80
|
-
if (segment === "" || segment === "." || segment === "..") {
|
|
81
|
-
throw new BadRequestError(`Invalid model path`);
|
|
82
|
-
}
|
|
83
|
-
if (segment.startsWith(".")) {
|
|
84
|
-
throw new BadRequestError(`Invalid model path`);
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
/**
|
|
90
|
-
* Reject anything that doesn't look like a server-controlled absolute
|
|
91
|
-
* filesystem path. Applied to `environmentPath` at the constructor
|
|
92
|
-
* gate so all downstream `path.join(this.environmentPath, …)` sites
|
|
93
|
-
* see a value that has cleared an allowlist check — the canonical
|
|
94
|
-
* sanitizer-barrier pattern CodeQL's `js/path-injection` query
|
|
95
|
-
* recognises.
|
|
96
|
-
*/
|
|
97
|
-
export function assertSafeEnvironmentPath(
|
|
98
|
-
environmentPath: unknown,
|
|
99
|
-
): asserts environmentPath is string {
|
|
100
|
-
if (typeof environmentPath !== "string") {
|
|
101
|
-
throw new BadRequestError(`Invalid environment path: must be a string`);
|
|
102
|
-
}
|
|
103
|
-
if (
|
|
104
|
-
environmentPath.length === 0 ||
|
|
105
|
-
environmentPath.length > MAX_ENVIRONMENT_PATH_LEN
|
|
106
|
-
) {
|
|
107
|
-
throw new BadRequestError(`Invalid environment path: bad length`);
|
|
108
|
-
}
|
|
109
|
-
if (environmentPath.indexOf("\0") !== -1) {
|
|
110
|
-
throw new BadRequestError(`Invalid environment path: contains NUL byte`);
|
|
111
|
-
}
|
|
112
|
-
// Sanitizer barrier in the shape `x.indexOf("..") !== -1` that the
|
|
113
|
-
// CodeQL `js/path-injection` query recognises as a traversal guard.
|
|
114
|
-
if (environmentPath.indexOf("..") !== -1) {
|
|
115
|
-
throw new BadRequestError(
|
|
116
|
-
`Invalid environment path: contains ".." traversal segment`,
|
|
117
|
-
);
|
|
118
|
-
}
|
|
119
|
-
if (!SAFE_ENVIRONMENT_PATH_RE.test(environmentPath)) {
|
|
120
|
-
throw new BadRequestError(
|
|
121
|
-
`Invalid environment path: must be an absolute path of printable ASCII characters`,
|
|
122
|
-
);
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
/**
|
|
127
|
-
* Resolve `path.join(root, ...segments)` and verify the result lives
|
|
128
|
-
* strictly inside `root` (or is `root` itself). Throws
|
|
129
|
-
* `BadRequestError` if the resolved path escapes the root via `..`,
|
|
130
|
-
* absolute segments, or symlink-style trickery in the input.
|
|
131
|
-
*
|
|
132
|
-
* Callers should still run `assertSafePackageName` / similar on
|
|
133
|
-
* user-controlled segments first — this helper is the second line of
|
|
134
|
-
* defense, not the first.
|
|
135
|
-
*/
|
|
136
|
-
export function safeJoinUnderRoot(root: string, ...segments: string[]): string {
|
|
137
|
-
const resolvedRoot = path.resolve(root);
|
|
138
|
-
const joined = path.resolve(resolvedRoot, ...segments);
|
|
139
|
-
const rootWithSep = resolvedRoot.endsWith(path.sep)
|
|
140
|
-
? resolvedRoot
|
|
141
|
-
: resolvedRoot + path.sep;
|
|
142
|
-
if (joined !== resolvedRoot && !joined.startsWith(rootWithSep)) {
|
|
143
|
-
throw new BadRequestError(`Resolved path is outside of root`);
|
|
144
|
-
}
|
|
145
|
-
return joined;
|
|
146
|
-
}
|
package/src/pg_helpers.spec.ts
DELETED
|
@@ -1,223 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from "bun:test";
|
|
2
|
-
import { redactPgSecrets } from "./pg_helpers";
|
|
3
|
-
|
|
4
|
-
describe("redactPgSecrets", () => {
|
|
5
|
-
it("redacts bare password values", () => {
|
|
6
|
-
expect(redactPgSecrets("host=h password=hunter2 dbname=d")).toBe(
|
|
7
|
-
"host=h password=*** dbname=d",
|
|
8
|
-
);
|
|
9
|
-
});
|
|
10
|
-
|
|
11
|
-
it("redacts single-quoted password values", () => {
|
|
12
|
-
expect(redactPgSecrets("host=h password='s3 cret' dbname=d")).toBe(
|
|
13
|
-
"host=h password=*** dbname=d",
|
|
14
|
-
);
|
|
15
|
-
});
|
|
16
|
-
|
|
17
|
-
it("leaves non-secret content alone", () => {
|
|
18
|
-
expect(redactPgSecrets("user=alice dbname=billing")).toBe(
|
|
19
|
-
"user=alice dbname=billing",
|
|
20
|
-
);
|
|
21
|
-
});
|
|
22
|
-
|
|
23
|
-
it("redacts the password in a URL-form connection string", () => {
|
|
24
|
-
expect(redactPgSecrets("postgres://user:pass@host/db")).toBe(
|
|
25
|
-
"postgres://user:***@host/db",
|
|
26
|
-
);
|
|
27
|
-
});
|
|
28
|
-
|
|
29
|
-
it("preserves port and query string around a URI password", () => {
|
|
30
|
-
expect(redactPgSecrets("postgresql://u:p@h:5432/d?sslmode=require")).toBe(
|
|
31
|
-
"postgresql://u:***@h:5432/d?sslmode=require",
|
|
32
|
-
);
|
|
33
|
-
});
|
|
34
|
-
|
|
35
|
-
it("redacts a percent-encoded password whole", () => {
|
|
36
|
-
expect(redactPgSecrets("postgres://u:p%40ss@h/d")).toBe(
|
|
37
|
-
"postgres://u:***@h/d",
|
|
38
|
-
);
|
|
39
|
-
});
|
|
40
|
-
|
|
41
|
-
it("redacts a literal @ in the password up to the last @", () => {
|
|
42
|
-
expect(redactPgSecrets("postgres://u:p@ss@h/d")).toBe(
|
|
43
|
-
"postgres://u:***@h/d",
|
|
44
|
-
);
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
it("redacts a password containing colons whole", () => {
|
|
48
|
-
expect(redactPgSecrets("postgres://u:pa:ss@h/d")).toBe(
|
|
49
|
-
"postgres://u:***@h/d",
|
|
50
|
-
);
|
|
51
|
-
});
|
|
52
|
-
|
|
53
|
-
it("redacts a raw slash in a pg password via the mop-up pass", () => {
|
|
54
|
-
expect(redactPgSecrets("postgres://u:pa/ss@h/d")).toBe(
|
|
55
|
-
"postgres://u:***@h/d",
|
|
56
|
-
);
|
|
57
|
-
});
|
|
58
|
-
|
|
59
|
-
it("redacts the password when the username is empty", () => {
|
|
60
|
-
expect(redactPgSecrets("postgres://:secret@h/d")).toBe(
|
|
61
|
-
"postgres://:***@h/d",
|
|
62
|
-
);
|
|
63
|
-
});
|
|
64
|
-
|
|
65
|
-
it("leaves IPv6 hosts intact", () => {
|
|
66
|
-
expect(redactPgSecrets("postgres://u:p@[::1]:5432/d")).toBe(
|
|
67
|
-
"postgres://u:***@[::1]:5432/d",
|
|
68
|
-
);
|
|
69
|
-
});
|
|
70
|
-
|
|
71
|
-
it("does not redact a username-only userinfo", () => {
|
|
72
|
-
expect(redactPgSecrets("postgres://user@host/db")).toBe(
|
|
73
|
-
"postgres://user@host/db",
|
|
74
|
-
);
|
|
75
|
-
});
|
|
76
|
-
|
|
77
|
-
it("leaves URIs without userinfo alone", () => {
|
|
78
|
-
expect(redactPgSecrets("postgres://host/db")).toBe("postgres://host/db");
|
|
79
|
-
expect(redactPgSecrets("postgres://host:5432/db")).toBe(
|
|
80
|
-
"postgres://host:5432/db",
|
|
81
|
-
);
|
|
82
|
-
});
|
|
83
|
-
|
|
84
|
-
it("does not mangle a passwordless URI with :...@ in its query", () => {
|
|
85
|
-
expect(redactPgSecrets("postgres://h?options=endpoint:foo@bar")).toBe(
|
|
86
|
-
"postgres://h?options=endpoint:foo@bar",
|
|
87
|
-
);
|
|
88
|
-
expect(redactPgSecrets("postgres://h/d?x=a@b")).toBe(
|
|
89
|
-
"postgres://h/d?x=a@b",
|
|
90
|
-
);
|
|
91
|
-
});
|
|
92
|
-
|
|
93
|
-
it("leaves email addresses in prose alone", () => {
|
|
94
|
-
expect(redactPgSecrets("contact admin@example.com for help")).toBe(
|
|
95
|
-
"contact admin@example.com for help",
|
|
96
|
-
);
|
|
97
|
-
});
|
|
98
|
-
|
|
99
|
-
it("does not mangle https URLs with @ in the path", () => {
|
|
100
|
-
expect(
|
|
101
|
-
redactPgSecrets("fetched https://github.com:443/@scope/pkg ok"),
|
|
102
|
-
).toBe("fetched https://github.com:443/@scope/pkg ok");
|
|
103
|
-
});
|
|
104
|
-
|
|
105
|
-
it("redacts every URI in a message independently", () => {
|
|
106
|
-
expect(
|
|
107
|
-
redactPgSecrets(
|
|
108
|
-
"tried postgres://u1:p1@h1/d1 then postgres://u2:p2@h2/d2",
|
|
109
|
-
),
|
|
110
|
-
).toBe("tried postgres://u1:***@h1/d1 then postgres://u2:***@h2/d2");
|
|
111
|
-
});
|
|
112
|
-
|
|
113
|
-
it("keeps comma-joined URIs in one token separate", () => {
|
|
114
|
-
expect(
|
|
115
|
-
redactPgSecrets(
|
|
116
|
-
"candidates: postgres://u1:p1@h1/d1,postgres://u2:p2@h2/d2",
|
|
117
|
-
),
|
|
118
|
-
).toBe("candidates: postgres://u1:***@h1/d1,postgres://u2:***@h2/d2");
|
|
119
|
-
});
|
|
120
|
-
|
|
121
|
-
it("leaves trailing sentence punctuation alone", () => {
|
|
122
|
-
expect(redactPgSecrets("could not connect to postgres://u:p@h/d.")).toBe(
|
|
123
|
-
"could not connect to postgres://u:***@h/d.",
|
|
124
|
-
);
|
|
125
|
-
});
|
|
126
|
-
|
|
127
|
-
it("handles keyword and URI forms in one message", () => {
|
|
128
|
-
expect(
|
|
129
|
-
redactPgSecrets(
|
|
130
|
-
"attach failed: host=h password=x and fallback postgres://u:p@h2/d",
|
|
131
|
-
),
|
|
132
|
-
).toBe(
|
|
133
|
-
"attach failed: host=h password=*** and fallback postgres://u:***@h2/d",
|
|
134
|
-
);
|
|
135
|
-
});
|
|
136
|
-
|
|
137
|
-
it("keeps the URI intact when a password contains 'password='", () => {
|
|
138
|
-
// URI passes run before the keyword pass; keyword-first would eat
|
|
139
|
-
// the @host/db tail via \S+.
|
|
140
|
-
expect(redactPgSecrets("postgres://u:password=abc@h/d")).toBe(
|
|
141
|
-
"postgres://u:***@h/d",
|
|
142
|
-
);
|
|
143
|
-
});
|
|
144
|
-
|
|
145
|
-
it("over-redacts rather than leaks a ?password= query parameter", () => {
|
|
146
|
-
// Pre-existing keyword-pass behavior: \S+ eats through &sslmode=...
|
|
147
|
-
// Over-redaction, never a leak.
|
|
148
|
-
expect(
|
|
149
|
-
redactPgSecrets("postgres://h/d?password=abc&sslmode=require"),
|
|
150
|
-
).toBe("postgres://h/d?password=***");
|
|
151
|
-
});
|
|
152
|
-
|
|
153
|
-
it("matches uppercase schemes", () => {
|
|
154
|
-
expect(redactPgSecrets("POSTGRES://U:Secret@H/D")).toBe(
|
|
155
|
-
"POSTGRES://U:***@H/D",
|
|
156
|
-
);
|
|
157
|
-
});
|
|
158
|
-
|
|
159
|
-
it("redacts non-pg scheme userinfo passwords too", () => {
|
|
160
|
-
expect(redactPgSecrets("amqp://guest:guest@rabbit:5672/vhost")).toBe(
|
|
161
|
-
"amqp://guest:***@rabbit:5672/vhost",
|
|
162
|
-
);
|
|
163
|
-
});
|
|
164
|
-
|
|
165
|
-
it("redacts an Azure user@servername username form", () => {
|
|
166
|
-
// Azure single-server usernames carry a literal @ (user@servername);
|
|
167
|
-
// userinfo splits at the LAST @, so the password after the first : is
|
|
168
|
-
// redacted while the @-bearing username stays intact.
|
|
169
|
-
expect(
|
|
170
|
-
redactPgSecrets(
|
|
171
|
-
"postgres://myadmin@myserver:Zx7SeCReT9q@myserver.postgres.database.azure.com:5432/db",
|
|
172
|
-
),
|
|
173
|
-
).toBe(
|
|
174
|
-
"postgres://myadmin@myserver:***@myserver.postgres.database.azure.com:5432/db",
|
|
175
|
-
);
|
|
176
|
-
});
|
|
177
|
-
|
|
178
|
-
it("redacts an @-in-username form whose password also contains @", () => {
|
|
179
|
-
expect(redactPgSecrets("postgres://myadmin@srv:p@ss@host/db")).toBe(
|
|
180
|
-
"postgres://myadmin@srv:***@host/db",
|
|
181
|
-
);
|
|
182
|
-
});
|
|
183
|
-
|
|
184
|
-
it("redacts an @-in-username form with a raw / in the password", () => {
|
|
185
|
-
expect(redactPgSecrets("postgresql://u@srv:sec/ret@h/d")).toBe(
|
|
186
|
-
"postgresql://u@srv:***@h/d",
|
|
187
|
-
);
|
|
188
|
-
});
|
|
189
|
-
|
|
190
|
-
it("redacts a scheme abutting a word char", () => {
|
|
191
|
-
// No leading anchor on the scheme, so a scheme glued to a preceding
|
|
192
|
-
// word char (an invalid-scheme prefix, or a stray digit) still redacts.
|
|
193
|
-
expect(redactPgSecrets("x_postgres://u:secretpw@h/d")).toBe(
|
|
194
|
-
"x_postgres://u:***@h/d",
|
|
195
|
-
);
|
|
196
|
-
expect(redactPgSecrets("3postgres://u:secretpw@h/d")).toBe(
|
|
197
|
-
"3postgres://u:***@h/d",
|
|
198
|
-
);
|
|
199
|
-
});
|
|
200
|
-
|
|
201
|
-
it("never reveals a secret when re-applied (monotonic)", () => {
|
|
202
|
-
// redactPgSecrets is not strictly idempotent in general (re-applying
|
|
203
|
-
// can redact further, e.g. an unquoted *** left by the keyword pass),
|
|
204
|
-
// but re-application must never turn a redacted secret back into
|
|
205
|
-
// cleartext.
|
|
206
|
-
const input = "postgres://u:p@ss@h/d and host=h password=hunter2";
|
|
207
|
-
const once = redactPgSecrets(input);
|
|
208
|
-
const twice = redactPgSecrets(once);
|
|
209
|
-
expect(once).not.toContain("hunter2");
|
|
210
|
-
expect(twice).not.toContain("hunter2");
|
|
211
|
-
expect(twice).not.toContain("p@ss");
|
|
212
|
-
});
|
|
213
|
-
|
|
214
|
-
it("redacts the DSN inside a real DuckDB attach error message", () => {
|
|
215
|
-
// Shape observed from DuckDB's postgres extension: the full DSN is
|
|
216
|
-
// echoed verbatim inside the IO Error text.
|
|
217
|
-
const msg =
|
|
218
|
-
'IO Error: Unable to connect to Postgres at "postgres://alice:supersecretpw@127.0.0.1:5432/mydb": connection to server at "127.0.0.1", port 5432 failed: Connection refused';
|
|
219
|
-
const redacted = redactPgSecrets(msg);
|
|
220
|
-
expect(redacted).toContain("postgres://alice:***@127.0.0.1:5432/mydb");
|
|
221
|
-
expect(redacted).not.toContain("supersecretpw");
|
|
222
|
-
});
|
|
223
|
-
});
|