@malloy-publisher/server 0.0.235 → 0.0.236

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (301) hide show
  1. package/dist/app/api-doc.yaml +47 -17
  2. package/dist/package_load_worker.mjs +8 -1
  3. package/dist/server.mjs +471 -54
  4. package/package.json +5 -2
  5. package/.eslintrc.json +0 -44
  6. package/.prettierignore +0 -1
  7. package/.prettierrc +0 -4
  8. package/build.ts +0 -97
  9. package/eslint.config.mjs +0 -76
  10. package/k6-tests/README.md +0 -144
  11. package/k6-tests/bun.lock +0 -797
  12. package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
  13. package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
  14. package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
  15. package/k6-tests/load-test/load-test-crud.ts +0 -142
  16. package/k6-tests/load-test/load-test.ts +0 -784
  17. package/k6-tests/package.json +0 -27
  18. package/k6-tests/scripts/run-with-otel.sh +0 -50
  19. package/k6-tests/smoke-test/smoke-test.ts +0 -158
  20. package/k6-tests/tsconfig.json +0 -29
  21. package/k6-tests/utils/client_factory.ts +0 -128
  22. package/k6-tests/utils/common.ts +0 -489
  23. package/k6-tests/utils/logger.ts +0 -62
  24. package/k6-tests/utils/types.d.ts +0 -35
  25. package/malloy_mcp.dxt +0 -0
  26. package/publisher.config.example.bigquery.json +0 -33
  27. package/publisher.config.example.duckdb.json +0 -23
  28. package/publisher.config.json +0 -23
  29. package/scripts/bake-duckdb-extensions.js +0 -110
  30. package/src/config.spec.ts +0 -1535
  31. package/src/config.theme.spec.ts +0 -178
  32. package/src/config.ts +0 -1142
  33. package/src/constants.ts +0 -105
  34. package/src/controller/compile.controller.ts +0 -40
  35. package/src/controller/connection.controller.spec.ts +0 -849
  36. package/src/controller/connection.controller.ts +0 -901
  37. package/src/controller/database.controller.ts +0 -24
  38. package/src/controller/materialization.controller.spec.ts +0 -293
  39. package/src/controller/materialization.controller.ts +0 -319
  40. package/src/controller/model.controller.ts +0 -164
  41. package/src/controller/package.controller.spec.ts +0 -273
  42. package/src/controller/package.controller.ts +0 -306
  43. package/src/controller/query.controller.ts +0 -152
  44. package/src/controller/theme.controller.ts +0 -83
  45. package/src/controller/watch-mode.controller.ts +0 -243
  46. package/src/data_styles.ts +0 -71
  47. package/src/default-publisher.config.json +0 -23
  48. package/src/ducklake_version.spec.ts +0 -163
  49. package/src/ducklake_version.ts +0 -153
  50. package/src/errors.spec.ts +0 -119
  51. package/src/errors.ts +0 -237
  52. package/src/filter_deprecation.spec.ts +0 -64
  53. package/src/filter_deprecation.ts +0 -42
  54. package/src/health.spec.ts +0 -123
  55. package/src/health.ts +0 -233
  56. package/src/heap_check.spec.ts +0 -144
  57. package/src/heap_check.ts +0 -144
  58. package/src/instrumentation.ts +0 -214
  59. package/src/json_utils.spec.ts +0 -51
  60. package/src/json_utils.ts +0 -33
  61. package/src/logger.spec.ts +0 -210
  62. package/src/logger.ts +0 -251
  63. package/src/malloy_pin_prereqs.spec.ts +0 -25
  64. package/src/materialization_metrics.spec.ts +0 -169
  65. package/src/materialization_metrics.ts +0 -346
  66. package/src/mcp/error_messages.spec.ts +0 -35
  67. package/src/mcp/error_messages.ts +0 -183
  68. package/src/mcp/handler_utils.spec.ts +0 -108
  69. package/src/mcp/handler_utils.ts +0 -251
  70. package/src/mcp/mcp_constants.ts +0 -6
  71. package/src/mcp/query_envelope.spec.ts +0 -229
  72. package/src/mcp/query_envelope.ts +0 -240
  73. package/src/mcp/server.protocol.spec.ts +0 -250
  74. package/src/mcp/server.ts +0 -87
  75. package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
  76. package/src/mcp/skills/build_skills_bundle.ts +0 -176
  77. package/src/mcp/skills/skills_bundle.json +0 -1
  78. package/src/mcp/skills/skills_bundle.spec.ts +0 -171
  79. package/src/mcp/tool_response.spec.ts +0 -108
  80. package/src/mcp/tool_response.ts +0 -138
  81. package/src/mcp/tools/compile_tool.spec.ts +0 -315
  82. package/src/mcp/tools/compile_tool.ts +0 -208
  83. package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
  84. package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
  85. package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
  86. package/src/mcp/tools/docs_search_tool.ts +0 -128
  87. package/src/mcp/tools/embedding_index.spec.ts +0 -1236
  88. package/src/mcp/tools/embedding_index.ts +0 -808
  89. package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
  90. package/src/mcp/tools/execute_query_tool.ts +0 -316
  91. package/src/mcp/tools/get_context_eval.ts +0 -275
  92. package/src/mcp/tools/get_context_tool.spec.ts +0 -592
  93. package/src/mcp/tools/get_context_tool.ts +0 -585
  94. package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
  95. package/src/mcp/tools/reload_package_tool.ts +0 -132
  96. package/src/mcp_config.spec.ts +0 -919
  97. package/src/mcp_config.ts +0 -425
  98. package/src/oom_guards.integration.spec.ts +0 -269
  99. package/src/package_load/package_load_pool.spec.ts +0 -252
  100. package/src/package_load/package_load_pool.ts +0 -931
  101. package/src/package_load/package_load_worker.ts +0 -1034
  102. package/src/package_load/protocol.ts +0 -364
  103. package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
  104. package/src/package_load/rpc_wait_accountant.ts +0 -76
  105. package/src/package_load_metrics.spec.ts +0 -114
  106. package/src/package_load_metrics.ts +0 -127
  107. package/src/path_safety.spec.ts +0 -158
  108. package/src/path_safety.ts +0 -146
  109. package/src/pg_helpers.spec.ts +0 -223
  110. package/src/pg_helpers.ts +0 -52
  111. package/src/query_cap_metrics.spec.ts +0 -89
  112. package/src/query_cap_metrics.ts +0 -116
  113. package/src/query_concurrency.spec.ts +0 -247
  114. package/src/query_concurrency.ts +0 -237
  115. package/src/query_metadata_metrics.ts +0 -49
  116. package/src/query_param_utils.ts +0 -29
  117. package/src/query_timeout.spec.ts +0 -224
  118. package/src/query_timeout.ts +0 -179
  119. package/src/runtime/publisher.js +0 -328
  120. package/src/server-old.ts +0 -988
  121. package/src/server.ts +0 -2059
  122. package/src/service/annotations.spec.ts +0 -118
  123. package/src/service/annotations.ts +0 -91
  124. package/src/service/authorize.spec.ts +0 -154
  125. package/src/service/authorize.ts +0 -498
  126. package/src/service/authorize_integration.spec.ts +0 -2041
  127. package/src/service/build_plan.spec.ts +0 -685
  128. package/src/service/build_plan.ts +0 -797
  129. package/src/service/compile_authorize.spec.ts +0 -85
  130. package/src/service/compile_fragment_techniques.spec.ts +0 -156
  131. package/src/service/connection.spec.ts +0 -2360
  132. package/src/service/connection.ts +0 -2187
  133. package/src/service/connection_config.spec.ts +0 -734
  134. package/src/service/connection_config.ts +0 -899
  135. package/src/service/connection_federation.spec.ts +0 -184
  136. package/src/service/connection_fingerprint.spec.ts +0 -102
  137. package/src/service/connection_service.spec.ts +0 -711
  138. package/src/service/connection_service.ts +0 -291
  139. package/src/service/cron_evaluator.spec.ts +0 -79
  140. package/src/service/cron_evaluator.ts +0 -105
  141. package/src/service/db_utils.spec.ts +0 -1085
  142. package/src/service/db_utils.ts +0 -1502
  143. package/src/service/duckdb_instance_isolation.spec.ts +0 -137
  144. package/src/service/ducklake_lazy_attach.spec.ts +0 -110
  145. package/src/service/embedding_provider.spec.ts +0 -329
  146. package/src/service/embedding_provider.ts +0 -236
  147. package/src/service/environment.ts +0 -2191
  148. package/src/service/environment_admission.spec.ts +0 -344
  149. package/src/service/environment_compile.spec.ts +0 -197
  150. package/src/service/environment_store.spec.ts +0 -2274
  151. package/src/service/environment_store.ts +0 -2434
  152. package/src/service/environment_store_anchoring.spec.ts +0 -107
  153. package/src/service/environment_store_clone.spec.ts +0 -350
  154. package/src/service/explore_visibility.spec.ts +0 -434
  155. package/src/service/exports_probe.spec.ts +0 -107
  156. package/src/service/extension_fetch_policy.spec.ts +0 -256
  157. package/src/service/filter.spec.ts +0 -458
  158. package/src/service/filter.ts +0 -341
  159. package/src/service/filter_bypass.spec.ts +0 -418
  160. package/src/service/filter_integration.spec.ts +0 -1004
  161. package/src/service/freshness.spec.ts +0 -183
  162. package/src/service/freshness.ts +0 -112
  163. package/src/service/gcs_s3_utils.ts +0 -427
  164. package/src/service/given.ts +0 -105
  165. package/src/service/givens_integration.spec.ts +0 -219
  166. package/src/service/manifest_loader.spec.ts +0 -189
  167. package/src/service/manifest_loader.ts +0 -156
  168. package/src/service/materialization_build_session.spec.ts +0 -435
  169. package/src/service/materialization_build_session.ts +0 -681
  170. package/src/service/materialization_config_validation.spec.ts +0 -99
  171. package/src/service/materialization_config_validation.ts +0 -120
  172. package/src/service/materialization_eligibility.spec.ts +0 -158
  173. package/src/service/materialization_eligibility.ts +0 -305
  174. package/src/service/materialization_schedule_surface.spec.ts +0 -256
  175. package/src/service/materialization_scheduler.spec.ts +0 -289
  176. package/src/service/materialization_scheduler.ts +0 -304
  177. package/src/service/materialization_serve_transform.spec.ts +0 -1003
  178. package/src/service/materialization_serve_transform.ts +0 -779
  179. package/src/service/materialization_service.spec.ts +0 -2373
  180. package/src/service/materialization_service.ts +0 -2342
  181. package/src/service/materialization_test_fixtures.ts +0 -249
  182. package/src/service/model.spec.ts +0 -1157
  183. package/src/service/model.ts +0 -3370
  184. package/src/service/model_limits.spec.ts +0 -209
  185. package/src/service/model_limits.ts +0 -131
  186. package/src/service/model_storage_serve.spec.ts +0 -193
  187. package/src/service/model_storage_serve_joins.spec.ts +0 -193
  188. package/src/service/package.spec.ts +0 -612
  189. package/src/service/package.ts +0 -1799
  190. package/src/service/package_manifest.spec.ts +0 -230
  191. package/src/service/package_manifest.ts +0 -241
  192. package/src/service/package_memory_governor.spec.ts +0 -173
  193. package/src/service/package_memory_governor.ts +0 -233
  194. package/src/service/package_quote_bound_tables.spec.ts +0 -171
  195. package/src/service/package_race.spec.ts +0 -208
  196. package/src/service/package_reload_safety.spec.ts +0 -193
  197. package/src/service/package_rollback.spec.ts +0 -190
  198. package/src/service/package_worker_path.spec.ts +0 -645
  199. package/src/service/path_injection.spec.ts +0 -39
  200. package/src/service/persist_annotation_validation.spec.ts +0 -89
  201. package/src/service/persist_annotation_validation.ts +0 -59
  202. package/src/service/persistence_policy.spec.ts +0 -374
  203. package/src/service/proxy.spec.ts +0 -414
  204. package/src/service/proxy.ts +0 -248
  205. package/src/service/query_boundary.spec.ts +0 -470
  206. package/src/service/query_metadata.spec.ts +0 -408
  207. package/src/service/query_metadata.ts +0 -492
  208. package/src/service/query_metadata_identity.spec.ts +0 -149
  209. package/src/service/query_text.spec.ts +0 -79
  210. package/src/service/query_text.ts +0 -43
  211. package/src/service/quoting.spec.ts +0 -135
  212. package/src/service/quoting.ts +0 -78
  213. package/src/service/resolve_environment.ts +0 -15
  214. package/src/service/restricted_mode.spec.ts +0 -299
  215. package/src/service/source_extraction.ts +0 -225
  216. package/src/service/theme_store.ts +0 -199
  217. package/src/storage/DatabaseInterface.ts +0 -198
  218. package/src/storage/StorageManager.ts +0 -113
  219. package/src/storage/duckdb/ConnectionRepository.ts +0 -143
  220. package/src/storage/duckdb/DuckDBConnection.ts +0 -123
  221. package/src/storage/duckdb/DuckDBRepository.ts +0 -208
  222. package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
  223. package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
  224. package/src/storage/duckdb/MaterializationRepository.ts +0 -336
  225. package/src/storage/duckdb/PackageRepository.ts +0 -135
  226. package/src/storage/duckdb/schema.ts +0 -229
  227. package/src/stream_helpers.spec.ts +0 -280
  228. package/src/stream_helpers.ts +0 -162
  229. package/src/telemetry.ts +0 -20
  230. package/src/test_helpers/metrics_harness.ts +0 -166
  231. package/src/theme_key_parity.spec.ts +0 -57
  232. package/src/utils.ts +0 -29
  233. package/tests/fixtures/authorize-compile/model.malloy +0 -9
  234. package/tests/fixtures/authorize-compile/publisher.json +0 -4
  235. package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
  236. package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
  237. package/tests/fixtures/html-pages-test/data.csv +0 -3
  238. package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
  239. package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
  240. package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
  241. package/tests/fixtures/html-pages-test/public/data.json +0 -1
  242. package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
  243. package/tests/fixtures/html-pages-test/public/index.html +0 -9
  244. package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
  245. package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
  246. package/tests/fixtures/html-pages-test/public/slides.html +0 -12
  247. package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
  248. package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
  249. package/tests/fixtures/html-pages-test/publisher.json +0 -5
  250. package/tests/fixtures/html-pages-test/report.malloy +0 -1
  251. package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
  252. package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
  253. package/tests/fixtures/persist-multi-level/publisher.json +0 -5
  254. package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
  255. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
  256. package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
  257. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
  258. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
  259. package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
  260. package/tests/fixtures/persist-test/data/orders.csv +0 -5
  261. package/tests/fixtures/persist-test/persist_test.malloy +0 -11
  262. package/tests/fixtures/persist-test/publisher.json +0 -5
  263. package/tests/fixtures/publisher.config.json +0 -15
  264. package/tests/fixtures/query-givens/data/orders.csv +0 -7
  265. package/tests/fixtures/query-givens/model.malloy +0 -34
  266. package/tests/fixtures/query-givens/publisher.json +0 -5
  267. package/tests/fixtures/xlsx/database.xlsx +0 -0
  268. package/tests/harness/mcp_test_setup.ts +0 -332
  269. package/tests/harness/rest_e2e.ts +0 -68
  270. package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
  271. package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
  272. package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
  273. package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
  274. package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
  275. package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
  276. package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
  277. package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
  278. package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
  279. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
  280. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
  281. package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
  282. package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
  283. package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
  284. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
  285. package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
  286. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
  287. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
  288. package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
  289. package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
  290. package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
  291. package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
  292. package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
  293. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
  294. package/tests/unit/controller/theme.controller.spec.ts +0 -141
  295. package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
  296. package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
  297. package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
  298. package/tests/unit/duckdb/repositories.test.ts +0 -208
  299. package/tests/unit/ducklake/ducklake.test.ts +0 -952
  300. package/tests/unit/service/theme_store.spec.ts +0 -139
  301. package/tsconfig.json +0 -32
@@ -1,127 +0,0 @@
1
- /**
2
- * Per-load phase timing for package loads.
3
- *
4
- * `malloy_package_load_duration` already reports the *total* time to load a
5
- * package, but an operator watching a slow load can't tell whether it's the
6
- * Malloy compile or the connection schema fetches that dominate — the two are
7
- * tuned very differently (CPU/worker sizing vs. warehouse round-trips). These
8
- * histograms split the load into those phases so a single dashboard answers
9
- * "where did the time go?". The worker measures the phases and reports them on
10
- * its result; see `package_load/package_load_worker.ts` and
11
- * `package_load/rpc_wait_accountant.ts`.
12
- *
13
- * Labels: `status` only. Deliberately NOT labelled by package name — unlike
14
- * `malloy_package_load_duration`, which carries `malloy_package_name` — to keep
15
- * cardinality bounded on deployments with many packages. Per-package latency,
16
- * when needed, is better answered by exemplars/traces than a label explosion.
17
- *
18
- * Lazy init for the same reason as `query_cap_metrics.ts` / `query_timeout.ts`:
19
- * instruments created before `setGlobalMeterProvider` bind to a NoOp meter
20
- * (https://github.com/open-telemetry/opentelemetry-js/issues/3505).
21
- */
22
-
23
- import { type Histogram } from "@opentelemetry/api";
24
- import { publisherMeter } from "./telemetry";
25
-
26
- /**
27
- * Explicit bucket boundaries (ms) for the load-duration histograms. OTel's
28
- * default boundaries top out at 10 000 ms, so any load slower than 10 s falls
29
- * in the `+Inf` bucket and `histogram_quantile` can't resolve the tail — which
30
- * is exactly the tail (large/slow packages) worth seeing. These extend to
31
- * 5 minutes while keeping sub-10 ms resolution for the fast common case.
32
- * Shared with the total `malloy_package_load_duration` histogram so all
33
- * load-timing histograms bucket consistently.
34
- */
35
- export const LOAD_DURATION_BUCKETS_MS = [
36
- 1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000,
37
- 120_000, 300_000,
38
- ];
39
-
40
- /** Bucket boundaries for the per-load schema-fetch count. */
41
- const SCHEMA_FETCH_COUNT_BUCKETS = [0, 1, 2, 5, 10, 25, 50, 100, 250, 500];
42
-
43
- /**
44
- * Terminal status of a package load, mirroring the `status` label on the
45
- * existing `malloy_package_load_duration` histogram so the phase metrics slice
46
- * the same way. In practice the phase histograms only ever carry `success`,
47
- * `compilation_error`, or `error` — a `pool_unavailable` failure happens before
48
- * the worker returns any timings, so there is nothing to record for it.
49
- */
50
- export type PackageLoadStatus =
51
- | "success"
52
- | "compilation_error"
53
- | "pool_unavailable"
54
- | "error";
55
-
56
- /** Phase timings a single package load reports (subset of the total). */
57
- export interface PackageLoadPhaseTimings {
58
- /** Ceiling on Malloy compile CPU (compile region minus schema-fetch wait). */
59
- compileDurationMs: number;
60
- /** Wall-clock awaiting proxied connection schema fetches. */
61
- schemaFetchDurationMs: number;
62
- /** Number of proxied connection schema fetches the load drove. */
63
- schemaFetchCount: number;
64
- }
65
-
66
- let compileDuration: Histogram | null = null;
67
- let schemaFetchDuration: Histogram | null = null;
68
- let schemaFetches: Histogram | null = null;
69
-
70
- function ensureInstruments(): void {
71
- if (compileDuration && schemaFetchDuration && schemaFetches) return;
72
- const meter = publisherMeter();
73
- compileDuration ??= meter.createHistogram(
74
- "malloy_package_load_compile_duration",
75
- {
76
- description:
77
- "Per-load Malloy compile time (compile region minus proxied schema-fetch wait; a ceiling on compile CPU). Label: status.",
78
- unit: "ms",
79
- advice: { explicitBucketBoundaries: LOAD_DURATION_BUCKETS_MS },
80
- },
81
- );
82
- schemaFetchDuration ??= meter.createHistogram(
83
- "malloy_package_load_schema_fetch_duration",
84
- {
85
- description:
86
- "Per-load wall-clock awaiting proxied connection schema fetches. Label: status.",
87
- unit: "ms",
88
- advice: { explicitBucketBoundaries: LOAD_DURATION_BUCKETS_MS },
89
- },
90
- );
91
- schemaFetches ??= meter.createHistogram(
92
- "malloy_package_load_schema_fetches",
93
- {
94
- description:
95
- "Per-load count of proxied connection schema fetches. Label: status.",
96
- advice: { explicitBucketBoundaries: SCHEMA_FETCH_COUNT_BUCKETS },
97
- },
98
- );
99
- }
100
-
101
- /**
102
- * Record one load's phase breakdown. Call once the worker has returned timings,
103
- * with the load's terminal `status` (so the phases can be sliced by outcome the
104
- * same way the total duration is). The terms are meaningless for a load that
105
- * never produced a worker result (e.g. a pool failure), so don't call it there.
106
- */
107
- export function recordPackageLoadPhases(
108
- timings: PackageLoadPhaseTimings,
109
- status: PackageLoadStatus,
110
- ): void {
111
- ensureInstruments();
112
- const attrs = { status };
113
- compileDuration?.record(timings.compileDurationMs, attrs);
114
- schemaFetchDuration?.record(timings.schemaFetchDurationMs, attrs);
115
- schemaFetches?.record(timings.schemaFetchCount, attrs);
116
- }
117
-
118
- /**
119
- * Visible for tests. Drops cached instruments so a fresh `MeterProvider`
120
- * (installed via `startMetricsHarness`) captures future emissions. Do NOT call
121
- * from production code.
122
- */
123
- export function resetPackageLoadMetricsForTesting(): void {
124
- compileDuration = null;
125
- schemaFetchDuration = null;
126
- schemaFetches = null;
127
- }
@@ -1,158 +0,0 @@
1
- import { describe, expect, it } from "bun:test";
2
- import * as path from "path";
3
-
4
- import { BadRequestError } from "./errors";
5
- import {
6
- assertSafeEnvironmentPath,
7
- assertSafePackageName,
8
- assertSafeRelativeModelPath,
9
- safeJoinUnderRoot,
10
- } from "./path_safety";
11
-
12
- describe("assertSafePackageName", () => {
13
- it.each([
14
- "pkg",
15
- "test_package",
16
- "test-package",
17
- "TestPackage1",
18
- "test.package.name",
19
- "a",
20
- "x".repeat(255),
21
- ])("accepts %p", (name) => {
22
- expect(() => assertSafePackageName(name)).not.toThrow();
23
- });
24
-
25
- it.each([
26
- ["empty", ""],
27
- ["dot", "."],
28
- ["dot-dot", ".."],
29
- ["leading dot", ".staging"],
30
- ["forward slash", "foo/bar"],
31
- ["backslash", "foo\\bar"],
32
- ["null byte", "foo\0bar"],
33
- ["traversal", "../etc/passwd"],
34
- ["abs", "/etc/passwd"],
35
- ["space", "my pkg"],
36
- ["unicode", "pkg\u202E"],
37
- ["too long", "x".repeat(256)],
38
- ])("rejects %s (%p)", (_label, name) => {
39
- expect(() => assertSafePackageName(name)).toThrow(BadRequestError);
40
- });
41
-
42
- it.each([
43
- ["number", 42],
44
- ["null", null],
45
- ["undefined", undefined],
46
- ["object", { name: "pkg" }],
47
- ])("rejects non-string %s (%p)", (_label, value) => {
48
- expect(() => assertSafePackageName(value)).toThrow(BadRequestError);
49
- });
50
- });
51
-
52
- describe("assertSafeRelativeModelPath", () => {
53
- it.each([
54
- "model.malloy",
55
- "models/foo.malloy",
56
- "a/b/c/d.malloynb",
57
- "deep/nested/file_name-1.malloy",
58
- ])("accepts %p", (modelPath) => {
59
- expect(() => assertSafeRelativeModelPath(modelPath)).not.toThrow();
60
- });
61
-
62
- it.each([
63
- ["empty", ""],
64
- ["leading slash (absolute)", "/etc/passwd"],
65
- ["traversal", "../etc/passwd"],
66
- ["embedded traversal", "models/../../../etc/passwd"],
67
- ["embedded dot segment", "models/./foo.malloy"],
68
- ["double slash", "models//foo.malloy"],
69
- ["trailing slash", "models/foo/"],
70
- ["backslash", "models\\foo.malloy"],
71
- ["null byte", "models/foo\0.malloy"],
72
- ["dotfile segment", ".staging/foo.malloy"],
73
- ["dotfile leaf", "models/.hidden.malloy"],
74
- ])("rejects %s (%p)", (_label, modelPath) => {
75
- expect(() => assertSafeRelativeModelPath(modelPath)).toThrow(
76
- BadRequestError,
77
- );
78
- });
79
-
80
- it("rejects non-string inputs", () => {
81
- expect(() => assertSafeRelativeModelPath(undefined)).toThrow(
82
- BadRequestError,
83
- );
84
- expect(() => assertSafeRelativeModelPath(123)).toThrow(BadRequestError);
85
- });
86
- });
87
-
88
- describe("assertSafeEnvironmentPath", () => {
89
- it.each([
90
- "/etc/publisher",
91
- "/var/lib/publisher/env1",
92
- "/Users/me/data",
93
- "/a",
94
- "C:\\Users\\me\\publisher",
95
- "C:/Users/me/publisher",
96
- ])("accepts %p", (p) => {
97
- expect(() => assertSafeEnvironmentPath(p)).not.toThrow();
98
- });
99
-
100
- it.each([
101
- ["empty", ""],
102
- ["relative", "publisher/data"],
103
- ["traversal in middle", "/var/lib/../../etc/passwd"],
104
- ["traversal at end", "/var/lib/publisher/.."],
105
- ["null byte", "/var/lib/publisher\0"],
106
- ["bare dot-dot", ".."],
107
- ["bare dot", "."],
108
- ["too long", "/" + "a".repeat(5000)],
109
- ])("rejects %s (%p)", (_label, p) => {
110
- expect(() => assertSafeEnvironmentPath(p)).toThrow(BadRequestError);
111
- });
112
-
113
- it("rejects non-string inputs", () => {
114
- expect(() => assertSafeEnvironmentPath(undefined)).toThrow(
115
- BadRequestError,
116
- );
117
- expect(() => assertSafeEnvironmentPath(null)).toThrow(BadRequestError);
118
- expect(() => assertSafeEnvironmentPath(42)).toThrow(BadRequestError);
119
- });
120
- });
121
-
122
- describe("safeJoinUnderRoot", () => {
123
- const root = "/tmp/test-root";
124
-
125
- it("returns the resolved root when joined with no segments", () => {
126
- expect(safeJoinUnderRoot(root)).toBe(path.resolve(root));
127
- });
128
-
129
- it("joins safe segments into a path under root", () => {
130
- expect(safeJoinUnderRoot(root, "pkg", "model.malloy")).toBe(
131
- path.resolve(root, "pkg", "model.malloy"),
132
- );
133
- });
134
-
135
- it("throws when traversal escapes the root", () => {
136
- expect(() => safeJoinUnderRoot(root, "..")).toThrow(BadRequestError);
137
- expect(() => safeJoinUnderRoot(root, "..", "etc", "passwd")).toThrow(
138
- BadRequestError,
139
- );
140
- expect(() => safeJoinUnderRoot(root, "pkg", "..", "..", "etc")).toThrow(
141
- BadRequestError,
142
- );
143
- });
144
-
145
- it("throws when an absolute segment overrides the root", () => {
146
- expect(() => safeJoinUnderRoot(root, "/etc/passwd")).toThrow(
147
- BadRequestError,
148
- );
149
- });
150
-
151
- it("does NOT match a sibling directory with the same prefix", () => {
152
- // path.resolve("/tmp/test-root", "../test-root-bad") -> "/tmp/test-root-bad"
153
- // which starts with "/tmp/test-root" textually but is NOT a child.
154
- expect(() => safeJoinUnderRoot(root, "..", "test-root-bad")).toThrow(
155
- BadRequestError,
156
- );
157
- });
158
- });
@@ -1,146 +0,0 @@
1
- import * as path from "path";
2
-
3
- import { BadRequestError } from "./errors";
4
-
5
- /**
6
- * Path-safety helpers used by `Environment` (and any other service that
7
- * builds an on-disk path from request data) to defend against directory
8
- * traversal. The intent is two-fold:
9
- *
10
- * 1. **Source-side allowlist**: `assertSafePackageName` /
11
- * `assertSafeRelativeModelPath` reject hostile inputs (`..`, leading
12
- * `/`, `\`, NUL, dotfiles) at the entry of every public service
13
- * method before any path-construction happens. These throw
14
- * `BadRequestError` so the controller layer's error mapper returns
15
- * HTTP 400.
16
- *
17
- * 2. **Sink-side containment**: `safeJoinUnderRoot` joins, resolves,
18
- * and verifies the result is strictly within the supplied root.
19
- * Even if a future caller forgets the source-side check, the sink
20
- * refuses to hand back an escaping path. This is the standard
21
- * "resolve-and-contain" pattern that CodeQL's `js/path-injection`
22
- * query recognises as a sanitizer.
23
- */
24
-
25
- // Single path segment: ASCII letters, digits, `-`, `_`, `.`. No leading
26
- // `.` so internal sibling dirs (`.staging`, `.retired`) and editor /
27
- // VCS dirs can't be addressed by name from outside.
28
- const SAFE_NAME_RE = /^(?!\.\.?$)(?!\.)[A-Za-z0-9._-]{1,255}$/;
29
-
30
- const MAX_MODEL_PATH_LEN = 1024;
31
-
32
- // An environment path is server-controlled (config / disk-derived), but
33
- // CodeQL conservatively treats it as tainted because Express handlers on
34
- // the same class touch user input. The combined regex test +
35
- // `..` / NUL / length check at the constructor gate is the sanitizer
36
- // barrier the `js/path-injection` query recognises. Printable ASCII
37
- // only; absolute POSIX-or-Windows path; no `..`, no NUL.
38
- const SAFE_ENVIRONMENT_PATH_RE = /^(?:\/|[A-Za-z]:[\\/])[\x20-\x7E]*$/;
39
- const MAX_ENVIRONMENT_PATH_LEN = 4096;
40
-
41
- /**
42
- * Reject anything that isn't a plausible single-segment package name.
43
- * The allowlist is deliberately conservative — every existing test and
44
- * production package name we've seen fits within it, and tightening
45
- * here costs nothing.
46
- */
47
- export function assertSafePackageName(
48
- packageName: unknown,
49
- ): asserts packageName is string {
50
- if (typeof packageName !== "string" || !SAFE_NAME_RE.test(packageName)) {
51
- throw new BadRequestError(
52
- `Invalid package name: must be 1-255 characters of letters, digits, "-", "_", or "." and must not start with "."`,
53
- );
54
- }
55
- }
56
-
57
- /**
58
- * Reject anything that isn't a plausible *relative* path to a model
59
- * file inside a package directory. Forward slashes are allowed (models
60
- * live in subdirectories like `models/foo.malloy`); backslashes,
61
- * absolute paths, NUL bytes, and `..` / `.` segments are not.
62
- */
63
- export function assertSafeRelativeModelPath(
64
- modelPath: unknown,
65
- ): asserts modelPath is string {
66
- if (
67
- typeof modelPath !== "string" ||
68
- modelPath.length === 0 ||
69
- modelPath.length > MAX_MODEL_PATH_LEN ||
70
- modelPath.includes("\0") ||
71
- modelPath.includes("\\") ||
72
- path.isAbsolute(modelPath) ||
73
- modelPath.startsWith("/")
74
- ) {
75
- throw new BadRequestError(`Invalid model path`);
76
- }
77
-
78
- const segments = modelPath.split("/");
79
- for (const segment of segments) {
80
- if (segment === "" || segment === "." || segment === "..") {
81
- throw new BadRequestError(`Invalid model path`);
82
- }
83
- if (segment.startsWith(".")) {
84
- throw new BadRequestError(`Invalid model path`);
85
- }
86
- }
87
- }
88
-
89
- /**
90
- * Reject anything that doesn't look like a server-controlled absolute
91
- * filesystem path. Applied to `environmentPath` at the constructor
92
- * gate so all downstream `path.join(this.environmentPath, …)` sites
93
- * see a value that has cleared an allowlist check — the canonical
94
- * sanitizer-barrier pattern CodeQL's `js/path-injection` query
95
- * recognises.
96
- */
97
- export function assertSafeEnvironmentPath(
98
- environmentPath: unknown,
99
- ): asserts environmentPath is string {
100
- if (typeof environmentPath !== "string") {
101
- throw new BadRequestError(`Invalid environment path: must be a string`);
102
- }
103
- if (
104
- environmentPath.length === 0 ||
105
- environmentPath.length > MAX_ENVIRONMENT_PATH_LEN
106
- ) {
107
- throw new BadRequestError(`Invalid environment path: bad length`);
108
- }
109
- if (environmentPath.indexOf("\0") !== -1) {
110
- throw new BadRequestError(`Invalid environment path: contains NUL byte`);
111
- }
112
- // Sanitizer barrier in the shape `x.indexOf("..") !== -1` that the
113
- // CodeQL `js/path-injection` query recognises as a traversal guard.
114
- if (environmentPath.indexOf("..") !== -1) {
115
- throw new BadRequestError(
116
- `Invalid environment path: contains ".." traversal segment`,
117
- );
118
- }
119
- if (!SAFE_ENVIRONMENT_PATH_RE.test(environmentPath)) {
120
- throw new BadRequestError(
121
- `Invalid environment path: must be an absolute path of printable ASCII characters`,
122
- );
123
- }
124
- }
125
-
126
- /**
127
- * Resolve `path.join(root, ...segments)` and verify the result lives
128
- * strictly inside `root` (or is `root` itself). Throws
129
- * `BadRequestError` if the resolved path escapes the root via `..`,
130
- * absolute segments, or symlink-style trickery in the input.
131
- *
132
- * Callers should still run `assertSafePackageName` / similar on
133
- * user-controlled segments first — this helper is the second line of
134
- * defense, not the first.
135
- */
136
- export function safeJoinUnderRoot(root: string, ...segments: string[]): string {
137
- const resolvedRoot = path.resolve(root);
138
- const joined = path.resolve(resolvedRoot, ...segments);
139
- const rootWithSep = resolvedRoot.endsWith(path.sep)
140
- ? resolvedRoot
141
- : resolvedRoot + path.sep;
142
- if (joined !== resolvedRoot && !joined.startsWith(rootWithSep)) {
143
- throw new BadRequestError(`Resolved path is outside of root`);
144
- }
145
- return joined;
146
- }
@@ -1,223 +0,0 @@
1
- import { describe, expect, it } from "bun:test";
2
- import { redactPgSecrets } from "./pg_helpers";
3
-
4
- describe("redactPgSecrets", () => {
5
- it("redacts bare password values", () => {
6
- expect(redactPgSecrets("host=h password=hunter2 dbname=d")).toBe(
7
- "host=h password=*** dbname=d",
8
- );
9
- });
10
-
11
- it("redacts single-quoted password values", () => {
12
- expect(redactPgSecrets("host=h password='s3 cret' dbname=d")).toBe(
13
- "host=h password=*** dbname=d",
14
- );
15
- });
16
-
17
- it("leaves non-secret content alone", () => {
18
- expect(redactPgSecrets("user=alice dbname=billing")).toBe(
19
- "user=alice dbname=billing",
20
- );
21
- });
22
-
23
- it("redacts the password in a URL-form connection string", () => {
24
- expect(redactPgSecrets("postgres://user:pass@host/db")).toBe(
25
- "postgres://user:***@host/db",
26
- );
27
- });
28
-
29
- it("preserves port and query string around a URI password", () => {
30
- expect(redactPgSecrets("postgresql://u:p@h:5432/d?sslmode=require")).toBe(
31
- "postgresql://u:***@h:5432/d?sslmode=require",
32
- );
33
- });
34
-
35
- it("redacts a percent-encoded password whole", () => {
36
- expect(redactPgSecrets("postgres://u:p%40ss@h/d")).toBe(
37
- "postgres://u:***@h/d",
38
- );
39
- });
40
-
41
- it("redacts a literal @ in the password up to the last @", () => {
42
- expect(redactPgSecrets("postgres://u:p@ss@h/d")).toBe(
43
- "postgres://u:***@h/d",
44
- );
45
- });
46
-
47
- it("redacts a password containing colons whole", () => {
48
- expect(redactPgSecrets("postgres://u:pa:ss@h/d")).toBe(
49
- "postgres://u:***@h/d",
50
- );
51
- });
52
-
53
- it("redacts a raw slash in a pg password via the mop-up pass", () => {
54
- expect(redactPgSecrets("postgres://u:pa/ss@h/d")).toBe(
55
- "postgres://u:***@h/d",
56
- );
57
- });
58
-
59
- it("redacts the password when the username is empty", () => {
60
- expect(redactPgSecrets("postgres://:secret@h/d")).toBe(
61
- "postgres://:***@h/d",
62
- );
63
- });
64
-
65
- it("leaves IPv6 hosts intact", () => {
66
- expect(redactPgSecrets("postgres://u:p@[::1]:5432/d")).toBe(
67
- "postgres://u:***@[::1]:5432/d",
68
- );
69
- });
70
-
71
- it("does not redact a username-only userinfo", () => {
72
- expect(redactPgSecrets("postgres://user@host/db")).toBe(
73
- "postgres://user@host/db",
74
- );
75
- });
76
-
77
- it("leaves URIs without userinfo alone", () => {
78
- expect(redactPgSecrets("postgres://host/db")).toBe("postgres://host/db");
79
- expect(redactPgSecrets("postgres://host:5432/db")).toBe(
80
- "postgres://host:5432/db",
81
- );
82
- });
83
-
84
- it("does not mangle a passwordless URI with :...@ in its query", () => {
85
- expect(redactPgSecrets("postgres://h?options=endpoint:foo@bar")).toBe(
86
- "postgres://h?options=endpoint:foo@bar",
87
- );
88
- expect(redactPgSecrets("postgres://h/d?x=a@b")).toBe(
89
- "postgres://h/d?x=a@b",
90
- );
91
- });
92
-
93
- it("leaves email addresses in prose alone", () => {
94
- expect(redactPgSecrets("contact admin@example.com for help")).toBe(
95
- "contact admin@example.com for help",
96
- );
97
- });
98
-
99
- it("does not mangle https URLs with @ in the path", () => {
100
- expect(
101
- redactPgSecrets("fetched https://github.com:443/@scope/pkg ok"),
102
- ).toBe("fetched https://github.com:443/@scope/pkg ok");
103
- });
104
-
105
- it("redacts every URI in a message independently", () => {
106
- expect(
107
- redactPgSecrets(
108
- "tried postgres://u1:p1@h1/d1 then postgres://u2:p2@h2/d2",
109
- ),
110
- ).toBe("tried postgres://u1:***@h1/d1 then postgres://u2:***@h2/d2");
111
- });
112
-
113
- it("keeps comma-joined URIs in one token separate", () => {
114
- expect(
115
- redactPgSecrets(
116
- "candidates: postgres://u1:p1@h1/d1,postgres://u2:p2@h2/d2",
117
- ),
118
- ).toBe("candidates: postgres://u1:***@h1/d1,postgres://u2:***@h2/d2");
119
- });
120
-
121
- it("leaves trailing sentence punctuation alone", () => {
122
- expect(redactPgSecrets("could not connect to postgres://u:p@h/d.")).toBe(
123
- "could not connect to postgres://u:***@h/d.",
124
- );
125
- });
126
-
127
- it("handles keyword and URI forms in one message", () => {
128
- expect(
129
- redactPgSecrets(
130
- "attach failed: host=h password=x and fallback postgres://u:p@h2/d",
131
- ),
132
- ).toBe(
133
- "attach failed: host=h password=*** and fallback postgres://u:***@h2/d",
134
- );
135
- });
136
-
137
- it("keeps the URI intact when a password contains 'password='", () => {
138
- // URI passes run before the keyword pass; keyword-first would eat
139
- // the @host/db tail via \S+.
140
- expect(redactPgSecrets("postgres://u:password=abc@h/d")).toBe(
141
- "postgres://u:***@h/d",
142
- );
143
- });
144
-
145
- it("over-redacts rather than leaks a ?password= query parameter", () => {
146
- // Pre-existing keyword-pass behavior: \S+ eats through &sslmode=...
147
- // Over-redaction, never a leak.
148
- expect(
149
- redactPgSecrets("postgres://h/d?password=abc&sslmode=require"),
150
- ).toBe("postgres://h/d?password=***");
151
- });
152
-
153
- it("matches uppercase schemes", () => {
154
- expect(redactPgSecrets("POSTGRES://U:Secret@H/D")).toBe(
155
- "POSTGRES://U:***@H/D",
156
- );
157
- });
158
-
159
- it("redacts non-pg scheme userinfo passwords too", () => {
160
- expect(redactPgSecrets("amqp://guest:guest@rabbit:5672/vhost")).toBe(
161
- "amqp://guest:***@rabbit:5672/vhost",
162
- );
163
- });
164
-
165
- it("redacts an Azure user@servername username form", () => {
166
- // Azure single-server usernames carry a literal @ (user@servername);
167
- // userinfo splits at the LAST @, so the password after the first : is
168
- // redacted while the @-bearing username stays intact.
169
- expect(
170
- redactPgSecrets(
171
- "postgres://myadmin@myserver:Zx7SeCReT9q@myserver.postgres.database.azure.com:5432/db",
172
- ),
173
- ).toBe(
174
- "postgres://myadmin@myserver:***@myserver.postgres.database.azure.com:5432/db",
175
- );
176
- });
177
-
178
- it("redacts an @-in-username form whose password also contains @", () => {
179
- expect(redactPgSecrets("postgres://myadmin@srv:p@ss@host/db")).toBe(
180
- "postgres://myadmin@srv:***@host/db",
181
- );
182
- });
183
-
184
- it("redacts an @-in-username form with a raw / in the password", () => {
185
- expect(redactPgSecrets("postgresql://u@srv:sec/ret@h/d")).toBe(
186
- "postgresql://u@srv:***@h/d",
187
- );
188
- });
189
-
190
- it("redacts a scheme abutting a word char", () => {
191
- // No leading anchor on the scheme, so a scheme glued to a preceding
192
- // word char (an invalid-scheme prefix, or a stray digit) still redacts.
193
- expect(redactPgSecrets("x_postgres://u:secretpw@h/d")).toBe(
194
- "x_postgres://u:***@h/d",
195
- );
196
- expect(redactPgSecrets("3postgres://u:secretpw@h/d")).toBe(
197
- "3postgres://u:***@h/d",
198
- );
199
- });
200
-
201
- it("never reveals a secret when re-applied (monotonic)", () => {
202
- // redactPgSecrets is not strictly idempotent in general (re-applying
203
- // can redact further, e.g. an unquoted *** left by the keyword pass),
204
- // but re-application must never turn a redacted secret back into
205
- // cleartext.
206
- const input = "postgres://u:p@ss@h/d and host=h password=hunter2";
207
- const once = redactPgSecrets(input);
208
- const twice = redactPgSecrets(once);
209
- expect(once).not.toContain("hunter2");
210
- expect(twice).not.toContain("hunter2");
211
- expect(twice).not.toContain("p@ss");
212
- });
213
-
214
- it("redacts the DSN inside a real DuckDB attach error message", () => {
215
- // Shape observed from DuckDB's postgres extension: the full DSN is
216
- // echoed verbatim inside the IO Error text.
217
- const msg =
218
- 'IO Error: Unable to connect to Postgres at "postgres://alice:supersecretpw@127.0.0.1:5432/mydb": connection to server at "127.0.0.1", port 5432 failed: Connection refused';
219
- const redacted = redactPgSecrets(msg);
220
- expect(redacted).toContain("postgres://alice:***@127.0.0.1:5432/mydb");
221
- expect(redacted).not.toContain("supersecretpw");
222
- });
223
- });