@malloy-publisher/server 0.0.234 → 0.0.236

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (318) hide show
  1. package/dist/app/api-doc.yaml +47 -17
  2. package/dist/app/assets/{EnvironmentPage-DTZQ4Gxc.js → EnvironmentPage-BsAnavYN.js} +1 -1
  3. package/dist/app/assets/{HomePage-C5mlDPXK.js → HomePage-CADE138j.js} +1 -1
  4. package/dist/app/assets/{LightMode-DGNmhG0u.js → LightMode-Cfh7KzN8.js} +1 -1
  5. package/dist/app/assets/{MainPage-CVL_wmP4.js → MainPage-CO3pRlnV.js} +1 -1
  6. package/dist/app/assets/{MaterializationsPage-DmzMBCpy.js → MaterializationsPage-p9YjkRXZ.js} +1 -1
  7. package/dist/app/assets/{ModelPage-Dbvf4QbB.js → ModelPage-C1OSTv-x.js} +1 -1
  8. package/dist/app/assets/{PackagePage-DxdHc2Qs.js → PackagePage-e4kN75YR.js} +1 -1
  9. package/dist/app/assets/{RouteError-OJdT4tCd.js → RouteError-CzbfOkng.js} +1 -1
  10. package/dist/app/assets/{ThemeEditorPage-Bk7s0KXY.js → ThemeEditorPage-CciagFTq.js} +1 -1
  11. package/dist/app/assets/{WorkbookPage-j_vCWdN3.js → WorkbookPage-DNWmkCXa.js} +1 -1
  12. package/dist/app/assets/{core-Rj_4rRnA.es-DoIfLxDJ.js → core-Rj_4rRnA.es-BZyvITuO.js} +1 -1
  13. package/dist/app/assets/{index-B_jKMR35.js → index-CH2AcDzc.js} +4 -4
  14. package/dist/app/assets/{index-hw-xn0X7.js → index-DQa463gC.js} +2 -2
  15. package/dist/app/assets/{index-DWIe_hK0.js → index-DQpV7MyA.js} +1 -1
  16. package/dist/app/assets/{index-BabP-V-S.js → index-DzaYbhnD.js} +1 -1
  17. package/dist/app/assets/{index-D-rDyK11.js → index-VBbcc8s6.js} +1 -1
  18. package/dist/app/index.html +1 -1
  19. package/dist/package_load_worker.mjs +8 -1
  20. package/dist/server.mjs +471 -54
  21. package/package.json +16 -13
  22. package/.eslintrc.json +0 -44
  23. package/.prettierignore +0 -1
  24. package/.prettierrc +0 -4
  25. package/build.ts +0 -97
  26. package/eslint.config.mjs +0 -76
  27. package/k6-tests/README.md +0 -144
  28. package/k6-tests/bun.lock +0 -797
  29. package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
  30. package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
  31. package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
  32. package/k6-tests/load-test/load-test-crud.ts +0 -142
  33. package/k6-tests/load-test/load-test.ts +0 -784
  34. package/k6-tests/package.json +0 -27
  35. package/k6-tests/scripts/run-with-otel.sh +0 -50
  36. package/k6-tests/smoke-test/smoke-test.ts +0 -158
  37. package/k6-tests/tsconfig.json +0 -29
  38. package/k6-tests/utils/client_factory.ts +0 -128
  39. package/k6-tests/utils/common.ts +0 -489
  40. package/k6-tests/utils/logger.ts +0 -62
  41. package/k6-tests/utils/types.d.ts +0 -35
  42. package/malloy_mcp.dxt +0 -0
  43. package/publisher.config.example.bigquery.json +0 -33
  44. package/publisher.config.example.duckdb.json +0 -23
  45. package/publisher.config.json +0 -23
  46. package/scripts/bake-duckdb-extensions.js +0 -110
  47. package/src/config.spec.ts +0 -1535
  48. package/src/config.theme.spec.ts +0 -178
  49. package/src/config.ts +0 -1142
  50. package/src/constants.ts +0 -105
  51. package/src/controller/compile.controller.ts +0 -40
  52. package/src/controller/connection.controller.spec.ts +0 -849
  53. package/src/controller/connection.controller.ts +0 -901
  54. package/src/controller/database.controller.ts +0 -24
  55. package/src/controller/materialization.controller.spec.ts +0 -293
  56. package/src/controller/materialization.controller.ts +0 -319
  57. package/src/controller/model.controller.ts +0 -164
  58. package/src/controller/package.controller.spec.ts +0 -273
  59. package/src/controller/package.controller.ts +0 -306
  60. package/src/controller/query.controller.ts +0 -152
  61. package/src/controller/theme.controller.ts +0 -83
  62. package/src/controller/watch-mode.controller.ts +0 -243
  63. package/src/data_styles.ts +0 -71
  64. package/src/default-publisher.config.json +0 -23
  65. package/src/ducklake_version.spec.ts +0 -163
  66. package/src/ducklake_version.ts +0 -153
  67. package/src/errors.spec.ts +0 -119
  68. package/src/errors.ts +0 -237
  69. package/src/filter_deprecation.spec.ts +0 -64
  70. package/src/filter_deprecation.ts +0 -42
  71. package/src/health.spec.ts +0 -123
  72. package/src/health.ts +0 -233
  73. package/src/heap_check.spec.ts +0 -144
  74. package/src/heap_check.ts +0 -144
  75. package/src/instrumentation.ts +0 -214
  76. package/src/json_utils.spec.ts +0 -51
  77. package/src/json_utils.ts +0 -33
  78. package/src/logger.spec.ts +0 -210
  79. package/src/logger.ts +0 -251
  80. package/src/malloy_pin_prereqs.spec.ts +0 -25
  81. package/src/materialization_metrics.spec.ts +0 -169
  82. package/src/materialization_metrics.ts +0 -346
  83. package/src/mcp/error_messages.spec.ts +0 -35
  84. package/src/mcp/error_messages.ts +0 -183
  85. package/src/mcp/handler_utils.spec.ts +0 -108
  86. package/src/mcp/handler_utils.ts +0 -251
  87. package/src/mcp/mcp_constants.ts +0 -6
  88. package/src/mcp/query_envelope.spec.ts +0 -229
  89. package/src/mcp/query_envelope.ts +0 -240
  90. package/src/mcp/server.protocol.spec.ts +0 -250
  91. package/src/mcp/server.ts +0 -87
  92. package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
  93. package/src/mcp/skills/build_skills_bundle.ts +0 -176
  94. package/src/mcp/skills/skills_bundle.json +0 -1
  95. package/src/mcp/skills/skills_bundle.spec.ts +0 -171
  96. package/src/mcp/tool_response.spec.ts +0 -108
  97. package/src/mcp/tool_response.ts +0 -138
  98. package/src/mcp/tools/compile_tool.spec.ts +0 -315
  99. package/src/mcp/tools/compile_tool.ts +0 -208
  100. package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
  101. package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
  102. package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
  103. package/src/mcp/tools/docs_search_tool.ts +0 -128
  104. package/src/mcp/tools/embedding_index.spec.ts +0 -1236
  105. package/src/mcp/tools/embedding_index.ts +0 -808
  106. package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
  107. package/src/mcp/tools/execute_query_tool.ts +0 -316
  108. package/src/mcp/tools/get_context_eval.ts +0 -275
  109. package/src/mcp/tools/get_context_tool.spec.ts +0 -592
  110. package/src/mcp/tools/get_context_tool.ts +0 -585
  111. package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
  112. package/src/mcp/tools/reload_package_tool.ts +0 -132
  113. package/src/mcp_config.spec.ts +0 -919
  114. package/src/mcp_config.ts +0 -425
  115. package/src/oom_guards.integration.spec.ts +0 -269
  116. package/src/package_load/package_load_pool.spec.ts +0 -252
  117. package/src/package_load/package_load_pool.ts +0 -931
  118. package/src/package_load/package_load_worker.ts +0 -1034
  119. package/src/package_load/protocol.ts +0 -364
  120. package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
  121. package/src/package_load/rpc_wait_accountant.ts +0 -76
  122. package/src/package_load_metrics.spec.ts +0 -114
  123. package/src/package_load_metrics.ts +0 -127
  124. package/src/path_safety.spec.ts +0 -158
  125. package/src/path_safety.ts +0 -146
  126. package/src/pg_helpers.spec.ts +0 -223
  127. package/src/pg_helpers.ts +0 -52
  128. package/src/query_cap_metrics.spec.ts +0 -89
  129. package/src/query_cap_metrics.ts +0 -116
  130. package/src/query_concurrency.spec.ts +0 -247
  131. package/src/query_concurrency.ts +0 -237
  132. package/src/query_metadata_metrics.ts +0 -49
  133. package/src/query_param_utils.ts +0 -29
  134. package/src/query_timeout.spec.ts +0 -224
  135. package/src/query_timeout.ts +0 -179
  136. package/src/runtime/publisher.js +0 -328
  137. package/src/server-old.ts +0 -988
  138. package/src/server.ts +0 -2059
  139. package/src/service/annotations.spec.ts +0 -118
  140. package/src/service/annotations.ts +0 -91
  141. package/src/service/authorize.spec.ts +0 -154
  142. package/src/service/authorize.ts +0 -498
  143. package/src/service/authorize_integration.spec.ts +0 -2041
  144. package/src/service/build_plan.spec.ts +0 -685
  145. package/src/service/build_plan.ts +0 -797
  146. package/src/service/compile_authorize.spec.ts +0 -85
  147. package/src/service/compile_fragment_techniques.spec.ts +0 -156
  148. package/src/service/connection.spec.ts +0 -2360
  149. package/src/service/connection.ts +0 -2187
  150. package/src/service/connection_config.spec.ts +0 -734
  151. package/src/service/connection_config.ts +0 -899
  152. package/src/service/connection_federation.spec.ts +0 -184
  153. package/src/service/connection_fingerprint.spec.ts +0 -102
  154. package/src/service/connection_service.spec.ts +0 -711
  155. package/src/service/connection_service.ts +0 -291
  156. package/src/service/cron_evaluator.spec.ts +0 -79
  157. package/src/service/cron_evaluator.ts +0 -105
  158. package/src/service/db_utils.spec.ts +0 -1085
  159. package/src/service/db_utils.ts +0 -1502
  160. package/src/service/duckdb_instance_isolation.spec.ts +0 -137
  161. package/src/service/ducklake_lazy_attach.spec.ts +0 -110
  162. package/src/service/embedding_provider.spec.ts +0 -329
  163. package/src/service/embedding_provider.ts +0 -236
  164. package/src/service/environment.ts +0 -2191
  165. package/src/service/environment_admission.spec.ts +0 -344
  166. package/src/service/environment_compile.spec.ts +0 -197
  167. package/src/service/environment_store.spec.ts +0 -2274
  168. package/src/service/environment_store.ts +0 -2434
  169. package/src/service/environment_store_anchoring.spec.ts +0 -107
  170. package/src/service/environment_store_clone.spec.ts +0 -350
  171. package/src/service/explore_visibility.spec.ts +0 -434
  172. package/src/service/exports_probe.spec.ts +0 -107
  173. package/src/service/extension_fetch_policy.spec.ts +0 -256
  174. package/src/service/filter.spec.ts +0 -458
  175. package/src/service/filter.ts +0 -341
  176. package/src/service/filter_bypass.spec.ts +0 -418
  177. package/src/service/filter_integration.spec.ts +0 -1004
  178. package/src/service/freshness.spec.ts +0 -183
  179. package/src/service/freshness.ts +0 -112
  180. package/src/service/gcs_s3_utils.ts +0 -427
  181. package/src/service/given.ts +0 -105
  182. package/src/service/givens_integration.spec.ts +0 -219
  183. package/src/service/manifest_loader.spec.ts +0 -189
  184. package/src/service/manifest_loader.ts +0 -156
  185. package/src/service/materialization_build_session.spec.ts +0 -435
  186. package/src/service/materialization_build_session.ts +0 -681
  187. package/src/service/materialization_config_validation.spec.ts +0 -99
  188. package/src/service/materialization_config_validation.ts +0 -120
  189. package/src/service/materialization_eligibility.spec.ts +0 -158
  190. package/src/service/materialization_eligibility.ts +0 -305
  191. package/src/service/materialization_schedule_surface.spec.ts +0 -256
  192. package/src/service/materialization_scheduler.spec.ts +0 -289
  193. package/src/service/materialization_scheduler.ts +0 -304
  194. package/src/service/materialization_serve_transform.spec.ts +0 -1003
  195. package/src/service/materialization_serve_transform.ts +0 -779
  196. package/src/service/materialization_service.spec.ts +0 -2373
  197. package/src/service/materialization_service.ts +0 -2342
  198. package/src/service/materialization_test_fixtures.ts +0 -249
  199. package/src/service/model.spec.ts +0 -1157
  200. package/src/service/model.ts +0 -3370
  201. package/src/service/model_limits.spec.ts +0 -209
  202. package/src/service/model_limits.ts +0 -131
  203. package/src/service/model_storage_serve.spec.ts +0 -193
  204. package/src/service/model_storage_serve_joins.spec.ts +0 -193
  205. package/src/service/package.spec.ts +0 -612
  206. package/src/service/package.ts +0 -1799
  207. package/src/service/package_manifest.spec.ts +0 -230
  208. package/src/service/package_manifest.ts +0 -241
  209. package/src/service/package_memory_governor.spec.ts +0 -173
  210. package/src/service/package_memory_governor.ts +0 -233
  211. package/src/service/package_quote_bound_tables.spec.ts +0 -171
  212. package/src/service/package_race.spec.ts +0 -208
  213. package/src/service/package_reload_safety.spec.ts +0 -193
  214. package/src/service/package_rollback.spec.ts +0 -190
  215. package/src/service/package_worker_path.spec.ts +0 -645
  216. package/src/service/path_injection.spec.ts +0 -39
  217. package/src/service/persist_annotation_validation.spec.ts +0 -89
  218. package/src/service/persist_annotation_validation.ts +0 -59
  219. package/src/service/persistence_policy.spec.ts +0 -374
  220. package/src/service/proxy.spec.ts +0 -414
  221. package/src/service/proxy.ts +0 -248
  222. package/src/service/query_boundary.spec.ts +0 -470
  223. package/src/service/query_metadata.spec.ts +0 -408
  224. package/src/service/query_metadata.ts +0 -492
  225. package/src/service/query_metadata_identity.spec.ts +0 -149
  226. package/src/service/query_text.spec.ts +0 -79
  227. package/src/service/query_text.ts +0 -43
  228. package/src/service/quoting.spec.ts +0 -135
  229. package/src/service/quoting.ts +0 -78
  230. package/src/service/resolve_environment.ts +0 -15
  231. package/src/service/restricted_mode.spec.ts +0 -299
  232. package/src/service/source_extraction.ts +0 -225
  233. package/src/service/theme_store.ts +0 -199
  234. package/src/storage/DatabaseInterface.ts +0 -198
  235. package/src/storage/StorageManager.ts +0 -113
  236. package/src/storage/duckdb/ConnectionRepository.ts +0 -143
  237. package/src/storage/duckdb/DuckDBConnection.ts +0 -123
  238. package/src/storage/duckdb/DuckDBRepository.ts +0 -208
  239. package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
  240. package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
  241. package/src/storage/duckdb/MaterializationRepository.ts +0 -336
  242. package/src/storage/duckdb/PackageRepository.ts +0 -135
  243. package/src/storage/duckdb/schema.ts +0 -229
  244. package/src/stream_helpers.spec.ts +0 -280
  245. package/src/stream_helpers.ts +0 -162
  246. package/src/telemetry.ts +0 -20
  247. package/src/test_helpers/metrics_harness.ts +0 -166
  248. package/src/theme_key_parity.spec.ts +0 -57
  249. package/src/utils.ts +0 -29
  250. package/tests/fixtures/authorize-compile/model.malloy +0 -9
  251. package/tests/fixtures/authorize-compile/publisher.json +0 -4
  252. package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
  253. package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
  254. package/tests/fixtures/html-pages-test/data.csv +0 -3
  255. package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
  256. package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
  257. package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
  258. package/tests/fixtures/html-pages-test/public/data.json +0 -1
  259. package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
  260. package/tests/fixtures/html-pages-test/public/index.html +0 -9
  261. package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
  262. package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
  263. package/tests/fixtures/html-pages-test/public/slides.html +0 -12
  264. package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
  265. package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
  266. package/tests/fixtures/html-pages-test/publisher.json +0 -5
  267. package/tests/fixtures/html-pages-test/report.malloy +0 -1
  268. package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
  269. package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
  270. package/tests/fixtures/persist-multi-level/publisher.json +0 -5
  271. package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
  272. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
  273. package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
  274. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
  275. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
  276. package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
  277. package/tests/fixtures/persist-test/data/orders.csv +0 -5
  278. package/tests/fixtures/persist-test/persist_test.malloy +0 -11
  279. package/tests/fixtures/persist-test/publisher.json +0 -5
  280. package/tests/fixtures/publisher.config.json +0 -15
  281. package/tests/fixtures/query-givens/data/orders.csv +0 -7
  282. package/tests/fixtures/query-givens/model.malloy +0 -34
  283. package/tests/fixtures/query-givens/publisher.json +0 -5
  284. package/tests/fixtures/xlsx/database.xlsx +0 -0
  285. package/tests/harness/mcp_test_setup.ts +0 -332
  286. package/tests/harness/rest_e2e.ts +0 -68
  287. package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
  288. package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
  289. package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
  290. package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
  291. package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
  292. package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
  293. package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
  294. package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
  295. package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
  296. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
  297. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
  298. package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
  299. package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
  300. package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
  301. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
  302. package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
  303. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
  304. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
  305. package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
  306. package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
  307. package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
  308. package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
  309. package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
  310. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
  311. package/tests/unit/controller/theme.controller.spec.ts +0 -141
  312. package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
  313. package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
  314. package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
  315. package/tests/unit/duckdb/repositories.test.ts +0 -208
  316. package/tests/unit/ducklake/ducklake.test.ts +0 -952
  317. package/tests/unit/service/theme_store.spec.ts +0 -139
  318. package/tsconfig.json +0 -32
@@ -1,162 +0,0 @@
1
- /**
2
- * Helpers for streaming the ad-hoc connection SQL endpoints
3
- * (`/environments/.../connections/.../sqlQuery`) so the publisher
4
- * process never has to hold a whole result set in memory before
5
- * returning it.
6
- *
7
- * The Step 1 row cap (`PUBLISHER_MAX_QUERY_ROWS`) is necessary but
8
- * not sufficient: row count is a poor proxy for memory pressure
9
- * because a single 10 MB JSON column blows past the 100k-row cap's
10
- * safe envelope. The byte cap (`PUBLISHER_MAX_RESPONSE_BYTES`) is
11
- * the actual memory bound, and bytes can only be enforced by
12
- * iterating row-at-a-time on `runSQLStream` — `runSQL` returns a
13
- * fully-buffered result, so by the time we'd count bytes the
14
- * connector has already done the damage.
15
- *
16
- * On streaming-capable connections (Postgres, DuckDB, ...) the
17
- * controller routes here. On other connections it stays on the
18
- * Step 1 path; client-side byte counting after the fact would be
19
- * security theatre.
20
- */
21
-
22
- import type {
23
- Connection,
24
- MalloyQueryData,
25
- QueryRecord,
26
- RunSQLOptions,
27
- StreamingConnection,
28
- } from "@malloydata/malloy";
29
-
30
- import { PayloadTooLargeError } from "./errors";
31
- import { recordQueryCapExceeded } from "./query_cap_metrics";
32
-
33
- /**
34
- * Runtime check + type narrow for streaming-capable connections.
35
- * `Connection.canStream` is declared as `this is StreamingConnection`
36
- * by the Malloy SDK, so a positive result is enough to safely call
37
- * `runSQLStream`.
38
- */
39
- export function isStreamingConnection(
40
- connection: Connection,
41
- ): connection is StreamingConnection {
42
- return (
43
- typeof (connection as { canStream?: () => boolean }).canStream ===
44
- "function" && (connection as StreamingConnection).canStream()
45
- );
46
- }
47
-
48
- export interface StreamBudget {
49
- /**
50
- * Maximum number of rows to return. A value of `0` disables the
51
- * row cap (the caller-supplied `rowLimit` in `runSQLOptions` may
52
- * still bound the stream, but the helper will not raise on
53
- * overflow).
54
- */
55
- maxRows: number;
56
- /**
57
- * Maximum aggregate JSON-serialized byte size of returned rows.
58
- * `0` disables the byte cap. Measured as the sum of
59
- * `Buffer.byteLength(JSON.stringify(row))` for each yielded row
60
- * — the same bytes the eventual response will contain inside
61
- * `result.rows`.
62
- */
63
- maxBytes: number;
64
- }
65
-
66
- /**
67
- * Drain `runSQLStream` into a `MalloyQueryData`-shaped buffer,
68
- * enforcing both a row cap and a byte cap. Aborts the underlying
69
- * iterator via an internal `AbortController` the moment either cap
70
- * is breached so the driver stops producing rows immediately
71
- * (Postgres' `pg-query-stream` and DuckDB's streaming iterator both
72
- * honor `abortSignal`).
73
- *
74
- * The row cap is detected by the `cap + 1` sentinel pattern: the
75
- * controller has already clamped `runSQLOptions.rowLimit` to
76
- * `min(callerLimit, cap + 1)`, so receiving `cap + 1` rows
77
- * unambiguously means the request would have overflowed. This
78
- * matches the non-streaming path's overflow detection so behavior
79
- * is identical regardless of which connector served the request.
80
- *
81
- * On overflow the helper throws `PayloadTooLargeError` directly —
82
- * the message includes the relevant env-var name so the operator
83
- * sees a self-contained tuning hint without having to cross-
84
- * reference the controller.
85
- */
86
- export async function streamSqlWithBudget(
87
- connection: StreamingConnection,
88
- sql: string,
89
- runSQLOptions: RunSQLOptions,
90
- budget: StreamBudget,
91
- ): Promise<MalloyQueryData> {
92
- const { maxRows, maxBytes } = budget;
93
- const capAc = new AbortController();
94
- const rows: QueryRecord[] = [];
95
- let byteTotal = 0;
96
- let overflowMessage: string | undefined;
97
-
98
- // Compose two abort sources so a caller-supplied signal (the
99
- // publisher's query timeout) and the internal cap-abort signal
100
- // *both* cancel the underlying iterator:
101
- //
102
- // - If the caller's signal fires first, the controller's
103
- // `runWithQueryTimeout` will throw `QueryTimeoutError` → 504.
104
- // - If the cap-abort fires first, we throw
105
- // `PayloadTooLargeError` → 413.
106
- //
107
- // Without composition the streaming branch would silently drop
108
- // the caller's signal — historically the only way to abort here
109
- // was the cap, so the legacy controller cleared
110
- // `runSQLOptions.abortSignal`. Step 5 reverses that: the caller's
111
- // signal is now authoritative.
112
- //
113
- // `AbortSignal.any` is widely available (Node 20+); guard with a
114
- // typeof check so a stale runtime falls back to the legacy
115
- // cap-only behavior instead of crashing at module load.
116
- const externalSignal = runSQLOptions.abortSignal;
117
- const composedSignal: AbortSignal =
118
- externalSignal && typeof AbortSignal.any === "function"
119
- ? AbortSignal.any([externalSignal, capAc.signal])
120
- : capAc.signal;
121
-
122
- try {
123
- for await (const row of connection.runSQLStream(sql, {
124
- ...runSQLOptions,
125
- abortSignal: composedSignal,
126
- })) {
127
- rows.push(row);
128
- if (maxBytes > 0) {
129
- // Measure exactly what the eventual response body will
130
- // contain for this row. O(rowSize) per row, duplicated
131
- // against the final `JSON.stringify(result)` — the
132
- // early-abort win on overflow dwarfs the bookkeeping
133
- // cost in the bounded-success case.
134
- byteTotal += Buffer.byteLength(JSON.stringify(row), "utf8");
135
- if (byteTotal > maxBytes) {
136
- recordQueryCapExceeded("bytes", "connection_sql");
137
- overflowMessage = `Query response exceeded ${maxBytes} bytes (had at least ${byteTotal}). Refine the query (project fewer columns, add a LIMIT, or filter wide values) or raise PUBLISHER_MAX_RESPONSE_BYTES.`;
138
- capAc.abort();
139
- break;
140
- }
141
- }
142
- if (maxRows > 0 && rows.length > maxRows) {
143
- recordQueryCapExceeded("rows", "connection_sql");
144
- overflowMessage = `Query returned more than ${maxRows} rows. Refine the query (add a LIMIT or more selective WHERE) or raise PUBLISHER_MAX_QUERY_ROWS.`;
145
- capAc.abort();
146
- break;
147
- }
148
- }
149
- } catch (err) {
150
- // `pg-query-stream` surfaces `query.destroy()` (which our
151
- // abort handler triggers) as a synthetic error in some
152
- // versions. Swallow it iff we triggered the abort ourselves —
153
- // otherwise it's a real connection error (or the caller's
154
- // timeout, which the controller's runWithQueryTimeout will
155
- // surface as a 504) that the controller must see.
156
- if (!overflowMessage) throw err;
157
- }
158
-
159
- if (overflowMessage) throw new PayloadTooLargeError(overflowMessage);
160
-
161
- return { rows, totalRows: rows.length };
162
- }
package/src/telemetry.ts DELETED
@@ -1,20 +0,0 @@
1
- import { metrics, type Meter } from "@opentelemetry/api";
2
-
3
- /**
4
- * Single source of truth for the OpenTelemetry meter name. Every
5
- * publisher-emitted metric registers under this name so a typo can't
6
- * silently split a metric onto a second meter and drop it from the
7
- * Prometheus scrape.
8
- */
9
- export const METER_NAME = "publisher";
10
-
11
- /**
12
- * The shared publisher meter. Resolves lazily through the OTel global
13
- * provider (a `ProxyMeter`), so modules may call this at import time;
14
- * the real provider can be installed afterward (the production SDK in
15
- * `instrumentation.ts`, or the in-memory test harness) and instruments
16
- * still route to it.
17
- */
18
- export function publisherMeter(): Meter {
19
- return metrics.getMeter(METER_NAME);
20
- }
@@ -1,166 +0,0 @@
1
- /**
2
- * Test helper: spin up an in-memory OpenTelemetry MeterProvider so
3
- * unit tests can assert that the new guardrails (admission gate,
4
- * query timeout, query concurrency, heap check) emit the expected
5
- * counters and gauges.
6
- *
7
- * Usage:
8
- *
9
- * const harness = await startMetricsHarness();
10
- * // ... exercise production code ...
11
- * const sums = await harness.collectCounter("publisher_query_timeout_total");
12
- * expect(sums).toBe(1);
13
- * await harness.shutdown();
14
- *
15
- * The OTel JS API resolves `metrics.getMeter(name)` lazily through a
16
- * `ProxyMeter`, so registering the MeterProvider here AFTER the
17
- * production modules have already cached their meter handles works
18
- * — subsequent `.add()` / observable callbacks route to this
19
- * harness's provider.
20
- *
21
- * The harness drains all of the provider's metrics via
22
- * `MetricReader.collect()` and reads the cumulative data points
23
- * directly, bypassing the exporter pipeline so test code doesn't
24
- * have to thread async callbacks.
25
- */
26
-
27
- import { metrics } from "@opentelemetry/api";
28
- import { MeterProvider, MetricReader } from "@opentelemetry/sdk-metrics";
29
-
30
- /**
31
- * The simplest possible MetricReader implementation: a no-op that
32
- * exists only so the `MeterProvider` has a reader to satisfy its
33
- * collection invariants. Tests call `collect()` directly via the
34
- * harness; the reader never pushes anywhere.
35
- */
36
- class CollectingMetricReader extends MetricReader {
37
- protected override async onForceFlush(): Promise<void> {
38
- // no-op; tests pull via `collect()` on demand
39
- }
40
- protected override async onShutdown(): Promise<void> {
41
- // no-op
42
- }
43
- }
44
-
45
- export interface MetricsHarness {
46
- readonly provider: MeterProvider;
47
- /**
48
- * Force a collection cycle and return the cumulative sum of all
49
- * data points for the named counter. Returns 0 when the counter
50
- * has not been emitted yet.
51
- *
52
- * `attributeFilter` lets a test scope the sum to a single label
53
- * set (e.g. only the `environment: "test-env"` data point).
54
- */
55
- collectCounter(
56
- name: string,
57
- attributeFilter?: Record<string, string | number | boolean>,
58
- ): Promise<number>;
59
- /**
60
- * Force a collection cycle and return the most-recently observed
61
- * value of the named gauge, or `undefined` if no callback has
62
- * fired for this gauge yet.
63
- */
64
- collectGauge(name: string): Promise<number | undefined>;
65
- /**
66
- * Force a collection cycle and return the aggregate `count`/`sum` across all
67
- * data points for the named histogram (optionally scoped to one label set).
68
- * `count` is 0 when the histogram has not been emitted yet.
69
- */
70
- collectHistogram(
71
- name: string,
72
- attributeFilter?: Record<string, string | number | boolean>,
73
- ): Promise<{ count: number; sum: number; boundaries: number[] }>;
74
- shutdown(): Promise<void>;
75
- }
76
-
77
- export async function startMetricsHarness(): Promise<MetricsHarness> {
78
- const reader = new CollectingMetricReader();
79
- const provider = new MeterProvider({ readers: [reader] });
80
- // OTel JS's `setGlobalMeterProvider` silently refuses to
81
- // overwrite an existing global (`registerGlobal` returns false).
82
- // Tests that run back-to-back would otherwise route their
83
- // metrics into a dead provider from the previous test. Disable
84
- // first to clear the slot.
85
- metrics.disable();
86
- metrics.setGlobalMeterProvider(provider);
87
-
88
- return {
89
- provider,
90
- async collectCounter(
91
- name: string,
92
- attributeFilter?: Record<string, string | number | boolean>,
93
- ): Promise<number> {
94
- const result = await reader.collect();
95
- let total = 0;
96
- for (const rm of result.resourceMetrics.scopeMetrics) {
97
- for (const metric of rm.metrics) {
98
- if (metric.descriptor.name !== name) continue;
99
- for (const dp of metric.dataPoints) {
100
- if (attributeFilter) {
101
- const allMatch = Object.entries(attributeFilter).every(
102
- ([k, v]) => dp.attributes?.[k] === v,
103
- );
104
- if (!allMatch) continue;
105
- }
106
- total += dp.value as number;
107
- }
108
- }
109
- }
110
- return total;
111
- },
112
- async collectHistogram(
113
- name: string,
114
- attributeFilter?: Record<string, string | number | boolean>,
115
- ): Promise<{ count: number; sum: number; boundaries: number[] }> {
116
- const result = await reader.collect();
117
- let count = 0;
118
- let sum = 0;
119
- let boundaries: number[] = [];
120
- for (const rm of result.resourceMetrics.scopeMetrics) {
121
- for (const metric of rm.metrics) {
122
- if (metric.descriptor.name !== name) continue;
123
- for (const dp of metric.dataPoints) {
124
- if (attributeFilter) {
125
- const allMatch = Object.entries(attributeFilter).every(
126
- ([k, v]) => dp.attributes?.[k] === v,
127
- );
128
- if (!allMatch) continue;
129
- }
130
- const point = dp.value as {
131
- count: number;
132
- sum?: number;
133
- buckets?: { boundaries: number[] };
134
- };
135
- count += point.count;
136
- sum += point.sum ?? 0;
137
- if (point.buckets) boundaries = point.buckets.boundaries;
138
- }
139
- }
140
- }
141
- return { count, sum, boundaries };
142
- },
143
- async collectGauge(name: string): Promise<number | undefined> {
144
- const result = await reader.collect();
145
- for (const rm of result.resourceMetrics.scopeMetrics) {
146
- for (const metric of rm.metrics) {
147
- if (metric.descriptor.name !== name) continue;
148
- // Observable gauges yield one data point per
149
- // attribute set per collection. Return the last
150
- // observed value across all data points so unlabeled
151
- // gauges (the common case in this code base) just
152
- // work without attribute plumbing.
153
- const last = metric.dataPoints[metric.dataPoints.length - 1];
154
- return last?.value as number | undefined;
155
- }
156
- }
157
- return undefined;
158
- },
159
- async shutdown(): Promise<void> {
160
- await reader.shutdown();
161
- // Clear the global so the next harness's
162
- // `setGlobalMeterProvider` is accepted.
163
- metrics.disable();
164
- },
165
- };
166
- }
@@ -1,57 +0,0 @@
1
- import { describe, expect, it } from "bun:test";
2
- import { readFileSync } from "fs";
3
- import { resolve } from "path";
4
- import { PER_MODE_COLOR_KEYS } from "./config";
5
-
6
- /**
7
- * The per-mode palette key list is hand-copied in three places: this server
8
- * copy (config.ts), the SDK copy (packages/sdk/src/theme/keys.ts), and the
9
- * api-doc.yaml Theme.palette schema. The server and the SDK are intentionally
10
- * decoupled (neither imports the other), and api-doc.yaml is their only shared
11
- * contract, so nothing but this test keeps the three in step. A silent drift
12
- * drops a colour: the server sanitizer or the SDK resolver ignores a key that
13
- * is not in its own copy.
14
- */
15
-
16
- function extractTsArray(source: string, constName: string): string[] {
17
- const decl = source.indexOf(`${constName} = [`);
18
- if (decl === -1) throw new Error(`${constName} array not found`);
19
- const open = source.indexOf("[", decl);
20
- const close = source.indexOf("]", open);
21
- return [...source.slice(open + 1, close).matchAll(/"([^"]+)"/g)].map(
22
- (m) => m[1],
23
- );
24
- }
25
-
26
- function extractApiDocPaletteKeys(apiDoc: string): string[] {
27
- const start = apiDoc.indexOf("\n palette:");
28
- if (start === -1) throw new Error("palette block not found in api-doc.yaml");
29
- const end = apiDoc.indexOf("\n font:", start);
30
- const block = apiDoc.slice(start, end === -1 ? undefined : end);
31
- // Direct children of `palette.properties` sit at 12-space indentation;
32
- // deeper keys (type, properties, light, dark) are more indented and skipped.
33
- return [...block.matchAll(/^ {12}(\w+):$/gm)].map((m) => m[1]);
34
- }
35
-
36
- describe("per-mode palette key parity", () => {
37
- const serverKeys = [...PER_MODE_COLOR_KEYS];
38
-
39
- it("server config.ts matches the SDK keys.ts copy", () => {
40
- const sdk = readFileSync(
41
- resolve(import.meta.dir, "../../sdk/src/theme/keys.ts"),
42
- "utf8",
43
- );
44
- expect(extractTsArray(sdk, "PER_MODE_COLOR_KEYS")).toEqual(serverKeys);
45
- });
46
-
47
- it("server config.ts matches the api-doc.yaml Theme.palette schema", () => {
48
- const apiDoc = readFileSync(
49
- resolve(import.meta.dir, "../../../api-doc.yaml"),
50
- "utf8",
51
- );
52
- // palette also declares `series` (a shared array, not a per-mode object).
53
- expect(new Set(extractApiDocPaletteKeys(apiDoc))).toEqual(
54
- new Set([...serverKeys, "series"]),
55
- );
56
- });
57
- });
package/src/utils.ts DELETED
@@ -1,29 +0,0 @@
1
- import { URLReader } from "@malloydata/malloy";
2
- import * as fs from "fs";
3
- import * as path from "path";
4
- import { fileURLToPath } from "url";
5
-
6
- export const URL_READER: URLReader = {
7
- readURL: (url: URL) => {
8
- let path = url.toString();
9
- if (url.protocol == "file:") {
10
- path = fileURLToPath(url);
11
- }
12
- return fs.promises.readFile(path, "utf8");
13
- },
14
- };
15
-
16
- /**
17
- * Skip dotfiles/dotdirs (.vscode, .git, .DS_Store, etc.) when walking a
18
- * package tree. These come from editors/VCS, never contain Malloy models
19
- * or databases, and have been a source of spurious ENOENTs when their
20
- * contents disappear mid-scan.
21
- */
22
- export function ignoreDotfiles(file: string): boolean {
23
- return path.basename(file).startsWith(".");
24
- }
25
-
26
- /** The message of a thrown value, stringifying non-Error throws. */
27
- export function errMessage(err: unknown): string {
28
- return err instanceof Error ? err.message : String(err);
29
- }
@@ -1,9 +0,0 @@
1
- ##! experimental.givens
2
-
3
- given:
4
- ROLE :: string
5
-
6
- #(authorize) "$ROLE = 'analyst'"
7
- source: gated is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }
8
-
9
- source: open_src is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }
@@ -1,4 +0,0 @@
1
- {
2
- "name": "authorize-compile",
3
- "description": "Fixture: a gated source for the /compile authorize HTTP integration test."
4
- }
@@ -1 +0,0 @@
1
- source: nums is duckdb.sql("SELECT 1 as n")
@@ -1,5 +0,0 @@
1
- {
2
- "name": "html-pages-nopublic",
3
- "version": "0.0.1",
4
- "description": "Package with no public/ directory; static file requests return 404."
5
- }
@@ -1,3 +0,0 @@
1
- id,name
2
- 1,alpha
3
- 2,beta
@@ -1,3 +0,0 @@
1
- body {
2
- font-family: system-ui, sans-serif;
3
- }
@@ -1,4 +0,0 @@
1
- <!doctype html>
2
- <title>Bare HTML</title>
3
- <!-- example only, must NOT opt in: <meta name="publisher:fit" content="viewport"> -->
4
- <h1>No head or body tags; the tag appears only in a comment.</h1>
@@ -1,8 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <title>Body Meta</title>
4
- <body>
5
- <meta name="publisher:fit" content="viewport" />
6
- <h1>A real fit meta in &lt;body&gt;; the head-only scan means no opt-in.</h1>
7
- </body>
8
- </html>
@@ -1 +0,0 @@
1
- { "ok": true }
@@ -1,11 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <!-- this deck does <body> styling tricks; the literal must not truncate the scan -->
5
- <meta name="publisher:fit" content="viewport" />
6
- <title>Fit After Comment</title>
7
- </head>
8
- <body style="height: 100vh; margin: 0; overflow: hidden">
9
- <section style="height: 100%">A full-screen slide</section>
10
- </body>
11
- </html>
@@ -1,9 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <title>Carrier Dashboard</title>
5
- </head>
6
- <body>
7
- <h1>Hello from the in-package data app</h1>
8
- </body>
9
- </html>
@@ -1,8 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <title>No Head Close Tag</title>
4
- <body>
5
- <h1>Omits the optional &lt;/head&gt; end tag</h1>
6
- <!-- shown in body, must NOT opt in: <meta name="publisher:fit" content="viewport"> -->
7
- </body>
8
- </html>
@@ -1,13 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <meta charset="utf-8" />
5
- <meta name="viewport" content="width=device-width, initial-scale=1" />
6
- <meta data-name="publisher:fit" content="viewport" />
7
- <title>Tall Dashboard</title>
8
- </head>
9
- <body>
10
- <h1>A normal dashboard that hugs its content</h1>
11
- <!-- shown in body, must NOT opt in: <meta name="publisher:fit" content="viewport"> -->
12
- </body>
13
- </html>
@@ -1,12 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <meta charset="utf-8" />
5
- <meta name="viewport" content="width=device-width, initial-scale=1" />
6
- <meta name="publisher:fit" content="viewport" />
7
- <title>FY27 Slide Deck</title>
8
- </head>
9
- <body style="height: 100vh; margin: 0; overflow: hidden">
10
- <section style="height: 100%">A full-screen slide</section>
11
- </body>
12
- </html>
@@ -1,9 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <title>Second Page</title>
5
- </head>
6
- <body>
7
- <p>A page in a subdirectory.</p>
8
- </body>
9
- </html>
@@ -1,10 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <title>Unterminated Comment</title>
5
- <!-- never closed, must NOT opt in: <meta name="publisher:fit" content="viewport">
6
- </head>
7
- <body>
8
- <h1>An unterminated comment swallows the tag.</h1>
9
- </body>
10
- </html>
@@ -1,5 +0,0 @@
1
- {
2
- "name": "html-pages-test",
3
- "version": "1.0.0",
4
- "description": "Fixture for in-package HTML static-file serving + /pages + /events"
5
- }
@@ -1 +0,0 @@
1
- source: report is duckdb.sql("SELECT 1 as n")
@@ -1,5 +0,0 @@
1
- category,amount
2
- electronics,100
3
- clothing,50
4
- electronics,200
5
- clothing,75
@@ -1,18 +0,0 @@
1
- ##! experimental.persistence
2
-
3
- source: raw_orders is duckdb.table('data/orders.csv')
4
-
5
- // Upstream persist source: an aggregate over the base CSV.
6
- #@ persist name="orders_base"
7
- source: orders_base is raw_orders -> {
8
- group_by: category
9
- aggregate: total_amount is amount.sum()
10
- }
11
-
12
- // Downstream persist source: references the persist upstream `orders_base`.
13
- // A per-unit build of `orders_rollup` alone must REFERENCE orders_base's
14
- // materialized table (via referenceManifest) rather than recompute it live.
15
- #@ persist name="orders_rollup"
16
- source: orders_rollup is orders_base -> {
17
- aggregate: category_count is count()
18
- }
@@ -1,5 +0,0 @@
1
- {
2
- "name": "persist-multi-level",
3
- "version": "1.0.0",
4
- "description": "DuckDB two-level persist DAG fixture (orders_base -> orders_rollup)"
5
- }
@@ -1,5 +0,0 @@
1
- category,amount
2
- electronics,100
3
- clothing,50
4
- electronics,200
5
- clothing,75
@@ -1,11 +0,0 @@
1
- ##! experimental.persistence
2
-
3
- source: raw_orders is duckdb.table('data/orders.csv')
4
-
5
- #@ persist name="order_summary"
6
- source: order_summary is raw_orders -> {
7
- group_by: category
8
- aggregate:
9
- total_orders is count()
10
- total_revenue is amount.sum()
11
- }
@@ -1,7 +0,0 @@
1
- {
2
- "name": "persist-schedule-test",
3
- "version": "1.0.0",
4
- "description": "DuckDB persist fixture with a version-scoped materialization schedule",
5
- "scope": "version",
6
- "materialization": { "schedule": "0 6 * * *" }
7
- }
@@ -1,5 +0,0 @@
1
- category,amount
2
- electronics,100
3
- clothing,50
4
- electronics,200
5
- clothing,75