@malloy-publisher/server 0.0.235 → 0.0.237

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (319) hide show
  1. package/dist/app/api-doc.yaml +83 -27
  2. package/dist/app/assets/{EnvironmentPage-BsAnavYN.js → EnvironmentPage-DaP6CpgK.js} +1 -1
  3. package/dist/app/assets/{HomePage-CADE138j.js → HomePage-C9yNw3iu.js} +1 -1
  4. package/dist/app/assets/{LightMode-Cfh7KzN8.js → LightMode-Be4-G-dX.js} +1 -1
  5. package/dist/app/assets/{MainPage-CO3pRlnV.js → MainPage-J_rU2SgI.js} +2 -2
  6. package/dist/app/assets/{MaterializationsPage-p9YjkRXZ.js → MaterializationsPage-DHgPI4MO.js} +1 -1
  7. package/dist/app/assets/ModelPage-BzXfjTu5.js +1 -0
  8. package/dist/app/assets/{PackagePage-e4kN75YR.js → PackagePage-ByoSTLJ4.js} +1 -1
  9. package/dist/app/assets/RouteError-CrSeKN2r.js +1 -0
  10. package/dist/app/assets/{ThemeEditorPage-CciagFTq.js → ThemeEditorPage-Btxnmhtk.js} +1 -1
  11. package/dist/app/assets/{WorkbookPage-DNWmkCXa.js → WorkbookPage-D6JLPnWO.js} +1 -1
  12. package/dist/app/assets/{core-Rj_4rRnA.es-BZyvITuO.js → core-BSRgt1mG.es-Bc_D2Lq4.js} +1 -1
  13. package/dist/app/assets/{index-CH2AcDzc.js → index-BRijWj6K.js} +15 -15
  14. package/dist/app/assets/{index-DQa463gC.js → index-BX4D94xw.js} +1 -1
  15. package/dist/app/assets/{index-VBbcc8s6.js → index-Cvy0xtb7.js} +1 -1
  16. package/dist/app/assets/{index-DQpV7MyA.js → index-DMAhIJNC.js} +1 -1
  17. package/dist/app/index.html +1 -1
  18. package/dist/package_load_worker.mjs +59 -20
  19. package/dist/server.mjs +1342 -616
  20. package/package.json +8 -2
  21. package/.eslintrc.json +0 -44
  22. package/.prettierignore +0 -1
  23. package/.prettierrc +0 -4
  24. package/build.ts +0 -97
  25. package/dist/app/assets/ModelPage-C1OSTv-x.js +0 -1
  26. package/dist/app/assets/RouteError-CzbfOkng.js +0 -1
  27. package/eslint.config.mjs +0 -76
  28. package/k6-tests/README.md +0 -144
  29. package/k6-tests/bun.lock +0 -797
  30. package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
  31. package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
  32. package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
  33. package/k6-tests/load-test/load-test-crud.ts +0 -142
  34. package/k6-tests/load-test/load-test.ts +0 -784
  35. package/k6-tests/package.json +0 -27
  36. package/k6-tests/scripts/run-with-otel.sh +0 -50
  37. package/k6-tests/smoke-test/smoke-test.ts +0 -158
  38. package/k6-tests/tsconfig.json +0 -29
  39. package/k6-tests/utils/client_factory.ts +0 -128
  40. package/k6-tests/utils/common.ts +0 -489
  41. package/k6-tests/utils/logger.ts +0 -62
  42. package/k6-tests/utils/types.d.ts +0 -35
  43. package/malloy_mcp.dxt +0 -0
  44. package/publisher.config.example.bigquery.json +0 -33
  45. package/publisher.config.example.duckdb.json +0 -23
  46. package/publisher.config.json +0 -23
  47. package/scripts/bake-duckdb-extensions.js +0 -110
  48. package/src/config.spec.ts +0 -1535
  49. package/src/config.theme.spec.ts +0 -178
  50. package/src/config.ts +0 -1142
  51. package/src/constants.ts +0 -105
  52. package/src/controller/compile.controller.ts +0 -40
  53. package/src/controller/connection.controller.spec.ts +0 -849
  54. package/src/controller/connection.controller.ts +0 -901
  55. package/src/controller/database.controller.ts +0 -24
  56. package/src/controller/materialization.controller.spec.ts +0 -293
  57. package/src/controller/materialization.controller.ts +0 -319
  58. package/src/controller/model.controller.ts +0 -164
  59. package/src/controller/package.controller.spec.ts +0 -273
  60. package/src/controller/package.controller.ts +0 -306
  61. package/src/controller/query.controller.ts +0 -152
  62. package/src/controller/theme.controller.ts +0 -83
  63. package/src/controller/watch-mode.controller.ts +0 -243
  64. package/src/data_styles.ts +0 -71
  65. package/src/default-publisher.config.json +0 -23
  66. package/src/ducklake_version.spec.ts +0 -163
  67. package/src/ducklake_version.ts +0 -153
  68. package/src/errors.spec.ts +0 -119
  69. package/src/errors.ts +0 -237
  70. package/src/filter_deprecation.spec.ts +0 -64
  71. package/src/filter_deprecation.ts +0 -42
  72. package/src/health.spec.ts +0 -123
  73. package/src/health.ts +0 -233
  74. package/src/heap_check.spec.ts +0 -144
  75. package/src/heap_check.ts +0 -144
  76. package/src/instrumentation.ts +0 -214
  77. package/src/json_utils.spec.ts +0 -51
  78. package/src/json_utils.ts +0 -33
  79. package/src/logger.spec.ts +0 -210
  80. package/src/logger.ts +0 -251
  81. package/src/malloy_pin_prereqs.spec.ts +0 -25
  82. package/src/materialization_metrics.spec.ts +0 -169
  83. package/src/materialization_metrics.ts +0 -346
  84. package/src/mcp/error_messages.spec.ts +0 -35
  85. package/src/mcp/error_messages.ts +0 -183
  86. package/src/mcp/handler_utils.spec.ts +0 -108
  87. package/src/mcp/handler_utils.ts +0 -251
  88. package/src/mcp/mcp_constants.ts +0 -6
  89. package/src/mcp/query_envelope.spec.ts +0 -229
  90. package/src/mcp/query_envelope.ts +0 -240
  91. package/src/mcp/server.protocol.spec.ts +0 -250
  92. package/src/mcp/server.ts +0 -87
  93. package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
  94. package/src/mcp/skills/build_skills_bundle.ts +0 -176
  95. package/src/mcp/skills/skills_bundle.json +0 -1
  96. package/src/mcp/skills/skills_bundle.spec.ts +0 -171
  97. package/src/mcp/tool_response.spec.ts +0 -108
  98. package/src/mcp/tool_response.ts +0 -138
  99. package/src/mcp/tools/compile_tool.spec.ts +0 -315
  100. package/src/mcp/tools/compile_tool.ts +0 -208
  101. package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
  102. package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
  103. package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
  104. package/src/mcp/tools/docs_search_tool.ts +0 -128
  105. package/src/mcp/tools/embedding_index.spec.ts +0 -1236
  106. package/src/mcp/tools/embedding_index.ts +0 -808
  107. package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
  108. package/src/mcp/tools/execute_query_tool.ts +0 -316
  109. package/src/mcp/tools/get_context_eval.ts +0 -275
  110. package/src/mcp/tools/get_context_tool.spec.ts +0 -592
  111. package/src/mcp/tools/get_context_tool.ts +0 -585
  112. package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
  113. package/src/mcp/tools/reload_package_tool.ts +0 -132
  114. package/src/mcp_config.spec.ts +0 -919
  115. package/src/mcp_config.ts +0 -425
  116. package/src/oom_guards.integration.spec.ts +0 -269
  117. package/src/package_load/package_load_pool.spec.ts +0 -252
  118. package/src/package_load/package_load_pool.ts +0 -931
  119. package/src/package_load/package_load_worker.ts +0 -1034
  120. package/src/package_load/protocol.ts +0 -364
  121. package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
  122. package/src/package_load/rpc_wait_accountant.ts +0 -76
  123. package/src/package_load_metrics.spec.ts +0 -114
  124. package/src/package_load_metrics.ts +0 -127
  125. package/src/path_safety.spec.ts +0 -158
  126. package/src/path_safety.ts +0 -146
  127. package/src/pg_helpers.spec.ts +0 -223
  128. package/src/pg_helpers.ts +0 -52
  129. package/src/query_cap_metrics.spec.ts +0 -89
  130. package/src/query_cap_metrics.ts +0 -116
  131. package/src/query_concurrency.spec.ts +0 -247
  132. package/src/query_concurrency.ts +0 -237
  133. package/src/query_metadata_metrics.ts +0 -49
  134. package/src/query_param_utils.ts +0 -29
  135. package/src/query_timeout.spec.ts +0 -224
  136. package/src/query_timeout.ts +0 -179
  137. package/src/runtime/publisher.js +0 -328
  138. package/src/server-old.ts +0 -988
  139. package/src/server.ts +0 -2059
  140. package/src/service/annotations.spec.ts +0 -118
  141. package/src/service/annotations.ts +0 -91
  142. package/src/service/authorize.spec.ts +0 -154
  143. package/src/service/authorize.ts +0 -498
  144. package/src/service/authorize_integration.spec.ts +0 -2041
  145. package/src/service/build_plan.spec.ts +0 -685
  146. package/src/service/build_plan.ts +0 -797
  147. package/src/service/compile_authorize.spec.ts +0 -85
  148. package/src/service/compile_fragment_techniques.spec.ts +0 -156
  149. package/src/service/connection.spec.ts +0 -2360
  150. package/src/service/connection.ts +0 -2187
  151. package/src/service/connection_config.spec.ts +0 -734
  152. package/src/service/connection_config.ts +0 -899
  153. package/src/service/connection_federation.spec.ts +0 -184
  154. package/src/service/connection_fingerprint.spec.ts +0 -102
  155. package/src/service/connection_service.spec.ts +0 -711
  156. package/src/service/connection_service.ts +0 -291
  157. package/src/service/cron_evaluator.spec.ts +0 -79
  158. package/src/service/cron_evaluator.ts +0 -105
  159. package/src/service/db_utils.spec.ts +0 -1085
  160. package/src/service/db_utils.ts +0 -1502
  161. package/src/service/duckdb_instance_isolation.spec.ts +0 -137
  162. package/src/service/ducklake_lazy_attach.spec.ts +0 -110
  163. package/src/service/embedding_provider.spec.ts +0 -329
  164. package/src/service/embedding_provider.ts +0 -236
  165. package/src/service/environment.ts +0 -2191
  166. package/src/service/environment_admission.spec.ts +0 -344
  167. package/src/service/environment_compile.spec.ts +0 -197
  168. package/src/service/environment_store.spec.ts +0 -2274
  169. package/src/service/environment_store.ts +0 -2434
  170. package/src/service/environment_store_anchoring.spec.ts +0 -107
  171. package/src/service/environment_store_clone.spec.ts +0 -350
  172. package/src/service/explore_visibility.spec.ts +0 -434
  173. package/src/service/exports_probe.spec.ts +0 -107
  174. package/src/service/extension_fetch_policy.spec.ts +0 -256
  175. package/src/service/filter.spec.ts +0 -458
  176. package/src/service/filter.ts +0 -341
  177. package/src/service/filter_bypass.spec.ts +0 -418
  178. package/src/service/filter_integration.spec.ts +0 -1004
  179. package/src/service/freshness.spec.ts +0 -183
  180. package/src/service/freshness.ts +0 -112
  181. package/src/service/gcs_s3_utils.ts +0 -427
  182. package/src/service/given.ts +0 -105
  183. package/src/service/givens_integration.spec.ts +0 -219
  184. package/src/service/manifest_loader.spec.ts +0 -189
  185. package/src/service/manifest_loader.ts +0 -156
  186. package/src/service/materialization_build_session.spec.ts +0 -435
  187. package/src/service/materialization_build_session.ts +0 -681
  188. package/src/service/materialization_config_validation.spec.ts +0 -99
  189. package/src/service/materialization_config_validation.ts +0 -120
  190. package/src/service/materialization_eligibility.spec.ts +0 -158
  191. package/src/service/materialization_eligibility.ts +0 -305
  192. package/src/service/materialization_schedule_surface.spec.ts +0 -256
  193. package/src/service/materialization_scheduler.spec.ts +0 -289
  194. package/src/service/materialization_scheduler.ts +0 -304
  195. package/src/service/materialization_serve_transform.spec.ts +0 -1003
  196. package/src/service/materialization_serve_transform.ts +0 -779
  197. package/src/service/materialization_service.spec.ts +0 -2373
  198. package/src/service/materialization_service.ts +0 -2342
  199. package/src/service/materialization_test_fixtures.ts +0 -249
  200. package/src/service/model.spec.ts +0 -1157
  201. package/src/service/model.ts +0 -3370
  202. package/src/service/model_limits.spec.ts +0 -209
  203. package/src/service/model_limits.ts +0 -131
  204. package/src/service/model_storage_serve.spec.ts +0 -193
  205. package/src/service/model_storage_serve_joins.spec.ts +0 -193
  206. package/src/service/package.spec.ts +0 -612
  207. package/src/service/package.ts +0 -1799
  208. package/src/service/package_manifest.spec.ts +0 -230
  209. package/src/service/package_manifest.ts +0 -241
  210. package/src/service/package_memory_governor.spec.ts +0 -173
  211. package/src/service/package_memory_governor.ts +0 -233
  212. package/src/service/package_quote_bound_tables.spec.ts +0 -171
  213. package/src/service/package_race.spec.ts +0 -208
  214. package/src/service/package_reload_safety.spec.ts +0 -193
  215. package/src/service/package_rollback.spec.ts +0 -190
  216. package/src/service/package_worker_path.spec.ts +0 -645
  217. package/src/service/path_injection.spec.ts +0 -39
  218. package/src/service/persist_annotation_validation.spec.ts +0 -89
  219. package/src/service/persist_annotation_validation.ts +0 -59
  220. package/src/service/persistence_policy.spec.ts +0 -374
  221. package/src/service/proxy.spec.ts +0 -414
  222. package/src/service/proxy.ts +0 -248
  223. package/src/service/query_boundary.spec.ts +0 -470
  224. package/src/service/query_metadata.spec.ts +0 -408
  225. package/src/service/query_metadata.ts +0 -492
  226. package/src/service/query_metadata_identity.spec.ts +0 -149
  227. package/src/service/query_text.spec.ts +0 -79
  228. package/src/service/query_text.ts +0 -43
  229. package/src/service/quoting.spec.ts +0 -135
  230. package/src/service/quoting.ts +0 -78
  231. package/src/service/resolve_environment.ts +0 -15
  232. package/src/service/restricted_mode.spec.ts +0 -299
  233. package/src/service/source_extraction.ts +0 -225
  234. package/src/service/theme_store.ts +0 -199
  235. package/src/storage/DatabaseInterface.ts +0 -198
  236. package/src/storage/StorageManager.ts +0 -113
  237. package/src/storage/duckdb/ConnectionRepository.ts +0 -143
  238. package/src/storage/duckdb/DuckDBConnection.ts +0 -123
  239. package/src/storage/duckdb/DuckDBRepository.ts +0 -208
  240. package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
  241. package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
  242. package/src/storage/duckdb/MaterializationRepository.ts +0 -336
  243. package/src/storage/duckdb/PackageRepository.ts +0 -135
  244. package/src/storage/duckdb/schema.ts +0 -229
  245. package/src/stream_helpers.spec.ts +0 -280
  246. package/src/stream_helpers.ts +0 -162
  247. package/src/telemetry.ts +0 -20
  248. package/src/test_helpers/metrics_harness.ts +0 -166
  249. package/src/theme_key_parity.spec.ts +0 -57
  250. package/src/utils.ts +0 -29
  251. package/tests/fixtures/authorize-compile/model.malloy +0 -9
  252. package/tests/fixtures/authorize-compile/publisher.json +0 -4
  253. package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
  254. package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
  255. package/tests/fixtures/html-pages-test/data.csv +0 -3
  256. package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
  257. package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
  258. package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
  259. package/tests/fixtures/html-pages-test/public/data.json +0 -1
  260. package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
  261. package/tests/fixtures/html-pages-test/public/index.html +0 -9
  262. package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
  263. package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
  264. package/tests/fixtures/html-pages-test/public/slides.html +0 -12
  265. package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
  266. package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
  267. package/tests/fixtures/html-pages-test/publisher.json +0 -5
  268. package/tests/fixtures/html-pages-test/report.malloy +0 -1
  269. package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
  270. package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
  271. package/tests/fixtures/persist-multi-level/publisher.json +0 -5
  272. package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
  273. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
  274. package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
  275. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
  276. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
  277. package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
  278. package/tests/fixtures/persist-test/data/orders.csv +0 -5
  279. package/tests/fixtures/persist-test/persist_test.malloy +0 -11
  280. package/tests/fixtures/persist-test/publisher.json +0 -5
  281. package/tests/fixtures/publisher.config.json +0 -15
  282. package/tests/fixtures/query-givens/data/orders.csv +0 -7
  283. package/tests/fixtures/query-givens/model.malloy +0 -34
  284. package/tests/fixtures/query-givens/publisher.json +0 -5
  285. package/tests/fixtures/xlsx/database.xlsx +0 -0
  286. package/tests/harness/mcp_test_setup.ts +0 -332
  287. package/tests/harness/rest_e2e.ts +0 -68
  288. package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
  289. package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
  290. package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
  291. package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
  292. package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
  293. package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
  294. package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
  295. package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
  296. package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
  297. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
  298. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
  299. package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
  300. package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
  301. package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
  302. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
  303. package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
  304. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
  305. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
  306. package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
  307. package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
  308. package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
  309. package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
  310. package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
  311. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
  312. package/tests/unit/controller/theme.controller.spec.ts +0 -141
  313. package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
  314. package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
  315. package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
  316. package/tests/unit/duckdb/repositories.test.ts +0 -208
  317. package/tests/unit/ducklake/ducklake.test.ts +0 -952
  318. package/tests/unit/service/theme_store.spec.ts +0 -139
  319. package/tsconfig.json +0 -32
@@ -1,162 +0,0 @@
1
- /**
2
- * Helpers for streaming the ad-hoc connection SQL endpoints
3
- * (`/environments/.../connections/.../sqlQuery`) so the publisher
4
- * process never has to hold a whole result set in memory before
5
- * returning it.
6
- *
7
- * The Step 1 row cap (`PUBLISHER_MAX_QUERY_ROWS`) is necessary but
8
- * not sufficient: row count is a poor proxy for memory pressure
9
- * because a single 10 MB JSON column blows past the 100k-row cap's
10
- * safe envelope. The byte cap (`PUBLISHER_MAX_RESPONSE_BYTES`) is
11
- * the actual memory bound, and bytes can only be enforced by
12
- * iterating row-at-a-time on `runSQLStream` — `runSQL` returns a
13
- * fully-buffered result, so by the time we'd count bytes the
14
- * connector has already done the damage.
15
- *
16
- * On streaming-capable connections (Postgres, DuckDB, ...) the
17
- * controller routes here. On other connections it stays on the
18
- * Step 1 path; client-side byte counting after the fact would be
19
- * security theatre.
20
- */
21
-
22
- import type {
23
- Connection,
24
- MalloyQueryData,
25
- QueryRecord,
26
- RunSQLOptions,
27
- StreamingConnection,
28
- } from "@malloydata/malloy";
29
-
30
- import { PayloadTooLargeError } from "./errors";
31
- import { recordQueryCapExceeded } from "./query_cap_metrics";
32
-
33
- /**
34
- * Runtime check + type narrow for streaming-capable connections.
35
- * `Connection.canStream` is declared as `this is StreamingConnection`
36
- * by the Malloy SDK, so a positive result is enough to safely call
37
- * `runSQLStream`.
38
- */
39
- export function isStreamingConnection(
40
- connection: Connection,
41
- ): connection is StreamingConnection {
42
- return (
43
- typeof (connection as { canStream?: () => boolean }).canStream ===
44
- "function" && (connection as StreamingConnection).canStream()
45
- );
46
- }
47
-
48
- export interface StreamBudget {
49
- /**
50
- * Maximum number of rows to return. A value of `0` disables the
51
- * row cap (the caller-supplied `rowLimit` in `runSQLOptions` may
52
- * still bound the stream, but the helper will not raise on
53
- * overflow).
54
- */
55
- maxRows: number;
56
- /**
57
- * Maximum aggregate JSON-serialized byte size of returned rows.
58
- * `0` disables the byte cap. Measured as the sum of
59
- * `Buffer.byteLength(JSON.stringify(row))` for each yielded row
60
- * — the same bytes the eventual response will contain inside
61
- * `result.rows`.
62
- */
63
- maxBytes: number;
64
- }
65
-
66
- /**
67
- * Drain `runSQLStream` into a `MalloyQueryData`-shaped buffer,
68
- * enforcing both a row cap and a byte cap. Aborts the underlying
69
- * iterator via an internal `AbortController` the moment either cap
70
- * is breached so the driver stops producing rows immediately
71
- * (Postgres' `pg-query-stream` and DuckDB's streaming iterator both
72
- * honor `abortSignal`).
73
- *
74
- * The row cap is detected by the `cap + 1` sentinel pattern: the
75
- * controller has already clamped `runSQLOptions.rowLimit` to
76
- * `min(callerLimit, cap + 1)`, so receiving `cap + 1` rows
77
- * unambiguously means the request would have overflowed. This
78
- * matches the non-streaming path's overflow detection so behavior
79
- * is identical regardless of which connector served the request.
80
- *
81
- * On overflow the helper throws `PayloadTooLargeError` directly —
82
- * the message includes the relevant env-var name so the operator
83
- * sees a self-contained tuning hint without having to cross-
84
- * reference the controller.
85
- */
86
- export async function streamSqlWithBudget(
87
- connection: StreamingConnection,
88
- sql: string,
89
- runSQLOptions: RunSQLOptions,
90
- budget: StreamBudget,
91
- ): Promise<MalloyQueryData> {
92
- const { maxRows, maxBytes } = budget;
93
- const capAc = new AbortController();
94
- const rows: QueryRecord[] = [];
95
- let byteTotal = 0;
96
- let overflowMessage: string | undefined;
97
-
98
- // Compose two abort sources so a caller-supplied signal (the
99
- // publisher's query timeout) and the internal cap-abort signal
100
- // *both* cancel the underlying iterator:
101
- //
102
- // - If the caller's signal fires first, the controller's
103
- // `runWithQueryTimeout` will throw `QueryTimeoutError` → 504.
104
- // - If the cap-abort fires first, we throw
105
- // `PayloadTooLargeError` → 413.
106
- //
107
- // Without composition the streaming branch would silently drop
108
- // the caller's signal — historically the only way to abort here
109
- // was the cap, so the legacy controller cleared
110
- // `runSQLOptions.abortSignal`. Step 5 reverses that: the caller's
111
- // signal is now authoritative.
112
- //
113
- // `AbortSignal.any` is widely available (Node 20+); guard with a
114
- // typeof check so a stale runtime falls back to the legacy
115
- // cap-only behavior instead of crashing at module load.
116
- const externalSignal = runSQLOptions.abortSignal;
117
- const composedSignal: AbortSignal =
118
- externalSignal && typeof AbortSignal.any === "function"
119
- ? AbortSignal.any([externalSignal, capAc.signal])
120
- : capAc.signal;
121
-
122
- try {
123
- for await (const row of connection.runSQLStream(sql, {
124
- ...runSQLOptions,
125
- abortSignal: composedSignal,
126
- })) {
127
- rows.push(row);
128
- if (maxBytes > 0) {
129
- // Measure exactly what the eventual response body will
130
- // contain for this row. O(rowSize) per row, duplicated
131
- // against the final `JSON.stringify(result)` — the
132
- // early-abort win on overflow dwarfs the bookkeeping
133
- // cost in the bounded-success case.
134
- byteTotal += Buffer.byteLength(JSON.stringify(row), "utf8");
135
- if (byteTotal > maxBytes) {
136
- recordQueryCapExceeded("bytes", "connection_sql");
137
- overflowMessage = `Query response exceeded ${maxBytes} bytes (had at least ${byteTotal}). Refine the query (project fewer columns, add a LIMIT, or filter wide values) or raise PUBLISHER_MAX_RESPONSE_BYTES.`;
138
- capAc.abort();
139
- break;
140
- }
141
- }
142
- if (maxRows > 0 && rows.length > maxRows) {
143
- recordQueryCapExceeded("rows", "connection_sql");
144
- overflowMessage = `Query returned more than ${maxRows} rows. Refine the query (add a LIMIT or more selective WHERE) or raise PUBLISHER_MAX_QUERY_ROWS.`;
145
- capAc.abort();
146
- break;
147
- }
148
- }
149
- } catch (err) {
150
- // `pg-query-stream` surfaces `query.destroy()` (which our
151
- // abort handler triggers) as a synthetic error in some
152
- // versions. Swallow it iff we triggered the abort ourselves —
153
- // otherwise it's a real connection error (or the caller's
154
- // timeout, which the controller's runWithQueryTimeout will
155
- // surface as a 504) that the controller must see.
156
- if (!overflowMessage) throw err;
157
- }
158
-
159
- if (overflowMessage) throw new PayloadTooLargeError(overflowMessage);
160
-
161
- return { rows, totalRows: rows.length };
162
- }
package/src/telemetry.ts DELETED
@@ -1,20 +0,0 @@
1
- import { metrics, type Meter } from "@opentelemetry/api";
2
-
3
- /**
4
- * Single source of truth for the OpenTelemetry meter name. Every
5
- * publisher-emitted metric registers under this name so a typo can't
6
- * silently split a metric onto a second meter and drop it from the
7
- * Prometheus scrape.
8
- */
9
- export const METER_NAME = "publisher";
10
-
11
- /**
12
- * The shared publisher meter. Resolves lazily through the OTel global
13
- * provider (a `ProxyMeter`), so modules may call this at import time;
14
- * the real provider can be installed afterward (the production SDK in
15
- * `instrumentation.ts`, or the in-memory test harness) and instruments
16
- * still route to it.
17
- */
18
- export function publisherMeter(): Meter {
19
- return metrics.getMeter(METER_NAME);
20
- }
@@ -1,166 +0,0 @@
1
- /**
2
- * Test helper: spin up an in-memory OpenTelemetry MeterProvider so
3
- * unit tests can assert that the new guardrails (admission gate,
4
- * query timeout, query concurrency, heap check) emit the expected
5
- * counters and gauges.
6
- *
7
- * Usage:
8
- *
9
- * const harness = await startMetricsHarness();
10
- * // ... exercise production code ...
11
- * const sums = await harness.collectCounter("publisher_query_timeout_total");
12
- * expect(sums).toBe(1);
13
- * await harness.shutdown();
14
- *
15
- * The OTel JS API resolves `metrics.getMeter(name)` lazily through a
16
- * `ProxyMeter`, so registering the MeterProvider here AFTER the
17
- * production modules have already cached their meter handles works
18
- * — subsequent `.add()` / observable callbacks route to this
19
- * harness's provider.
20
- *
21
- * The harness drains all of the provider's metrics via
22
- * `MetricReader.collect()` and reads the cumulative data points
23
- * directly, bypassing the exporter pipeline so test code doesn't
24
- * have to thread async callbacks.
25
- */
26
-
27
- import { metrics } from "@opentelemetry/api";
28
- import { MeterProvider, MetricReader } from "@opentelemetry/sdk-metrics";
29
-
30
- /**
31
- * The simplest possible MetricReader implementation: a no-op that
32
- * exists only so the `MeterProvider` has a reader to satisfy its
33
- * collection invariants. Tests call `collect()` directly via the
34
- * harness; the reader never pushes anywhere.
35
- */
36
- class CollectingMetricReader extends MetricReader {
37
- protected override async onForceFlush(): Promise<void> {
38
- // no-op; tests pull via `collect()` on demand
39
- }
40
- protected override async onShutdown(): Promise<void> {
41
- // no-op
42
- }
43
- }
44
-
45
- export interface MetricsHarness {
46
- readonly provider: MeterProvider;
47
- /**
48
- * Force a collection cycle and return the cumulative sum of all
49
- * data points for the named counter. Returns 0 when the counter
50
- * has not been emitted yet.
51
- *
52
- * `attributeFilter` lets a test scope the sum to a single label
53
- * set (e.g. only the `environment: "test-env"` data point).
54
- */
55
- collectCounter(
56
- name: string,
57
- attributeFilter?: Record<string, string | number | boolean>,
58
- ): Promise<number>;
59
- /**
60
- * Force a collection cycle and return the most-recently observed
61
- * value of the named gauge, or `undefined` if no callback has
62
- * fired for this gauge yet.
63
- */
64
- collectGauge(name: string): Promise<number | undefined>;
65
- /**
66
- * Force a collection cycle and return the aggregate `count`/`sum` across all
67
- * data points for the named histogram (optionally scoped to one label set).
68
- * `count` is 0 when the histogram has not been emitted yet.
69
- */
70
- collectHistogram(
71
- name: string,
72
- attributeFilter?: Record<string, string | number | boolean>,
73
- ): Promise<{ count: number; sum: number; boundaries: number[] }>;
74
- shutdown(): Promise<void>;
75
- }
76
-
77
- export async function startMetricsHarness(): Promise<MetricsHarness> {
78
- const reader = new CollectingMetricReader();
79
- const provider = new MeterProvider({ readers: [reader] });
80
- // OTel JS's `setGlobalMeterProvider` silently refuses to
81
- // overwrite an existing global (`registerGlobal` returns false).
82
- // Tests that run back-to-back would otherwise route their
83
- // metrics into a dead provider from the previous test. Disable
84
- // first to clear the slot.
85
- metrics.disable();
86
- metrics.setGlobalMeterProvider(provider);
87
-
88
- return {
89
- provider,
90
- async collectCounter(
91
- name: string,
92
- attributeFilter?: Record<string, string | number | boolean>,
93
- ): Promise<number> {
94
- const result = await reader.collect();
95
- let total = 0;
96
- for (const rm of result.resourceMetrics.scopeMetrics) {
97
- for (const metric of rm.metrics) {
98
- if (metric.descriptor.name !== name) continue;
99
- for (const dp of metric.dataPoints) {
100
- if (attributeFilter) {
101
- const allMatch = Object.entries(attributeFilter).every(
102
- ([k, v]) => dp.attributes?.[k] === v,
103
- );
104
- if (!allMatch) continue;
105
- }
106
- total += dp.value as number;
107
- }
108
- }
109
- }
110
- return total;
111
- },
112
- async collectHistogram(
113
- name: string,
114
- attributeFilter?: Record<string, string | number | boolean>,
115
- ): Promise<{ count: number; sum: number; boundaries: number[] }> {
116
- const result = await reader.collect();
117
- let count = 0;
118
- let sum = 0;
119
- let boundaries: number[] = [];
120
- for (const rm of result.resourceMetrics.scopeMetrics) {
121
- for (const metric of rm.metrics) {
122
- if (metric.descriptor.name !== name) continue;
123
- for (const dp of metric.dataPoints) {
124
- if (attributeFilter) {
125
- const allMatch = Object.entries(attributeFilter).every(
126
- ([k, v]) => dp.attributes?.[k] === v,
127
- );
128
- if (!allMatch) continue;
129
- }
130
- const point = dp.value as {
131
- count: number;
132
- sum?: number;
133
- buckets?: { boundaries: number[] };
134
- };
135
- count += point.count;
136
- sum += point.sum ?? 0;
137
- if (point.buckets) boundaries = point.buckets.boundaries;
138
- }
139
- }
140
- }
141
- return { count, sum, boundaries };
142
- },
143
- async collectGauge(name: string): Promise<number | undefined> {
144
- const result = await reader.collect();
145
- for (const rm of result.resourceMetrics.scopeMetrics) {
146
- for (const metric of rm.metrics) {
147
- if (metric.descriptor.name !== name) continue;
148
- // Observable gauges yield one data point per
149
- // attribute set per collection. Return the last
150
- // observed value across all data points so unlabeled
151
- // gauges (the common case in this code base) just
152
- // work without attribute plumbing.
153
- const last = metric.dataPoints[metric.dataPoints.length - 1];
154
- return last?.value as number | undefined;
155
- }
156
- }
157
- return undefined;
158
- },
159
- async shutdown(): Promise<void> {
160
- await reader.shutdown();
161
- // Clear the global so the next harness's
162
- // `setGlobalMeterProvider` is accepted.
163
- metrics.disable();
164
- },
165
- };
166
- }
@@ -1,57 +0,0 @@
1
- import { describe, expect, it } from "bun:test";
2
- import { readFileSync } from "fs";
3
- import { resolve } from "path";
4
- import { PER_MODE_COLOR_KEYS } from "./config";
5
-
6
- /**
7
- * The per-mode palette key list is hand-copied in three places: this server
8
- * copy (config.ts), the SDK copy (packages/sdk/src/theme/keys.ts), and the
9
- * api-doc.yaml Theme.palette schema. The server and the SDK are intentionally
10
- * decoupled (neither imports the other), and api-doc.yaml is their only shared
11
- * contract, so nothing but this test keeps the three in step. A silent drift
12
- * drops a colour: the server sanitizer or the SDK resolver ignores a key that
13
- * is not in its own copy.
14
- */
15
-
16
- function extractTsArray(source: string, constName: string): string[] {
17
- const decl = source.indexOf(`${constName} = [`);
18
- if (decl === -1) throw new Error(`${constName} array not found`);
19
- const open = source.indexOf("[", decl);
20
- const close = source.indexOf("]", open);
21
- return [...source.slice(open + 1, close).matchAll(/"([^"]+)"/g)].map(
22
- (m) => m[1],
23
- );
24
- }
25
-
26
- function extractApiDocPaletteKeys(apiDoc: string): string[] {
27
- const start = apiDoc.indexOf("\n palette:");
28
- if (start === -1) throw new Error("palette block not found in api-doc.yaml");
29
- const end = apiDoc.indexOf("\n font:", start);
30
- const block = apiDoc.slice(start, end === -1 ? undefined : end);
31
- // Direct children of `palette.properties` sit at 12-space indentation;
32
- // deeper keys (type, properties, light, dark) are more indented and skipped.
33
- return [...block.matchAll(/^ {12}(\w+):$/gm)].map((m) => m[1]);
34
- }
35
-
36
- describe("per-mode palette key parity", () => {
37
- const serverKeys = [...PER_MODE_COLOR_KEYS];
38
-
39
- it("server config.ts matches the SDK keys.ts copy", () => {
40
- const sdk = readFileSync(
41
- resolve(import.meta.dir, "../../sdk/src/theme/keys.ts"),
42
- "utf8",
43
- );
44
- expect(extractTsArray(sdk, "PER_MODE_COLOR_KEYS")).toEqual(serverKeys);
45
- });
46
-
47
- it("server config.ts matches the api-doc.yaml Theme.palette schema", () => {
48
- const apiDoc = readFileSync(
49
- resolve(import.meta.dir, "../../../api-doc.yaml"),
50
- "utf8",
51
- );
52
- // palette also declares `series` (a shared array, not a per-mode object).
53
- expect(new Set(extractApiDocPaletteKeys(apiDoc))).toEqual(
54
- new Set([...serverKeys, "series"]),
55
- );
56
- });
57
- });
package/src/utils.ts DELETED
@@ -1,29 +0,0 @@
1
- import { URLReader } from "@malloydata/malloy";
2
- import * as fs from "fs";
3
- import * as path from "path";
4
- import { fileURLToPath } from "url";
5
-
6
- export const URL_READER: URLReader = {
7
- readURL: (url: URL) => {
8
- let path = url.toString();
9
- if (url.protocol == "file:") {
10
- path = fileURLToPath(url);
11
- }
12
- return fs.promises.readFile(path, "utf8");
13
- },
14
- };
15
-
16
- /**
17
- * Skip dotfiles/dotdirs (.vscode, .git, .DS_Store, etc.) when walking a
18
- * package tree. These come from editors/VCS, never contain Malloy models
19
- * or databases, and have been a source of spurious ENOENTs when their
20
- * contents disappear mid-scan.
21
- */
22
- export function ignoreDotfiles(file: string): boolean {
23
- return path.basename(file).startsWith(".");
24
- }
25
-
26
- /** The message of a thrown value, stringifying non-Error throws. */
27
- export function errMessage(err: unknown): string {
28
- return err instanceof Error ? err.message : String(err);
29
- }
@@ -1,9 +0,0 @@
1
- ##! experimental.givens
2
-
3
- given:
4
- ROLE :: string
5
-
6
- #(authorize) "$ROLE = 'analyst'"
7
- source: gated is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }
8
-
9
- source: open_src is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }
@@ -1,4 +0,0 @@
1
- {
2
- "name": "authorize-compile",
3
- "description": "Fixture: a gated source for the /compile authorize HTTP integration test."
4
- }
@@ -1 +0,0 @@
1
- source: nums is duckdb.sql("SELECT 1 as n")
@@ -1,5 +0,0 @@
1
- {
2
- "name": "html-pages-nopublic",
3
- "version": "0.0.1",
4
- "description": "Package with no public/ directory; static file requests return 404."
5
- }
@@ -1,3 +0,0 @@
1
- id,name
2
- 1,alpha
3
- 2,beta
@@ -1,3 +0,0 @@
1
- body {
2
- font-family: system-ui, sans-serif;
3
- }
@@ -1,4 +0,0 @@
1
- <!doctype html>
2
- <title>Bare HTML</title>
3
- <!-- example only, must NOT opt in: <meta name="publisher:fit" content="viewport"> -->
4
- <h1>No head or body tags; the tag appears only in a comment.</h1>
@@ -1,8 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <title>Body Meta</title>
4
- <body>
5
- <meta name="publisher:fit" content="viewport" />
6
- <h1>A real fit meta in &lt;body&gt;; the head-only scan means no opt-in.</h1>
7
- </body>
8
- </html>
@@ -1 +0,0 @@
1
- { "ok": true }
@@ -1,11 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <!-- this deck does <body> styling tricks; the literal must not truncate the scan -->
5
- <meta name="publisher:fit" content="viewport" />
6
- <title>Fit After Comment</title>
7
- </head>
8
- <body style="height: 100vh; margin: 0; overflow: hidden">
9
- <section style="height: 100%">A full-screen slide</section>
10
- </body>
11
- </html>
@@ -1,9 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <title>Carrier Dashboard</title>
5
- </head>
6
- <body>
7
- <h1>Hello from the in-package data app</h1>
8
- </body>
9
- </html>
@@ -1,8 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <title>No Head Close Tag</title>
4
- <body>
5
- <h1>Omits the optional &lt;/head&gt; end tag</h1>
6
- <!-- shown in body, must NOT opt in: <meta name="publisher:fit" content="viewport"> -->
7
- </body>
8
- </html>
@@ -1,13 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <meta charset="utf-8" />
5
- <meta name="viewport" content="width=device-width, initial-scale=1" />
6
- <meta data-name="publisher:fit" content="viewport" />
7
- <title>Tall Dashboard</title>
8
- </head>
9
- <body>
10
- <h1>A normal dashboard that hugs its content</h1>
11
- <!-- shown in body, must NOT opt in: <meta name="publisher:fit" content="viewport"> -->
12
- </body>
13
- </html>
@@ -1,12 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <meta charset="utf-8" />
5
- <meta name="viewport" content="width=device-width, initial-scale=1" />
6
- <meta name="publisher:fit" content="viewport" />
7
- <title>FY27 Slide Deck</title>
8
- </head>
9
- <body style="height: 100vh; margin: 0; overflow: hidden">
10
- <section style="height: 100%">A full-screen slide</section>
11
- </body>
12
- </html>
@@ -1,9 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <title>Second Page</title>
5
- </head>
6
- <body>
7
- <p>A page in a subdirectory.</p>
8
- </body>
9
- </html>
@@ -1,10 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <title>Unterminated Comment</title>
5
- <!-- never closed, must NOT opt in: <meta name="publisher:fit" content="viewport">
6
- </head>
7
- <body>
8
- <h1>An unterminated comment swallows the tag.</h1>
9
- </body>
10
- </html>
@@ -1,5 +0,0 @@
1
- {
2
- "name": "html-pages-test",
3
- "version": "1.0.0",
4
- "description": "Fixture for in-package HTML static-file serving + /pages + /events"
5
- }
@@ -1 +0,0 @@
1
- source: report is duckdb.sql("SELECT 1 as n")
@@ -1,5 +0,0 @@
1
- category,amount
2
- electronics,100
3
- clothing,50
4
- electronics,200
5
- clothing,75
@@ -1,18 +0,0 @@
1
- ##! experimental.persistence
2
-
3
- source: raw_orders is duckdb.table('data/orders.csv')
4
-
5
- // Upstream persist source: an aggregate over the base CSV.
6
- #@ persist name="orders_base"
7
- source: orders_base is raw_orders -> {
8
- group_by: category
9
- aggregate: total_amount is amount.sum()
10
- }
11
-
12
- // Downstream persist source: references the persist upstream `orders_base`.
13
- // A per-unit build of `orders_rollup` alone must REFERENCE orders_base's
14
- // materialized table (via referenceManifest) rather than recompute it live.
15
- #@ persist name="orders_rollup"
16
- source: orders_rollup is orders_base -> {
17
- aggregate: category_count is count()
18
- }
@@ -1,5 +0,0 @@
1
- {
2
- "name": "persist-multi-level",
3
- "version": "1.0.0",
4
- "description": "DuckDB two-level persist DAG fixture (orders_base -> orders_rollup)"
5
- }
@@ -1,5 +0,0 @@
1
- category,amount
2
- electronics,100
3
- clothing,50
4
- electronics,200
5
- clothing,75
@@ -1,11 +0,0 @@
1
- ##! experimental.persistence
2
-
3
- source: raw_orders is duckdb.table('data/orders.csv')
4
-
5
- #@ persist name="order_summary"
6
- source: order_summary is raw_orders -> {
7
- group_by: category
8
- aggregate:
9
- total_orders is count()
10
- total_revenue is amount.sum()
11
- }
@@ -1,7 +0,0 @@
1
- {
2
- "name": "persist-schedule-test",
3
- "version": "1.0.0",
4
- "description": "DuckDB persist fixture with a version-scoped materialization schedule",
5
- "scope": "version",
6
- "materialization": { "schedule": "0 6 * * *" }
7
- }
@@ -1,5 +0,0 @@
1
- category,amount
2
- electronics,100
3
- clothing,50
4
- electronics,200
5
- clothing,75