@malloy-publisher/server 0.0.235 → 0.0.236

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (301) hide show
  1. package/dist/app/api-doc.yaml +47 -17
  2. package/dist/package_load_worker.mjs +8 -1
  3. package/dist/server.mjs +471 -54
  4. package/package.json +5 -2
  5. package/.eslintrc.json +0 -44
  6. package/.prettierignore +0 -1
  7. package/.prettierrc +0 -4
  8. package/build.ts +0 -97
  9. package/eslint.config.mjs +0 -76
  10. package/k6-tests/README.md +0 -144
  11. package/k6-tests/bun.lock +0 -797
  12. package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
  13. package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
  14. package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
  15. package/k6-tests/load-test/load-test-crud.ts +0 -142
  16. package/k6-tests/load-test/load-test.ts +0 -784
  17. package/k6-tests/package.json +0 -27
  18. package/k6-tests/scripts/run-with-otel.sh +0 -50
  19. package/k6-tests/smoke-test/smoke-test.ts +0 -158
  20. package/k6-tests/tsconfig.json +0 -29
  21. package/k6-tests/utils/client_factory.ts +0 -128
  22. package/k6-tests/utils/common.ts +0 -489
  23. package/k6-tests/utils/logger.ts +0 -62
  24. package/k6-tests/utils/types.d.ts +0 -35
  25. package/malloy_mcp.dxt +0 -0
  26. package/publisher.config.example.bigquery.json +0 -33
  27. package/publisher.config.example.duckdb.json +0 -23
  28. package/publisher.config.json +0 -23
  29. package/scripts/bake-duckdb-extensions.js +0 -110
  30. package/src/config.spec.ts +0 -1535
  31. package/src/config.theme.spec.ts +0 -178
  32. package/src/config.ts +0 -1142
  33. package/src/constants.ts +0 -105
  34. package/src/controller/compile.controller.ts +0 -40
  35. package/src/controller/connection.controller.spec.ts +0 -849
  36. package/src/controller/connection.controller.ts +0 -901
  37. package/src/controller/database.controller.ts +0 -24
  38. package/src/controller/materialization.controller.spec.ts +0 -293
  39. package/src/controller/materialization.controller.ts +0 -319
  40. package/src/controller/model.controller.ts +0 -164
  41. package/src/controller/package.controller.spec.ts +0 -273
  42. package/src/controller/package.controller.ts +0 -306
  43. package/src/controller/query.controller.ts +0 -152
  44. package/src/controller/theme.controller.ts +0 -83
  45. package/src/controller/watch-mode.controller.ts +0 -243
  46. package/src/data_styles.ts +0 -71
  47. package/src/default-publisher.config.json +0 -23
  48. package/src/ducklake_version.spec.ts +0 -163
  49. package/src/ducklake_version.ts +0 -153
  50. package/src/errors.spec.ts +0 -119
  51. package/src/errors.ts +0 -237
  52. package/src/filter_deprecation.spec.ts +0 -64
  53. package/src/filter_deprecation.ts +0 -42
  54. package/src/health.spec.ts +0 -123
  55. package/src/health.ts +0 -233
  56. package/src/heap_check.spec.ts +0 -144
  57. package/src/heap_check.ts +0 -144
  58. package/src/instrumentation.ts +0 -214
  59. package/src/json_utils.spec.ts +0 -51
  60. package/src/json_utils.ts +0 -33
  61. package/src/logger.spec.ts +0 -210
  62. package/src/logger.ts +0 -251
  63. package/src/malloy_pin_prereqs.spec.ts +0 -25
  64. package/src/materialization_metrics.spec.ts +0 -169
  65. package/src/materialization_metrics.ts +0 -346
  66. package/src/mcp/error_messages.spec.ts +0 -35
  67. package/src/mcp/error_messages.ts +0 -183
  68. package/src/mcp/handler_utils.spec.ts +0 -108
  69. package/src/mcp/handler_utils.ts +0 -251
  70. package/src/mcp/mcp_constants.ts +0 -6
  71. package/src/mcp/query_envelope.spec.ts +0 -229
  72. package/src/mcp/query_envelope.ts +0 -240
  73. package/src/mcp/server.protocol.spec.ts +0 -250
  74. package/src/mcp/server.ts +0 -87
  75. package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
  76. package/src/mcp/skills/build_skills_bundle.ts +0 -176
  77. package/src/mcp/skills/skills_bundle.json +0 -1
  78. package/src/mcp/skills/skills_bundle.spec.ts +0 -171
  79. package/src/mcp/tool_response.spec.ts +0 -108
  80. package/src/mcp/tool_response.ts +0 -138
  81. package/src/mcp/tools/compile_tool.spec.ts +0 -315
  82. package/src/mcp/tools/compile_tool.ts +0 -208
  83. package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
  84. package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
  85. package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
  86. package/src/mcp/tools/docs_search_tool.ts +0 -128
  87. package/src/mcp/tools/embedding_index.spec.ts +0 -1236
  88. package/src/mcp/tools/embedding_index.ts +0 -808
  89. package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
  90. package/src/mcp/tools/execute_query_tool.ts +0 -316
  91. package/src/mcp/tools/get_context_eval.ts +0 -275
  92. package/src/mcp/tools/get_context_tool.spec.ts +0 -592
  93. package/src/mcp/tools/get_context_tool.ts +0 -585
  94. package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
  95. package/src/mcp/tools/reload_package_tool.ts +0 -132
  96. package/src/mcp_config.spec.ts +0 -919
  97. package/src/mcp_config.ts +0 -425
  98. package/src/oom_guards.integration.spec.ts +0 -269
  99. package/src/package_load/package_load_pool.spec.ts +0 -252
  100. package/src/package_load/package_load_pool.ts +0 -931
  101. package/src/package_load/package_load_worker.ts +0 -1034
  102. package/src/package_load/protocol.ts +0 -364
  103. package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
  104. package/src/package_load/rpc_wait_accountant.ts +0 -76
  105. package/src/package_load_metrics.spec.ts +0 -114
  106. package/src/package_load_metrics.ts +0 -127
  107. package/src/path_safety.spec.ts +0 -158
  108. package/src/path_safety.ts +0 -146
  109. package/src/pg_helpers.spec.ts +0 -223
  110. package/src/pg_helpers.ts +0 -52
  111. package/src/query_cap_metrics.spec.ts +0 -89
  112. package/src/query_cap_metrics.ts +0 -116
  113. package/src/query_concurrency.spec.ts +0 -247
  114. package/src/query_concurrency.ts +0 -237
  115. package/src/query_metadata_metrics.ts +0 -49
  116. package/src/query_param_utils.ts +0 -29
  117. package/src/query_timeout.spec.ts +0 -224
  118. package/src/query_timeout.ts +0 -179
  119. package/src/runtime/publisher.js +0 -328
  120. package/src/server-old.ts +0 -988
  121. package/src/server.ts +0 -2059
  122. package/src/service/annotations.spec.ts +0 -118
  123. package/src/service/annotations.ts +0 -91
  124. package/src/service/authorize.spec.ts +0 -154
  125. package/src/service/authorize.ts +0 -498
  126. package/src/service/authorize_integration.spec.ts +0 -2041
  127. package/src/service/build_plan.spec.ts +0 -685
  128. package/src/service/build_plan.ts +0 -797
  129. package/src/service/compile_authorize.spec.ts +0 -85
  130. package/src/service/compile_fragment_techniques.spec.ts +0 -156
  131. package/src/service/connection.spec.ts +0 -2360
  132. package/src/service/connection.ts +0 -2187
  133. package/src/service/connection_config.spec.ts +0 -734
  134. package/src/service/connection_config.ts +0 -899
  135. package/src/service/connection_federation.spec.ts +0 -184
  136. package/src/service/connection_fingerprint.spec.ts +0 -102
  137. package/src/service/connection_service.spec.ts +0 -711
  138. package/src/service/connection_service.ts +0 -291
  139. package/src/service/cron_evaluator.spec.ts +0 -79
  140. package/src/service/cron_evaluator.ts +0 -105
  141. package/src/service/db_utils.spec.ts +0 -1085
  142. package/src/service/db_utils.ts +0 -1502
  143. package/src/service/duckdb_instance_isolation.spec.ts +0 -137
  144. package/src/service/ducklake_lazy_attach.spec.ts +0 -110
  145. package/src/service/embedding_provider.spec.ts +0 -329
  146. package/src/service/embedding_provider.ts +0 -236
  147. package/src/service/environment.ts +0 -2191
  148. package/src/service/environment_admission.spec.ts +0 -344
  149. package/src/service/environment_compile.spec.ts +0 -197
  150. package/src/service/environment_store.spec.ts +0 -2274
  151. package/src/service/environment_store.ts +0 -2434
  152. package/src/service/environment_store_anchoring.spec.ts +0 -107
  153. package/src/service/environment_store_clone.spec.ts +0 -350
  154. package/src/service/explore_visibility.spec.ts +0 -434
  155. package/src/service/exports_probe.spec.ts +0 -107
  156. package/src/service/extension_fetch_policy.spec.ts +0 -256
  157. package/src/service/filter.spec.ts +0 -458
  158. package/src/service/filter.ts +0 -341
  159. package/src/service/filter_bypass.spec.ts +0 -418
  160. package/src/service/filter_integration.spec.ts +0 -1004
  161. package/src/service/freshness.spec.ts +0 -183
  162. package/src/service/freshness.ts +0 -112
  163. package/src/service/gcs_s3_utils.ts +0 -427
  164. package/src/service/given.ts +0 -105
  165. package/src/service/givens_integration.spec.ts +0 -219
  166. package/src/service/manifest_loader.spec.ts +0 -189
  167. package/src/service/manifest_loader.ts +0 -156
  168. package/src/service/materialization_build_session.spec.ts +0 -435
  169. package/src/service/materialization_build_session.ts +0 -681
  170. package/src/service/materialization_config_validation.spec.ts +0 -99
  171. package/src/service/materialization_config_validation.ts +0 -120
  172. package/src/service/materialization_eligibility.spec.ts +0 -158
  173. package/src/service/materialization_eligibility.ts +0 -305
  174. package/src/service/materialization_schedule_surface.spec.ts +0 -256
  175. package/src/service/materialization_scheduler.spec.ts +0 -289
  176. package/src/service/materialization_scheduler.ts +0 -304
  177. package/src/service/materialization_serve_transform.spec.ts +0 -1003
  178. package/src/service/materialization_serve_transform.ts +0 -779
  179. package/src/service/materialization_service.spec.ts +0 -2373
  180. package/src/service/materialization_service.ts +0 -2342
  181. package/src/service/materialization_test_fixtures.ts +0 -249
  182. package/src/service/model.spec.ts +0 -1157
  183. package/src/service/model.ts +0 -3370
  184. package/src/service/model_limits.spec.ts +0 -209
  185. package/src/service/model_limits.ts +0 -131
  186. package/src/service/model_storage_serve.spec.ts +0 -193
  187. package/src/service/model_storage_serve_joins.spec.ts +0 -193
  188. package/src/service/package.spec.ts +0 -612
  189. package/src/service/package.ts +0 -1799
  190. package/src/service/package_manifest.spec.ts +0 -230
  191. package/src/service/package_manifest.ts +0 -241
  192. package/src/service/package_memory_governor.spec.ts +0 -173
  193. package/src/service/package_memory_governor.ts +0 -233
  194. package/src/service/package_quote_bound_tables.spec.ts +0 -171
  195. package/src/service/package_race.spec.ts +0 -208
  196. package/src/service/package_reload_safety.spec.ts +0 -193
  197. package/src/service/package_rollback.spec.ts +0 -190
  198. package/src/service/package_worker_path.spec.ts +0 -645
  199. package/src/service/path_injection.spec.ts +0 -39
  200. package/src/service/persist_annotation_validation.spec.ts +0 -89
  201. package/src/service/persist_annotation_validation.ts +0 -59
  202. package/src/service/persistence_policy.spec.ts +0 -374
  203. package/src/service/proxy.spec.ts +0 -414
  204. package/src/service/proxy.ts +0 -248
  205. package/src/service/query_boundary.spec.ts +0 -470
  206. package/src/service/query_metadata.spec.ts +0 -408
  207. package/src/service/query_metadata.ts +0 -492
  208. package/src/service/query_metadata_identity.spec.ts +0 -149
  209. package/src/service/query_text.spec.ts +0 -79
  210. package/src/service/query_text.ts +0 -43
  211. package/src/service/quoting.spec.ts +0 -135
  212. package/src/service/quoting.ts +0 -78
  213. package/src/service/resolve_environment.ts +0 -15
  214. package/src/service/restricted_mode.spec.ts +0 -299
  215. package/src/service/source_extraction.ts +0 -225
  216. package/src/service/theme_store.ts +0 -199
  217. package/src/storage/DatabaseInterface.ts +0 -198
  218. package/src/storage/StorageManager.ts +0 -113
  219. package/src/storage/duckdb/ConnectionRepository.ts +0 -143
  220. package/src/storage/duckdb/DuckDBConnection.ts +0 -123
  221. package/src/storage/duckdb/DuckDBRepository.ts +0 -208
  222. package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
  223. package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
  224. package/src/storage/duckdb/MaterializationRepository.ts +0 -336
  225. package/src/storage/duckdb/PackageRepository.ts +0 -135
  226. package/src/storage/duckdb/schema.ts +0 -229
  227. package/src/stream_helpers.spec.ts +0 -280
  228. package/src/stream_helpers.ts +0 -162
  229. package/src/telemetry.ts +0 -20
  230. package/src/test_helpers/metrics_harness.ts +0 -166
  231. package/src/theme_key_parity.spec.ts +0 -57
  232. package/src/utils.ts +0 -29
  233. package/tests/fixtures/authorize-compile/model.malloy +0 -9
  234. package/tests/fixtures/authorize-compile/publisher.json +0 -4
  235. package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
  236. package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
  237. package/tests/fixtures/html-pages-test/data.csv +0 -3
  238. package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
  239. package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
  240. package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
  241. package/tests/fixtures/html-pages-test/public/data.json +0 -1
  242. package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
  243. package/tests/fixtures/html-pages-test/public/index.html +0 -9
  244. package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
  245. package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
  246. package/tests/fixtures/html-pages-test/public/slides.html +0 -12
  247. package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
  248. package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
  249. package/tests/fixtures/html-pages-test/publisher.json +0 -5
  250. package/tests/fixtures/html-pages-test/report.malloy +0 -1
  251. package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
  252. package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
  253. package/tests/fixtures/persist-multi-level/publisher.json +0 -5
  254. package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
  255. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
  256. package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
  257. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
  258. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
  259. package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
  260. package/tests/fixtures/persist-test/data/orders.csv +0 -5
  261. package/tests/fixtures/persist-test/persist_test.malloy +0 -11
  262. package/tests/fixtures/persist-test/publisher.json +0 -5
  263. package/tests/fixtures/publisher.config.json +0 -15
  264. package/tests/fixtures/query-givens/data/orders.csv +0 -7
  265. package/tests/fixtures/query-givens/model.malloy +0 -34
  266. package/tests/fixtures/query-givens/publisher.json +0 -5
  267. package/tests/fixtures/xlsx/database.xlsx +0 -0
  268. package/tests/harness/mcp_test_setup.ts +0 -332
  269. package/tests/harness/rest_e2e.ts +0 -68
  270. package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
  271. package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
  272. package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
  273. package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
  274. package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
  275. package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
  276. package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
  277. package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
  278. package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
  279. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
  280. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
  281. package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
  282. package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
  283. package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
  284. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
  285. package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
  286. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
  287. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
  288. package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
  289. package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
  290. package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
  291. package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
  292. package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
  293. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
  294. package/tests/unit/controller/theme.controller.spec.ts +0 -141
  295. package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
  296. package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
  297. package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
  298. package/tests/unit/duckdb/repositories.test.ts +0 -208
  299. package/tests/unit/ducklake/ducklake.test.ts +0 -952
  300. package/tests/unit/service/theme_store.spec.ts +0 -139
  301. package/tsconfig.json +0 -32
@@ -1,162 +0,0 @@
1
- /**
2
- * Helpers for streaming the ad-hoc connection SQL endpoints
3
- * (`/environments/.../connections/.../sqlQuery`) so the publisher
4
- * process never has to hold a whole result set in memory before
5
- * returning it.
6
- *
7
- * The Step 1 row cap (`PUBLISHER_MAX_QUERY_ROWS`) is necessary but
8
- * not sufficient: row count is a poor proxy for memory pressure
9
- * because a single 10 MB JSON column blows past the 100k-row cap's
10
- * safe envelope. The byte cap (`PUBLISHER_MAX_RESPONSE_BYTES`) is
11
- * the actual memory bound, and bytes can only be enforced by
12
- * iterating row-at-a-time on `runSQLStream` — `runSQL` returns a
13
- * fully-buffered result, so by the time we'd count bytes the
14
- * connector has already done the damage.
15
- *
16
- * On streaming-capable connections (Postgres, DuckDB, ...) the
17
- * controller routes here. On other connections it stays on the
18
- * Step 1 path; client-side byte counting after the fact would be
19
- * security theatre.
20
- */
21
-
22
- import type {
23
- Connection,
24
- MalloyQueryData,
25
- QueryRecord,
26
- RunSQLOptions,
27
- StreamingConnection,
28
- } from "@malloydata/malloy";
29
-
30
- import { PayloadTooLargeError } from "./errors";
31
- import { recordQueryCapExceeded } from "./query_cap_metrics";
32
-
33
- /**
34
- * Runtime check + type narrow for streaming-capable connections.
35
- * `Connection.canStream` is declared as `this is StreamingConnection`
36
- * by the Malloy SDK, so a positive result is enough to safely call
37
- * `runSQLStream`.
38
- */
39
- export function isStreamingConnection(
40
- connection: Connection,
41
- ): connection is StreamingConnection {
42
- return (
43
- typeof (connection as { canStream?: () => boolean }).canStream ===
44
- "function" && (connection as StreamingConnection).canStream()
45
- );
46
- }
47
-
48
- export interface StreamBudget {
49
- /**
50
- * Maximum number of rows to return. A value of `0` disables the
51
- * row cap (the caller-supplied `rowLimit` in `runSQLOptions` may
52
- * still bound the stream, but the helper will not raise on
53
- * overflow).
54
- */
55
- maxRows: number;
56
- /**
57
- * Maximum aggregate JSON-serialized byte size of returned rows.
58
- * `0` disables the byte cap. Measured as the sum of
59
- * `Buffer.byteLength(JSON.stringify(row))` for each yielded row
60
- * — the same bytes the eventual response will contain inside
61
- * `result.rows`.
62
- */
63
- maxBytes: number;
64
- }
65
-
66
- /**
67
- * Drain `runSQLStream` into a `MalloyQueryData`-shaped buffer,
68
- * enforcing both a row cap and a byte cap. Aborts the underlying
69
- * iterator via an internal `AbortController` the moment either cap
70
- * is breached so the driver stops producing rows immediately
71
- * (Postgres' `pg-query-stream` and DuckDB's streaming iterator both
72
- * honor `abortSignal`).
73
- *
74
- * The row cap is detected by the `cap + 1` sentinel pattern: the
75
- * controller has already clamped `runSQLOptions.rowLimit` to
76
- * `min(callerLimit, cap + 1)`, so receiving `cap + 1` rows
77
- * unambiguously means the request would have overflowed. This
78
- * matches the non-streaming path's overflow detection so behavior
79
- * is identical regardless of which connector served the request.
80
- *
81
- * On overflow the helper throws `PayloadTooLargeError` directly —
82
- * the message includes the relevant env-var name so the operator
83
- * sees a self-contained tuning hint without having to cross-
84
- * reference the controller.
85
- */
86
- export async function streamSqlWithBudget(
87
- connection: StreamingConnection,
88
- sql: string,
89
- runSQLOptions: RunSQLOptions,
90
- budget: StreamBudget,
91
- ): Promise<MalloyQueryData> {
92
- const { maxRows, maxBytes } = budget;
93
- const capAc = new AbortController();
94
- const rows: QueryRecord[] = [];
95
- let byteTotal = 0;
96
- let overflowMessage: string | undefined;
97
-
98
- // Compose two abort sources so a caller-supplied signal (the
99
- // publisher's query timeout) and the internal cap-abort signal
100
- // *both* cancel the underlying iterator:
101
- //
102
- // - If the caller's signal fires first, the controller's
103
- // `runWithQueryTimeout` will throw `QueryTimeoutError` → 504.
104
- // - If the cap-abort fires first, we throw
105
- // `PayloadTooLargeError` → 413.
106
- //
107
- // Without composition the streaming branch would silently drop
108
- // the caller's signal — historically the only way to abort here
109
- // was the cap, so the legacy controller cleared
110
- // `runSQLOptions.abortSignal`. Step 5 reverses that: the caller's
111
- // signal is now authoritative.
112
- //
113
- // `AbortSignal.any` is widely available (Node 20+); guard with a
114
- // typeof check so a stale runtime falls back to the legacy
115
- // cap-only behavior instead of crashing at module load.
116
- const externalSignal = runSQLOptions.abortSignal;
117
- const composedSignal: AbortSignal =
118
- externalSignal && typeof AbortSignal.any === "function"
119
- ? AbortSignal.any([externalSignal, capAc.signal])
120
- : capAc.signal;
121
-
122
- try {
123
- for await (const row of connection.runSQLStream(sql, {
124
- ...runSQLOptions,
125
- abortSignal: composedSignal,
126
- })) {
127
- rows.push(row);
128
- if (maxBytes > 0) {
129
- // Measure exactly what the eventual response body will
130
- // contain for this row. O(rowSize) per row, duplicated
131
- // against the final `JSON.stringify(result)` — the
132
- // early-abort win on overflow dwarfs the bookkeeping
133
- // cost in the bounded-success case.
134
- byteTotal += Buffer.byteLength(JSON.stringify(row), "utf8");
135
- if (byteTotal > maxBytes) {
136
- recordQueryCapExceeded("bytes", "connection_sql");
137
- overflowMessage = `Query response exceeded ${maxBytes} bytes (had at least ${byteTotal}). Refine the query (project fewer columns, add a LIMIT, or filter wide values) or raise PUBLISHER_MAX_RESPONSE_BYTES.`;
138
- capAc.abort();
139
- break;
140
- }
141
- }
142
- if (maxRows > 0 && rows.length > maxRows) {
143
- recordQueryCapExceeded("rows", "connection_sql");
144
- overflowMessage = `Query returned more than ${maxRows} rows. Refine the query (add a LIMIT or more selective WHERE) or raise PUBLISHER_MAX_QUERY_ROWS.`;
145
- capAc.abort();
146
- break;
147
- }
148
- }
149
- } catch (err) {
150
- // `pg-query-stream` surfaces `query.destroy()` (which our
151
- // abort handler triggers) as a synthetic error in some
152
- // versions. Swallow it iff we triggered the abort ourselves —
153
- // otherwise it's a real connection error (or the caller's
154
- // timeout, which the controller's runWithQueryTimeout will
155
- // surface as a 504) that the controller must see.
156
- if (!overflowMessage) throw err;
157
- }
158
-
159
- if (overflowMessage) throw new PayloadTooLargeError(overflowMessage);
160
-
161
- return { rows, totalRows: rows.length };
162
- }
package/src/telemetry.ts DELETED
@@ -1,20 +0,0 @@
1
- import { metrics, type Meter } from "@opentelemetry/api";
2
-
3
- /**
4
- * Single source of truth for the OpenTelemetry meter name. Every
5
- * publisher-emitted metric registers under this name so a typo can't
6
- * silently split a metric onto a second meter and drop it from the
7
- * Prometheus scrape.
8
- */
9
- export const METER_NAME = "publisher";
10
-
11
- /**
12
- * The shared publisher meter. Resolves lazily through the OTel global
13
- * provider (a `ProxyMeter`), so modules may call this at import time;
14
- * the real provider can be installed afterward (the production SDK in
15
- * `instrumentation.ts`, or the in-memory test harness) and instruments
16
- * still route to it.
17
- */
18
- export function publisherMeter(): Meter {
19
- return metrics.getMeter(METER_NAME);
20
- }
@@ -1,166 +0,0 @@
1
- /**
2
- * Test helper: spin up an in-memory OpenTelemetry MeterProvider so
3
- * unit tests can assert that the new guardrails (admission gate,
4
- * query timeout, query concurrency, heap check) emit the expected
5
- * counters and gauges.
6
- *
7
- * Usage:
8
- *
9
- * const harness = await startMetricsHarness();
10
- * // ... exercise production code ...
11
- * const sums = await harness.collectCounter("publisher_query_timeout_total");
12
- * expect(sums).toBe(1);
13
- * await harness.shutdown();
14
- *
15
- * The OTel JS API resolves `metrics.getMeter(name)` lazily through a
16
- * `ProxyMeter`, so registering the MeterProvider here AFTER the
17
- * production modules have already cached their meter handles works
18
- * — subsequent `.add()` / observable callbacks route to this
19
- * harness's provider.
20
- *
21
- * The harness drains all of the provider's metrics via
22
- * `MetricReader.collect()` and reads the cumulative data points
23
- * directly, bypassing the exporter pipeline so test code doesn't
24
- * have to thread async callbacks.
25
- */
26
-
27
- import { metrics } from "@opentelemetry/api";
28
- import { MeterProvider, MetricReader } from "@opentelemetry/sdk-metrics";
29
-
30
- /**
31
- * The simplest possible MetricReader implementation: a no-op that
32
- * exists only so the `MeterProvider` has a reader to satisfy its
33
- * collection invariants. Tests call `collect()` directly via the
34
- * harness; the reader never pushes anywhere.
35
- */
36
- class CollectingMetricReader extends MetricReader {
37
- protected override async onForceFlush(): Promise<void> {
38
- // no-op; tests pull via `collect()` on demand
39
- }
40
- protected override async onShutdown(): Promise<void> {
41
- // no-op
42
- }
43
- }
44
-
45
- export interface MetricsHarness {
46
- readonly provider: MeterProvider;
47
- /**
48
- * Force a collection cycle and return the cumulative sum of all
49
- * data points for the named counter. Returns 0 when the counter
50
- * has not been emitted yet.
51
- *
52
- * `attributeFilter` lets a test scope the sum to a single label
53
- * set (e.g. only the `environment: "test-env"` data point).
54
- */
55
- collectCounter(
56
- name: string,
57
- attributeFilter?: Record<string, string | number | boolean>,
58
- ): Promise<number>;
59
- /**
60
- * Force a collection cycle and return the most-recently observed
61
- * value of the named gauge, or `undefined` if no callback has
62
- * fired for this gauge yet.
63
- */
64
- collectGauge(name: string): Promise<number | undefined>;
65
- /**
66
- * Force a collection cycle and return the aggregate `count`/`sum` across all
67
- * data points for the named histogram (optionally scoped to one label set).
68
- * `count` is 0 when the histogram has not been emitted yet.
69
- */
70
- collectHistogram(
71
- name: string,
72
- attributeFilter?: Record<string, string | number | boolean>,
73
- ): Promise<{ count: number; sum: number; boundaries: number[] }>;
74
- shutdown(): Promise<void>;
75
- }
76
-
77
- export async function startMetricsHarness(): Promise<MetricsHarness> {
78
- const reader = new CollectingMetricReader();
79
- const provider = new MeterProvider({ readers: [reader] });
80
- // OTel JS's `setGlobalMeterProvider` silently refuses to
81
- // overwrite an existing global (`registerGlobal` returns false).
82
- // Tests that run back-to-back would otherwise route their
83
- // metrics into a dead provider from the previous test. Disable
84
- // first to clear the slot.
85
- metrics.disable();
86
- metrics.setGlobalMeterProvider(provider);
87
-
88
- return {
89
- provider,
90
- async collectCounter(
91
- name: string,
92
- attributeFilter?: Record<string, string | number | boolean>,
93
- ): Promise<number> {
94
- const result = await reader.collect();
95
- let total = 0;
96
- for (const rm of result.resourceMetrics.scopeMetrics) {
97
- for (const metric of rm.metrics) {
98
- if (metric.descriptor.name !== name) continue;
99
- for (const dp of metric.dataPoints) {
100
- if (attributeFilter) {
101
- const allMatch = Object.entries(attributeFilter).every(
102
- ([k, v]) => dp.attributes?.[k] === v,
103
- );
104
- if (!allMatch) continue;
105
- }
106
- total += dp.value as number;
107
- }
108
- }
109
- }
110
- return total;
111
- },
112
- async collectHistogram(
113
- name: string,
114
- attributeFilter?: Record<string, string | number | boolean>,
115
- ): Promise<{ count: number; sum: number; boundaries: number[] }> {
116
- const result = await reader.collect();
117
- let count = 0;
118
- let sum = 0;
119
- let boundaries: number[] = [];
120
- for (const rm of result.resourceMetrics.scopeMetrics) {
121
- for (const metric of rm.metrics) {
122
- if (metric.descriptor.name !== name) continue;
123
- for (const dp of metric.dataPoints) {
124
- if (attributeFilter) {
125
- const allMatch = Object.entries(attributeFilter).every(
126
- ([k, v]) => dp.attributes?.[k] === v,
127
- );
128
- if (!allMatch) continue;
129
- }
130
- const point = dp.value as {
131
- count: number;
132
- sum?: number;
133
- buckets?: { boundaries: number[] };
134
- };
135
- count += point.count;
136
- sum += point.sum ?? 0;
137
- if (point.buckets) boundaries = point.buckets.boundaries;
138
- }
139
- }
140
- }
141
- return { count, sum, boundaries };
142
- },
143
- async collectGauge(name: string): Promise<number | undefined> {
144
- const result = await reader.collect();
145
- for (const rm of result.resourceMetrics.scopeMetrics) {
146
- for (const metric of rm.metrics) {
147
- if (metric.descriptor.name !== name) continue;
148
- // Observable gauges yield one data point per
149
- // attribute set per collection. Return the last
150
- // observed value across all data points so unlabeled
151
- // gauges (the common case in this code base) just
152
- // work without attribute plumbing.
153
- const last = metric.dataPoints[metric.dataPoints.length - 1];
154
- return last?.value as number | undefined;
155
- }
156
- }
157
- return undefined;
158
- },
159
- async shutdown(): Promise<void> {
160
- await reader.shutdown();
161
- // Clear the global so the next harness's
162
- // `setGlobalMeterProvider` is accepted.
163
- metrics.disable();
164
- },
165
- };
166
- }
@@ -1,57 +0,0 @@
1
- import { describe, expect, it } from "bun:test";
2
- import { readFileSync } from "fs";
3
- import { resolve } from "path";
4
- import { PER_MODE_COLOR_KEYS } from "./config";
5
-
6
- /**
7
- * The per-mode palette key list is hand-copied in three places: this server
8
- * copy (config.ts), the SDK copy (packages/sdk/src/theme/keys.ts), and the
9
- * api-doc.yaml Theme.palette schema. The server and the SDK are intentionally
10
- * decoupled (neither imports the other), and api-doc.yaml is their only shared
11
- * contract, so nothing but this test keeps the three in step. A silent drift
12
- * drops a colour: the server sanitizer or the SDK resolver ignores a key that
13
- * is not in its own copy.
14
- */
15
-
16
- function extractTsArray(source: string, constName: string): string[] {
17
- const decl = source.indexOf(`${constName} = [`);
18
- if (decl === -1) throw new Error(`${constName} array not found`);
19
- const open = source.indexOf("[", decl);
20
- const close = source.indexOf("]", open);
21
- return [...source.slice(open + 1, close).matchAll(/"([^"]+)"/g)].map(
22
- (m) => m[1],
23
- );
24
- }
25
-
26
- function extractApiDocPaletteKeys(apiDoc: string): string[] {
27
- const start = apiDoc.indexOf("\n palette:");
28
- if (start === -1) throw new Error("palette block not found in api-doc.yaml");
29
- const end = apiDoc.indexOf("\n font:", start);
30
- const block = apiDoc.slice(start, end === -1 ? undefined : end);
31
- // Direct children of `palette.properties` sit at 12-space indentation;
32
- // deeper keys (type, properties, light, dark) are more indented and skipped.
33
- return [...block.matchAll(/^ {12}(\w+):$/gm)].map((m) => m[1]);
34
- }
35
-
36
- describe("per-mode palette key parity", () => {
37
- const serverKeys = [...PER_MODE_COLOR_KEYS];
38
-
39
- it("server config.ts matches the SDK keys.ts copy", () => {
40
- const sdk = readFileSync(
41
- resolve(import.meta.dir, "../../sdk/src/theme/keys.ts"),
42
- "utf8",
43
- );
44
- expect(extractTsArray(sdk, "PER_MODE_COLOR_KEYS")).toEqual(serverKeys);
45
- });
46
-
47
- it("server config.ts matches the api-doc.yaml Theme.palette schema", () => {
48
- const apiDoc = readFileSync(
49
- resolve(import.meta.dir, "../../../api-doc.yaml"),
50
- "utf8",
51
- );
52
- // palette also declares `series` (a shared array, not a per-mode object).
53
- expect(new Set(extractApiDocPaletteKeys(apiDoc))).toEqual(
54
- new Set([...serverKeys, "series"]),
55
- );
56
- });
57
- });
package/src/utils.ts DELETED
@@ -1,29 +0,0 @@
1
- import { URLReader } from "@malloydata/malloy";
2
- import * as fs from "fs";
3
- import * as path from "path";
4
- import { fileURLToPath } from "url";
5
-
6
- export const URL_READER: URLReader = {
7
- readURL: (url: URL) => {
8
- let path = url.toString();
9
- if (url.protocol == "file:") {
10
- path = fileURLToPath(url);
11
- }
12
- return fs.promises.readFile(path, "utf8");
13
- },
14
- };
15
-
16
- /**
17
- * Skip dotfiles/dotdirs (.vscode, .git, .DS_Store, etc.) when walking a
18
- * package tree. These come from editors/VCS, never contain Malloy models
19
- * or databases, and have been a source of spurious ENOENTs when their
20
- * contents disappear mid-scan.
21
- */
22
- export function ignoreDotfiles(file: string): boolean {
23
- return path.basename(file).startsWith(".");
24
- }
25
-
26
- /** The message of a thrown value, stringifying non-Error throws. */
27
- export function errMessage(err: unknown): string {
28
- return err instanceof Error ? err.message : String(err);
29
- }
@@ -1,9 +0,0 @@
1
- ##! experimental.givens
2
-
3
- given:
4
- ROLE :: string
5
-
6
- #(authorize) "$ROLE = 'analyst'"
7
- source: gated is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }
8
-
9
- source: open_src is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }
@@ -1,4 +0,0 @@
1
- {
2
- "name": "authorize-compile",
3
- "description": "Fixture: a gated source for the /compile authorize HTTP integration test."
4
- }
@@ -1 +0,0 @@
1
- source: nums is duckdb.sql("SELECT 1 as n")
@@ -1,5 +0,0 @@
1
- {
2
- "name": "html-pages-nopublic",
3
- "version": "0.0.1",
4
- "description": "Package with no public/ directory; static file requests return 404."
5
- }
@@ -1,3 +0,0 @@
1
- id,name
2
- 1,alpha
3
- 2,beta
@@ -1,3 +0,0 @@
1
- body {
2
- font-family: system-ui, sans-serif;
3
- }
@@ -1,4 +0,0 @@
1
- <!doctype html>
2
- <title>Bare HTML</title>
3
- <!-- example only, must NOT opt in: <meta name="publisher:fit" content="viewport"> -->
4
- <h1>No head or body tags; the tag appears only in a comment.</h1>
@@ -1,8 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <title>Body Meta</title>
4
- <body>
5
- <meta name="publisher:fit" content="viewport" />
6
- <h1>A real fit meta in &lt;body&gt;; the head-only scan means no opt-in.</h1>
7
- </body>
8
- </html>
@@ -1 +0,0 @@
1
- { "ok": true }
@@ -1,11 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <!-- this deck does <body> styling tricks; the literal must not truncate the scan -->
5
- <meta name="publisher:fit" content="viewport" />
6
- <title>Fit After Comment</title>
7
- </head>
8
- <body style="height: 100vh; margin: 0; overflow: hidden">
9
- <section style="height: 100%">A full-screen slide</section>
10
- </body>
11
- </html>
@@ -1,9 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <title>Carrier Dashboard</title>
5
- </head>
6
- <body>
7
- <h1>Hello from the in-package data app</h1>
8
- </body>
9
- </html>
@@ -1,8 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <title>No Head Close Tag</title>
4
- <body>
5
- <h1>Omits the optional &lt;/head&gt; end tag</h1>
6
- <!-- shown in body, must NOT opt in: <meta name="publisher:fit" content="viewport"> -->
7
- </body>
8
- </html>
@@ -1,13 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <meta charset="utf-8" />
5
- <meta name="viewport" content="width=device-width, initial-scale=1" />
6
- <meta data-name="publisher:fit" content="viewport" />
7
- <title>Tall Dashboard</title>
8
- </head>
9
- <body>
10
- <h1>A normal dashboard that hugs its content</h1>
11
- <!-- shown in body, must NOT opt in: <meta name="publisher:fit" content="viewport"> -->
12
- </body>
13
- </html>
@@ -1,12 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <meta charset="utf-8" />
5
- <meta name="viewport" content="width=device-width, initial-scale=1" />
6
- <meta name="publisher:fit" content="viewport" />
7
- <title>FY27 Slide Deck</title>
8
- </head>
9
- <body style="height: 100vh; margin: 0; overflow: hidden">
10
- <section style="height: 100%">A full-screen slide</section>
11
- </body>
12
- </html>
@@ -1,9 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <title>Second Page</title>
5
- </head>
6
- <body>
7
- <p>A page in a subdirectory.</p>
8
- </body>
9
- </html>
@@ -1,10 +0,0 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <title>Unterminated Comment</title>
5
- <!-- never closed, must NOT opt in: <meta name="publisher:fit" content="viewport">
6
- </head>
7
- <body>
8
- <h1>An unterminated comment swallows the tag.</h1>
9
- </body>
10
- </html>
@@ -1,5 +0,0 @@
1
- {
2
- "name": "html-pages-test",
3
- "version": "1.0.0",
4
- "description": "Fixture for in-package HTML static-file serving + /pages + /events"
5
- }
@@ -1 +0,0 @@
1
- source: report is duckdb.sql("SELECT 1 as n")
@@ -1,5 +0,0 @@
1
- category,amount
2
- electronics,100
3
- clothing,50
4
- electronics,200
5
- clothing,75
@@ -1,18 +0,0 @@
1
- ##! experimental.persistence
2
-
3
- source: raw_orders is duckdb.table('data/orders.csv')
4
-
5
- // Upstream persist source: an aggregate over the base CSV.
6
- #@ persist name="orders_base"
7
- source: orders_base is raw_orders -> {
8
- group_by: category
9
- aggregate: total_amount is amount.sum()
10
- }
11
-
12
- // Downstream persist source: references the persist upstream `orders_base`.
13
- // A per-unit build of `orders_rollup` alone must REFERENCE orders_base's
14
- // materialized table (via referenceManifest) rather than recompute it live.
15
- #@ persist name="orders_rollup"
16
- source: orders_rollup is orders_base -> {
17
- aggregate: category_count is count()
18
- }
@@ -1,5 +0,0 @@
1
- {
2
- "name": "persist-multi-level",
3
- "version": "1.0.0",
4
- "description": "DuckDB two-level persist DAG fixture (orders_base -> orders_rollup)"
5
- }
@@ -1,5 +0,0 @@
1
- category,amount
2
- electronics,100
3
- clothing,50
4
- electronics,200
5
- clothing,75
@@ -1,11 +0,0 @@
1
- ##! experimental.persistence
2
-
3
- source: raw_orders is duckdb.table('data/orders.csv')
4
-
5
- #@ persist name="order_summary"
6
- source: order_summary is raw_orders -> {
7
- group_by: category
8
- aggregate:
9
- total_orders is count()
10
- total_revenue is amount.sum()
11
- }
@@ -1,7 +0,0 @@
1
- {
2
- "name": "persist-schedule-test",
3
- "version": "1.0.0",
4
- "description": "DuckDB persist fixture with a version-scoped materialization schedule",
5
- "scope": "version",
6
- "materialization": { "schedule": "0 6 * * *" }
7
- }
@@ -1,5 +0,0 @@
1
- category,amount
2
- electronics,100
3
- clothing,50
4
- electronics,200
5
- clothing,75