@malloy-publisher/server 0.0.235 → 0.0.237

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (319) hide show
  1. package/dist/app/api-doc.yaml +83 -27
  2. package/dist/app/assets/{EnvironmentPage-BsAnavYN.js → EnvironmentPage-DaP6CpgK.js} +1 -1
  3. package/dist/app/assets/{HomePage-CADE138j.js → HomePage-C9yNw3iu.js} +1 -1
  4. package/dist/app/assets/{LightMode-Cfh7KzN8.js → LightMode-Be4-G-dX.js} +1 -1
  5. package/dist/app/assets/{MainPage-CO3pRlnV.js → MainPage-J_rU2SgI.js} +2 -2
  6. package/dist/app/assets/{MaterializationsPage-p9YjkRXZ.js → MaterializationsPage-DHgPI4MO.js} +1 -1
  7. package/dist/app/assets/ModelPage-BzXfjTu5.js +1 -0
  8. package/dist/app/assets/{PackagePage-e4kN75YR.js → PackagePage-ByoSTLJ4.js} +1 -1
  9. package/dist/app/assets/RouteError-CrSeKN2r.js +1 -0
  10. package/dist/app/assets/{ThemeEditorPage-CciagFTq.js → ThemeEditorPage-Btxnmhtk.js} +1 -1
  11. package/dist/app/assets/{WorkbookPage-DNWmkCXa.js → WorkbookPage-D6JLPnWO.js} +1 -1
  12. package/dist/app/assets/{core-Rj_4rRnA.es-BZyvITuO.js → core-BSRgt1mG.es-Bc_D2Lq4.js} +1 -1
  13. package/dist/app/assets/{index-CH2AcDzc.js → index-BRijWj6K.js} +15 -15
  14. package/dist/app/assets/{index-DQa463gC.js → index-BX4D94xw.js} +1 -1
  15. package/dist/app/assets/{index-VBbcc8s6.js → index-Cvy0xtb7.js} +1 -1
  16. package/dist/app/assets/{index-DQpV7MyA.js → index-DMAhIJNC.js} +1 -1
  17. package/dist/app/index.html +1 -1
  18. package/dist/package_load_worker.mjs +59 -20
  19. package/dist/server.mjs +1342 -616
  20. package/package.json +8 -2
  21. package/.eslintrc.json +0 -44
  22. package/.prettierignore +0 -1
  23. package/.prettierrc +0 -4
  24. package/build.ts +0 -97
  25. package/dist/app/assets/ModelPage-C1OSTv-x.js +0 -1
  26. package/dist/app/assets/RouteError-CzbfOkng.js +0 -1
  27. package/eslint.config.mjs +0 -76
  28. package/k6-tests/README.md +0 -144
  29. package/k6-tests/bun.lock +0 -797
  30. package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
  31. package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
  32. package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
  33. package/k6-tests/load-test/load-test-crud.ts +0 -142
  34. package/k6-tests/load-test/load-test.ts +0 -784
  35. package/k6-tests/package.json +0 -27
  36. package/k6-tests/scripts/run-with-otel.sh +0 -50
  37. package/k6-tests/smoke-test/smoke-test.ts +0 -158
  38. package/k6-tests/tsconfig.json +0 -29
  39. package/k6-tests/utils/client_factory.ts +0 -128
  40. package/k6-tests/utils/common.ts +0 -489
  41. package/k6-tests/utils/logger.ts +0 -62
  42. package/k6-tests/utils/types.d.ts +0 -35
  43. package/malloy_mcp.dxt +0 -0
  44. package/publisher.config.example.bigquery.json +0 -33
  45. package/publisher.config.example.duckdb.json +0 -23
  46. package/publisher.config.json +0 -23
  47. package/scripts/bake-duckdb-extensions.js +0 -110
  48. package/src/config.spec.ts +0 -1535
  49. package/src/config.theme.spec.ts +0 -178
  50. package/src/config.ts +0 -1142
  51. package/src/constants.ts +0 -105
  52. package/src/controller/compile.controller.ts +0 -40
  53. package/src/controller/connection.controller.spec.ts +0 -849
  54. package/src/controller/connection.controller.ts +0 -901
  55. package/src/controller/database.controller.ts +0 -24
  56. package/src/controller/materialization.controller.spec.ts +0 -293
  57. package/src/controller/materialization.controller.ts +0 -319
  58. package/src/controller/model.controller.ts +0 -164
  59. package/src/controller/package.controller.spec.ts +0 -273
  60. package/src/controller/package.controller.ts +0 -306
  61. package/src/controller/query.controller.ts +0 -152
  62. package/src/controller/theme.controller.ts +0 -83
  63. package/src/controller/watch-mode.controller.ts +0 -243
  64. package/src/data_styles.ts +0 -71
  65. package/src/default-publisher.config.json +0 -23
  66. package/src/ducklake_version.spec.ts +0 -163
  67. package/src/ducklake_version.ts +0 -153
  68. package/src/errors.spec.ts +0 -119
  69. package/src/errors.ts +0 -237
  70. package/src/filter_deprecation.spec.ts +0 -64
  71. package/src/filter_deprecation.ts +0 -42
  72. package/src/health.spec.ts +0 -123
  73. package/src/health.ts +0 -233
  74. package/src/heap_check.spec.ts +0 -144
  75. package/src/heap_check.ts +0 -144
  76. package/src/instrumentation.ts +0 -214
  77. package/src/json_utils.spec.ts +0 -51
  78. package/src/json_utils.ts +0 -33
  79. package/src/logger.spec.ts +0 -210
  80. package/src/logger.ts +0 -251
  81. package/src/malloy_pin_prereqs.spec.ts +0 -25
  82. package/src/materialization_metrics.spec.ts +0 -169
  83. package/src/materialization_metrics.ts +0 -346
  84. package/src/mcp/error_messages.spec.ts +0 -35
  85. package/src/mcp/error_messages.ts +0 -183
  86. package/src/mcp/handler_utils.spec.ts +0 -108
  87. package/src/mcp/handler_utils.ts +0 -251
  88. package/src/mcp/mcp_constants.ts +0 -6
  89. package/src/mcp/query_envelope.spec.ts +0 -229
  90. package/src/mcp/query_envelope.ts +0 -240
  91. package/src/mcp/server.protocol.spec.ts +0 -250
  92. package/src/mcp/server.ts +0 -87
  93. package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
  94. package/src/mcp/skills/build_skills_bundle.ts +0 -176
  95. package/src/mcp/skills/skills_bundle.json +0 -1
  96. package/src/mcp/skills/skills_bundle.spec.ts +0 -171
  97. package/src/mcp/tool_response.spec.ts +0 -108
  98. package/src/mcp/tool_response.ts +0 -138
  99. package/src/mcp/tools/compile_tool.spec.ts +0 -315
  100. package/src/mcp/tools/compile_tool.ts +0 -208
  101. package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
  102. package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
  103. package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
  104. package/src/mcp/tools/docs_search_tool.ts +0 -128
  105. package/src/mcp/tools/embedding_index.spec.ts +0 -1236
  106. package/src/mcp/tools/embedding_index.ts +0 -808
  107. package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
  108. package/src/mcp/tools/execute_query_tool.ts +0 -316
  109. package/src/mcp/tools/get_context_eval.ts +0 -275
  110. package/src/mcp/tools/get_context_tool.spec.ts +0 -592
  111. package/src/mcp/tools/get_context_tool.ts +0 -585
  112. package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
  113. package/src/mcp/tools/reload_package_tool.ts +0 -132
  114. package/src/mcp_config.spec.ts +0 -919
  115. package/src/mcp_config.ts +0 -425
  116. package/src/oom_guards.integration.spec.ts +0 -269
  117. package/src/package_load/package_load_pool.spec.ts +0 -252
  118. package/src/package_load/package_load_pool.ts +0 -931
  119. package/src/package_load/package_load_worker.ts +0 -1034
  120. package/src/package_load/protocol.ts +0 -364
  121. package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
  122. package/src/package_load/rpc_wait_accountant.ts +0 -76
  123. package/src/package_load_metrics.spec.ts +0 -114
  124. package/src/package_load_metrics.ts +0 -127
  125. package/src/path_safety.spec.ts +0 -158
  126. package/src/path_safety.ts +0 -146
  127. package/src/pg_helpers.spec.ts +0 -223
  128. package/src/pg_helpers.ts +0 -52
  129. package/src/query_cap_metrics.spec.ts +0 -89
  130. package/src/query_cap_metrics.ts +0 -116
  131. package/src/query_concurrency.spec.ts +0 -247
  132. package/src/query_concurrency.ts +0 -237
  133. package/src/query_metadata_metrics.ts +0 -49
  134. package/src/query_param_utils.ts +0 -29
  135. package/src/query_timeout.spec.ts +0 -224
  136. package/src/query_timeout.ts +0 -179
  137. package/src/runtime/publisher.js +0 -328
  138. package/src/server-old.ts +0 -988
  139. package/src/server.ts +0 -2059
  140. package/src/service/annotations.spec.ts +0 -118
  141. package/src/service/annotations.ts +0 -91
  142. package/src/service/authorize.spec.ts +0 -154
  143. package/src/service/authorize.ts +0 -498
  144. package/src/service/authorize_integration.spec.ts +0 -2041
  145. package/src/service/build_plan.spec.ts +0 -685
  146. package/src/service/build_plan.ts +0 -797
  147. package/src/service/compile_authorize.spec.ts +0 -85
  148. package/src/service/compile_fragment_techniques.spec.ts +0 -156
  149. package/src/service/connection.spec.ts +0 -2360
  150. package/src/service/connection.ts +0 -2187
  151. package/src/service/connection_config.spec.ts +0 -734
  152. package/src/service/connection_config.ts +0 -899
  153. package/src/service/connection_federation.spec.ts +0 -184
  154. package/src/service/connection_fingerprint.spec.ts +0 -102
  155. package/src/service/connection_service.spec.ts +0 -711
  156. package/src/service/connection_service.ts +0 -291
  157. package/src/service/cron_evaluator.spec.ts +0 -79
  158. package/src/service/cron_evaluator.ts +0 -105
  159. package/src/service/db_utils.spec.ts +0 -1085
  160. package/src/service/db_utils.ts +0 -1502
  161. package/src/service/duckdb_instance_isolation.spec.ts +0 -137
  162. package/src/service/ducklake_lazy_attach.spec.ts +0 -110
  163. package/src/service/embedding_provider.spec.ts +0 -329
  164. package/src/service/embedding_provider.ts +0 -236
  165. package/src/service/environment.ts +0 -2191
  166. package/src/service/environment_admission.spec.ts +0 -344
  167. package/src/service/environment_compile.spec.ts +0 -197
  168. package/src/service/environment_store.spec.ts +0 -2274
  169. package/src/service/environment_store.ts +0 -2434
  170. package/src/service/environment_store_anchoring.spec.ts +0 -107
  171. package/src/service/environment_store_clone.spec.ts +0 -350
  172. package/src/service/explore_visibility.spec.ts +0 -434
  173. package/src/service/exports_probe.spec.ts +0 -107
  174. package/src/service/extension_fetch_policy.spec.ts +0 -256
  175. package/src/service/filter.spec.ts +0 -458
  176. package/src/service/filter.ts +0 -341
  177. package/src/service/filter_bypass.spec.ts +0 -418
  178. package/src/service/filter_integration.spec.ts +0 -1004
  179. package/src/service/freshness.spec.ts +0 -183
  180. package/src/service/freshness.ts +0 -112
  181. package/src/service/gcs_s3_utils.ts +0 -427
  182. package/src/service/given.ts +0 -105
  183. package/src/service/givens_integration.spec.ts +0 -219
  184. package/src/service/manifest_loader.spec.ts +0 -189
  185. package/src/service/manifest_loader.ts +0 -156
  186. package/src/service/materialization_build_session.spec.ts +0 -435
  187. package/src/service/materialization_build_session.ts +0 -681
  188. package/src/service/materialization_config_validation.spec.ts +0 -99
  189. package/src/service/materialization_config_validation.ts +0 -120
  190. package/src/service/materialization_eligibility.spec.ts +0 -158
  191. package/src/service/materialization_eligibility.ts +0 -305
  192. package/src/service/materialization_schedule_surface.spec.ts +0 -256
  193. package/src/service/materialization_scheduler.spec.ts +0 -289
  194. package/src/service/materialization_scheduler.ts +0 -304
  195. package/src/service/materialization_serve_transform.spec.ts +0 -1003
  196. package/src/service/materialization_serve_transform.ts +0 -779
  197. package/src/service/materialization_service.spec.ts +0 -2373
  198. package/src/service/materialization_service.ts +0 -2342
  199. package/src/service/materialization_test_fixtures.ts +0 -249
  200. package/src/service/model.spec.ts +0 -1157
  201. package/src/service/model.ts +0 -3370
  202. package/src/service/model_limits.spec.ts +0 -209
  203. package/src/service/model_limits.ts +0 -131
  204. package/src/service/model_storage_serve.spec.ts +0 -193
  205. package/src/service/model_storage_serve_joins.spec.ts +0 -193
  206. package/src/service/package.spec.ts +0 -612
  207. package/src/service/package.ts +0 -1799
  208. package/src/service/package_manifest.spec.ts +0 -230
  209. package/src/service/package_manifest.ts +0 -241
  210. package/src/service/package_memory_governor.spec.ts +0 -173
  211. package/src/service/package_memory_governor.ts +0 -233
  212. package/src/service/package_quote_bound_tables.spec.ts +0 -171
  213. package/src/service/package_race.spec.ts +0 -208
  214. package/src/service/package_reload_safety.spec.ts +0 -193
  215. package/src/service/package_rollback.spec.ts +0 -190
  216. package/src/service/package_worker_path.spec.ts +0 -645
  217. package/src/service/path_injection.spec.ts +0 -39
  218. package/src/service/persist_annotation_validation.spec.ts +0 -89
  219. package/src/service/persist_annotation_validation.ts +0 -59
  220. package/src/service/persistence_policy.spec.ts +0 -374
  221. package/src/service/proxy.spec.ts +0 -414
  222. package/src/service/proxy.ts +0 -248
  223. package/src/service/query_boundary.spec.ts +0 -470
  224. package/src/service/query_metadata.spec.ts +0 -408
  225. package/src/service/query_metadata.ts +0 -492
  226. package/src/service/query_metadata_identity.spec.ts +0 -149
  227. package/src/service/query_text.spec.ts +0 -79
  228. package/src/service/query_text.ts +0 -43
  229. package/src/service/quoting.spec.ts +0 -135
  230. package/src/service/quoting.ts +0 -78
  231. package/src/service/resolve_environment.ts +0 -15
  232. package/src/service/restricted_mode.spec.ts +0 -299
  233. package/src/service/source_extraction.ts +0 -225
  234. package/src/service/theme_store.ts +0 -199
  235. package/src/storage/DatabaseInterface.ts +0 -198
  236. package/src/storage/StorageManager.ts +0 -113
  237. package/src/storage/duckdb/ConnectionRepository.ts +0 -143
  238. package/src/storage/duckdb/DuckDBConnection.ts +0 -123
  239. package/src/storage/duckdb/DuckDBRepository.ts +0 -208
  240. package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
  241. package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
  242. package/src/storage/duckdb/MaterializationRepository.ts +0 -336
  243. package/src/storage/duckdb/PackageRepository.ts +0 -135
  244. package/src/storage/duckdb/schema.ts +0 -229
  245. package/src/stream_helpers.spec.ts +0 -280
  246. package/src/stream_helpers.ts +0 -162
  247. package/src/telemetry.ts +0 -20
  248. package/src/test_helpers/metrics_harness.ts +0 -166
  249. package/src/theme_key_parity.spec.ts +0 -57
  250. package/src/utils.ts +0 -29
  251. package/tests/fixtures/authorize-compile/model.malloy +0 -9
  252. package/tests/fixtures/authorize-compile/publisher.json +0 -4
  253. package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
  254. package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
  255. package/tests/fixtures/html-pages-test/data.csv +0 -3
  256. package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
  257. package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
  258. package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
  259. package/tests/fixtures/html-pages-test/public/data.json +0 -1
  260. package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
  261. package/tests/fixtures/html-pages-test/public/index.html +0 -9
  262. package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
  263. package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
  264. package/tests/fixtures/html-pages-test/public/slides.html +0 -12
  265. package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
  266. package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
  267. package/tests/fixtures/html-pages-test/publisher.json +0 -5
  268. package/tests/fixtures/html-pages-test/report.malloy +0 -1
  269. package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
  270. package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
  271. package/tests/fixtures/persist-multi-level/publisher.json +0 -5
  272. package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
  273. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
  274. package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
  275. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
  276. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
  277. package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
  278. package/tests/fixtures/persist-test/data/orders.csv +0 -5
  279. package/tests/fixtures/persist-test/persist_test.malloy +0 -11
  280. package/tests/fixtures/persist-test/publisher.json +0 -5
  281. package/tests/fixtures/publisher.config.json +0 -15
  282. package/tests/fixtures/query-givens/data/orders.csv +0 -7
  283. package/tests/fixtures/query-givens/model.malloy +0 -34
  284. package/tests/fixtures/query-givens/publisher.json +0 -5
  285. package/tests/fixtures/xlsx/database.xlsx +0 -0
  286. package/tests/harness/mcp_test_setup.ts +0 -332
  287. package/tests/harness/rest_e2e.ts +0 -68
  288. package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
  289. package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
  290. package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
  291. package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
  292. package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
  293. package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
  294. package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
  295. package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
  296. package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
  297. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
  298. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
  299. package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
  300. package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
  301. package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
  302. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
  303. package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
  304. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
  305. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
  306. package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
  307. package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
  308. package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
  309. package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
  310. package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
  311. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
  312. package/tests/unit/controller/theme.controller.spec.ts +0 -141
  313. package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
  314. package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
  315. package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
  316. package/tests/unit/duckdb/repositories.test.ts +0 -208
  317. package/tests/unit/ducklake/ducklake.test.ts +0 -952
  318. package/tests/unit/service/theme_store.spec.ts +0 -139
  319. package/tsconfig.json +0 -32
@@ -1,492 +0,0 @@
1
- /**
2
- * Per-query metadata: the publisher's side of Malloy's
3
- * `RunSQLOptions.queryMetadata` — a flat bag of string properties each connector
4
- * attaches to the statement it issues (Snowflake per-statement `QUERY_TAG`,
5
- * BigQuery per-job labels, a leading SQL comment on the rest). It describes the
6
- * query for the warehouse's own reporting — cost attribution, workload
7
- * classification, tracing — and never affects results or data identity.
8
- *
9
- * Two jobs live here:
10
- *
11
- * 1. **Resolve.** One bag per query, merged least-specific-first: connection
12
- * default → model-side resolved value → request override → server context.
13
- * Context is applied last because it describes what the server is actually
14
- * doing (which package, which class of work) and a caller must not be able
15
- * to overwrite that attribution.
16
- *
17
- * 2. **Keep the bag legal.** Malloy validates at dispatch and THROWS on a bag
18
- * that violates its contract, so an assembled bag that is one character too
19
- * long would fail a customer query. {@link mergeQueryMetadata} therefore
20
- * never throws: it sanitizes, truncates and drops, metering every drop.
21
- * Declaration boundaries — a publish, a connection update, an API request —
22
- * use {@link queryMetadataViolations} instead and fail fast with a message.
23
- * Config LOAD is the exception: a connection default that violates the
24
- * contract warns, because a tag must never be the reason an environment
25
- * refuses to come up.
26
- */
27
-
28
- import type { QueryMetadata } from "@malloydata/malloy";
29
- import { getQueryMetadataMode } from "../config";
30
- import {
31
- recordQueryMetadataApplied,
32
- recordQueryMetadataDropped,
33
- } from "../query_metadata_metrics";
34
-
35
- export type { QueryMetadata };
36
-
37
- /**
38
- * What class of work issued the query. Attribution's primary axis: a warehouse
39
- * bill answers "how much of this is interactive traffic vs builds vs indexing"
40
- * only if every statement says which it is.
41
- */
42
- export const QUERY_CLASSES = [
43
- "interactive",
44
- "materialize",
45
- "index",
46
- "ops",
47
- ] as const;
48
- export type QueryClass = (typeof QUERY_CLASSES)[number];
49
-
50
- /**
51
- * Malloy's contract for a bag, mirrored (`@malloydata/malloy` exports the
52
- * `QueryMetadata` type but not its validator):
53
- *
54
- * - property names: ASCII alphanumerics and underscore, <=128 chars
55
- * - property values: printable ASCII except `"`, <=256 chars
56
- * - at most 20 properties
57
- *
58
- * Kept in one place with the upstream pin below so a future core change is a
59
- * one-line update here rather than a scattered hunt. A bag that satisfies this
60
- * is renderable by every connector, including into the `-- NAME="value"` comment
61
- * form.
62
- *
63
- * Pinned to @malloydata/malloy 0.0.426 (`packages/malloy/src/query_metadata.ts`).
64
- */
65
- export const MAX_PROPERTY_NAME_LENGTH = 128;
66
- export const MAX_PROPERTY_VALUE_LENGTH = 256;
67
- export const MAX_PROPERTIES = 20;
68
- const PROPERTY_NAME_RE = /^[A-Za-z0-9_]+$/;
69
- /**
70
- * Snowflake's `QUERY_TAG` holds the whole bag as one JSON string capped at 2000
71
- * chars, and db-snowflake clamps by slicing — which truncates mid-JSON and
72
- * leaves an unparseable tag, i.e. silently unqueryable in `QUERY_HISTORY`,
73
- * which is the entire point of tagging. So the budget is enforced here, on the
74
- * serialized size, by dropping whole properties.
75
- */
76
- const MAX_SERIALIZED_LENGTH = 2000;
77
-
78
- /** Printable ASCII (0x20-0x7e) except `"`, which would break the comment form. */
79
- function isRenderableValue(value: string): boolean {
80
- for (let i = 0; i < value.length; i++) {
81
- const code = value.charCodeAt(i);
82
- if (code < 0x20 || code > 0x7e || code === 0x22) return false;
83
- }
84
- return true;
85
- }
86
-
87
- /**
88
- * The ways `raw` violates the contract, empty when it conforms. For boundaries
89
- * where a human is present: a publish, a connection update, an API request. The
90
- * runtime path clamps instead (see {@link mergeQueryMetadata}).
91
- */
92
- export function queryMetadataViolations(raw: unknown): string[] {
93
- if (raw === undefined || raw === null) return [];
94
- if (typeof raw !== "object" || Array.isArray(raw)) {
95
- return ["queryMetadata must be an object of string properties"];
96
- }
97
- const problems: string[] = [];
98
- const entries = Object.entries(raw as Record<string, unknown>);
99
- if (entries.length > MAX_PROPERTIES) {
100
- problems.push(
101
- `queryMetadata declares ${entries.length} properties; at most ${MAX_PROPERTIES} are allowed`,
102
- );
103
- }
104
- for (const [name, value] of entries) {
105
- if (!PROPERTY_NAME_RE.test(name)) {
106
- problems.push(
107
- `queryMetadata property name '${name}' must be ASCII alphanumerics and underscore`,
108
- );
109
- } else if (name.length > MAX_PROPERTY_NAME_LENGTH) {
110
- problems.push(
111
- `queryMetadata property name '${name}' exceeds ${MAX_PROPERTY_NAME_LENGTH} characters`,
112
- );
113
- }
114
- if (typeof value !== "string") {
115
- problems.push(
116
- `queryMetadata property '${name}' must be a string (got ${typeof value})`,
117
- );
118
- continue;
119
- }
120
- if (!isRenderableValue(value)) {
121
- problems.push(
122
- `queryMetadata property '${name}' has a value outside printable ASCII, or containing '"'`,
123
- );
124
- }
125
- if (value.length > MAX_PROPERTY_VALUE_LENGTH) {
126
- problems.push(
127
- `queryMetadata property '${name}' value exceeds ${MAX_PROPERTY_VALUE_LENGTH} characters`,
128
- );
129
- }
130
- }
131
- return problems;
132
- }
133
-
134
- /**
135
- * Advisory notes about a bag that satisfies the contract but still will not do
136
- * what it says, so it is reported at declaration rather than discovered in a
137
- * warehouse console. Two kinds today:
138
- *
139
- * - **A backend quietly discards it.** BigQuery's label grammar needs a name
140
- * that starts with a lowercase letter after transformation; db-bigquery drops
141
- * one it cannot make valid while every other connector keeps it. Same
142
- * declaration, present on Snowflake, absent on BigQuery.
143
- * - **The server will not have room for it.** The contract's cap covers the
144
- * whole bag, and the server's context is added on top of what was declared.
145
- */
146
- export function queryMetadataAdvisoryWarnings(meta: QueryMetadata): string[] {
147
- const warnings: string[] = [];
148
- for (const name of Object.keys(meta)) {
149
- if (!/^[A-Za-z]/.test(name)) {
150
- warnings.push(
151
- `queryMetadata property '${name}' does not start with a letter, so BigQuery drops it (other backends keep it); rename it to start with a letter`,
152
- );
153
- }
154
- }
155
- return warnings;
156
- }
157
-
158
- /**
159
- * The budget warning for a declaration of `declared` properties, or undefined
160
- * when it fits.
161
- *
162
- * Separate from {@link queryMetadataAdvisoryWarnings} because the count that
163
- * matters is not always one bag's: a connection declares a default AND an
164
- * enforced map, and they share the budget, so summing them is the caller's job.
165
- * The contract's 20 covers the whole bag, and the server's own context lands on
166
- * top of everything declared — so a declaration that fills the contract publishes
167
- * clean and then loses properties on every statement, visible only as a metric.
168
- */
169
- export function queryMetadataBudgetWarning(
170
- declared: number,
171
- ): string | undefined {
172
- const authorBudget = MAX_PROPERTIES - RESERVED_CONTEXT_PROPERTIES;
173
- if (declared <= authorBudget) return undefined;
174
- return (
175
- `queryMetadata declares ${declared} properties; the server adds up to ` +
176
- `${RESERVED_CONTEXT_PROPERTIES} of its own to every statement, so a bag over ` +
177
- `${authorBudget} loses its least specific properties at query time`
178
- );
179
- }
180
-
181
- /**
182
- * The server's own description of the query, injected on every statement so
183
- * attribution needs no modeling work from the author.
184
- *
185
- * Neutral by design: a platform that wants its own dimensions (tenant,
186
- * deployment, cost centre) supplies them as a connection default or a
187
- * per-request override, and the publisher does not invent them.
188
- *
189
- * Every field except the correlation id is a property of the unit of work rather
190
- * than of the individual call, so a repeated query produces the same values. The
191
- * correlation id is per call by definition — it is the join key a response hands
192
- * back — which on a backend with no native tag mechanism makes the statement text
193
- * unique and so bypasses an exact-text result cache. That is the deliberate cost
194
- * of being able to find one query again.
195
- */
196
- export interface QueryContext {
197
- /** What issued the query. Absent only where the class is genuinely unknown. */
198
- queryClass?: QueryClass;
199
- environment?: string;
200
- package?: string;
201
- /**
202
- * Published version id, when the query runs against a versioned package.
203
- *
204
- * Reserved, not yet reachable: the only path that supplies it is a query
205
- * request's `versionId`, which the route rejects with 501 until versioned
206
- * packages land. It keeps its slot in {@link CONTEXT_SHED_ORDER} so that
207
- * wiring it later is a one-line change rather than a shed-order revision.
208
- */
209
- version?: string;
210
- /** Package-relative model path, for query paths. */
211
- model?: string;
212
- /** Persist source or index dimension, for build paths. */
213
- source?: string;
214
- /** What started a build: `publish`, `on_demand`, `scheduler`. */
215
- trigger?: string;
216
- /**
217
- * Groups every statement of one build. The publisher mints it (the
218
- * materialization id) unless the caller supplies its own.
219
- */
220
- runId?: string;
221
- /**
222
- * Identifies this single query, minted by the request boundary that returns
223
- * it to the caller (see {@link mintCorrelationId}). Set only where there is a
224
- * response field to carry it: an id nobody can read would cost the result
225
- * cache and buy nothing.
226
- */
227
- correlationId?: string;
228
- }
229
-
230
- /**
231
- * A fresh correlation id for one query. Minted by the publisher rather than
232
- * accepted from the caller because it is the platform's join key: a caller-owned
233
- * value can be omitted, reused across calls, or collide with another caller's,
234
- * and then a response has nothing meaningful to hand back. A caller that wants
235
- * its own request id in the bag declares it under its own property name.
236
- */
237
- export function mintCorrelationId(): string {
238
- return crypto.randomUUID();
239
- }
240
-
241
- /**
242
- * The order context properties are given up in when even the context alone will
243
- * not fit: least load-bearing first. `class` and `query_id` are last because
244
- * they are the two that make the rest worth reading — the workload split, and
245
- * the id a response handed a caller as its join key.
246
- */
247
- export const CONTEXT_SHED_ORDER = [
248
- "version",
249
- "model",
250
- "source",
251
- "trigger",
252
- "environment",
253
- "package",
254
- "run_id",
255
- "query_id",
256
- "class",
257
- ];
258
-
259
- /**
260
- * How many properties the context can occupy, so a declaration boundary can warn
261
- * about an author budget that will not survive the merge.
262
- */
263
- export const RESERVED_CONTEXT_PROPERTIES = CONTEXT_SHED_ORDER.length;
264
-
265
- /** The context as bag properties, dropping every absent field. */
266
- export function queryContextProperties(context: QueryContext): QueryMetadata {
267
- const out: QueryMetadata = {};
268
- const put = (name: string, value: string | undefined) => {
269
- if (value !== undefined && value !== "") out[name] = value;
270
- };
271
- put("class", context.queryClass);
272
- put("environment", context.environment);
273
- put("package", context.package);
274
- put("version", context.version);
275
- put("model", context.model);
276
- put("source", context.source);
277
- put("trigger", context.trigger);
278
- put("run_id", context.runId);
279
- put("query_id", context.correlationId);
280
- return out;
281
- }
282
-
283
- /** One layer of a resolved bag, least specific first. */
284
- export interface QueryMetadataLayers {
285
- /** The executing connection's default (least specific). */
286
- connection?: QueryMetadata | null;
287
- /** The value the build plan resolved for this materialization unit. */
288
- model?: QueryMetadata | null;
289
- /** The caller's per-request override. */
290
- request?: QueryMetadata | null;
291
- /**
292
- * Properties the connection declares as the deployment's own, which no
293
- * declaration can overwrite and which outlive every author property when the
294
- * bag has to shrink.
295
- *
296
- * A host that runs one Publisher for several tenants needs this layer to win
297
- * for the properties it is billed by — otherwise the tenant label is both
298
- * forgeable (a `#@ persist queryMetadata.tenant=…` outranks a connection
299
- * default) and the first thing shed under budget, which is a poor showing for
300
- * the property finance reads.
301
- *
302
- * The trust split it rests on is the DEPLOYMENT's, not this module's: nothing
303
- * here authenticates a connection write, so it separates operator from author
304
- * only where something in front of Publisher does.
305
- */
306
- enforced?: QueryMetadata | null;
307
- /** The server's own context, which wins over every other layer. */
308
- context?: QueryContext;
309
- }
310
-
311
- /** How a property was lost, for the drop metric and the caller's log. */
312
- export type QueryMetadataDropReason =
313
- | "invalid_name"
314
- | "invalid_value"
315
- | "property_cap"
316
- | "serialized_cap";
317
-
318
- export interface ResolvedQueryMetadata {
319
- /** The bag to hand to Malloy; undefined when there is nothing to attach. */
320
- metadata?: QueryMetadata;
321
- /** Properties that did not survive, most useful in a warning log. */
322
- drops: { name: string; reason: QueryMetadataDropReason }[];
323
- }
324
-
325
- /** Truncate a value to the contract and strip anything unrenderable. */
326
- function sanitizeValue(value: string): string {
327
- let out = "";
328
- for (
329
- let i = 0;
330
- i < value.length && out.length < MAX_PROPERTY_VALUE_LENGTH;
331
- i++
332
- ) {
333
- const code = value.charCodeAt(i);
334
- out += code < 0x20 || code > 0x7e || code === 0x22 ? "_" : value[i];
335
- }
336
- return out;
337
- }
338
-
339
- /**
340
- * Merge the layers into one bag Malloy will accept, and report what was lost.
341
- *
342
- * Never throws — metadata must not be the reason a query or a build fails —
343
- * with one precondition: `getQueryMetadataMode()` rejects an unrecognized
344
- * `PUBLISHER_QUERY_METADATA`, and it is the boot probe in `server.ts` that
345
- * turns that into a startup failure rather than a per-query one. A host that
346
- * embeds this module without that probe owes itself the same check.
347
- *
348
- * Precedence is most-specific-wins per property across the author layers
349
- * (connection < model < request), then the connection's ENFORCED properties,
350
- * then the server's context — so neither a declaration nor a caller can
351
- * overwrite what the deployment or the server says about the query.
352
- *
353
- * Shedding runs in the same order, from the bottom: the 20-property cap and
354
- * Snowflake's 2000-char serialized tag take author properties first (losing a
355
- * caller's `team` label degrades attribution), then enforced properties, and
356
- * only then context (losing `class` or `query_id` breaks it).
357
- *
358
- * Returns no metadata at all when `PUBLISHER_QUERY_METADATA=off`, which is the
359
- * escape hatch for a deployment that does not want the publisher touching the
360
- * statements it sends.
361
- */
362
- export function mergeQueryMetadata(
363
- layers: QueryMetadataLayers,
364
- ): ResolvedQueryMetadata {
365
- if (getQueryMetadataMode() === "off") return { drops: [] };
366
-
367
- const drops: { name: string; reason: QueryMetadataDropReason }[] = [];
368
- const contextProperties = queryContextProperties(layers.context ?? {});
369
-
370
- // Layers in precedence order, least specific first, then context on top. Each
371
- // property remembers the layer whose value SURVIVED, not the first layer that
372
- // mentioned it, so the shed order below matches the precedence that produced
373
- // the bag.
374
- // Null-prototype: `__proto__` satisfies the contract's name rule, so it
375
- // reaches an assignment that an object literal would route to the prototype
376
- // setter — which ignores a string and drops the property with no record of
377
- // it. Every other invalid name is dropped loudly; this one would not be.
378
- const merged: QueryMetadata = Object.create(null) as QueryMetadata;
379
- const winningLayer = new Map<string, number>();
380
- const orderedLayers = [
381
- layers.connection,
382
- layers.model,
383
- layers.request,
384
- layers.enforced,
385
- ];
386
- orderedLayers.forEach((layer, index) => {
387
- for (const [name, value] of Object.entries(layer ?? {})) {
388
- if (
389
- !PROPERTY_NAME_RE.test(name) ||
390
- name.length > MAX_PROPERTY_NAME_LENGTH
391
- ) {
392
- drops.push({ name, reason: "invalid_name" });
393
- continue;
394
- }
395
- if (typeof value !== "string") {
396
- drops.push({ name, reason: "invalid_value" });
397
- continue;
398
- }
399
- winningLayer.set(name, index);
400
- merged[name] = sanitizeValue(value);
401
- }
402
- });
403
- for (const [name, value] of Object.entries(contextProperties)) {
404
- merged[name] = sanitizeValue(value);
405
- }
406
-
407
- // Shedding runs least-specific-first, the same order that decides which value
408
- // wins: a connection-wide default is the cheapest thing to lose, the caller's
409
- // own per-request property — most likely its join key into whatever it is
410
- // correlating — outlives it, and an enforced property outlives them all,
411
- // because the deployment is billed by it.
412
- const shedOrder = [...winningLayer.entries()]
413
- // hasOwnProperty, not `in`: `__proto__` is a contract-legal property name
414
- // and `in` finds it on every object literal's prototype, which would take
415
- // a declared one off the shed list and make it unsheddable.
416
- .filter(
417
- ([name]) =>
418
- !Object.prototype.hasOwnProperty.call(contextProperties, name),
419
- )
420
- .sort((a, b) => a[1] - b[1])
421
- .map(([name]) => name);
422
- const shed = (reason: QueryMetadataDropReason) => {
423
- const name = shedOrder.shift() as string;
424
- delete merged[name];
425
- drops.push({ name, reason });
426
- };
427
- while (Object.keys(merged).length > MAX_PROPERTIES && shedOrder.length > 0) {
428
- shed("property_cap");
429
- }
430
- while (
431
- JSON.stringify(merged).length > MAX_SERIALIZED_LENGTH &&
432
- shedOrder.length > 0
433
- ) {
434
- shed("serialized_cap");
435
- }
436
- // Context alone over either budget is a server bug, not a caller's — but a
437
- // thrown query would be worse than a partial tag, so shed context too. Least
438
- // load-bearing first, ending at the two that make the bag worth reading: the
439
- // workload class, and the id a response handed back as a join key.
440
- const contextShedOrder = CONTEXT_SHED_ORDER.filter(
441
- (name) => name in contextProperties,
442
- );
443
- while (
444
- (Object.keys(merged).length > MAX_PROPERTIES ||
445
- JSON.stringify(merged).length > MAX_SERIALIZED_LENGTH) &&
446
- contextShedOrder.length > 0
447
- ) {
448
- const name = contextShedOrder.shift() as string;
449
- const overCount = Object.keys(merged).length > MAX_PROPERTIES;
450
- delete merged[name];
451
- drops.push({
452
- name,
453
- reason: overCount ? "property_cap" : "serialized_cap",
454
- });
455
- }
456
-
457
- for (const drop of drops) recordQueryMetadataDropped(drop.reason);
458
- if (Object.keys(merged).length === 0) return { drops };
459
- recordQueryMetadataApplied(layers.context?.queryClass ?? "unknown");
460
- return { metadata: merged, drops };
461
- }
462
-
463
- /**
464
- * Read a caller-supplied bag, rejecting rather than clamping: a request or a
465
- * connection update has a human behind it, so a bad property should come back
466
- * as a message instead of silently not doing what it says.
467
- *
468
- * @throws {Error} listing every violation, for the caller to map to a 400.
469
- */
470
- export function parseSuppliedQueryMetadata(
471
- raw: unknown,
472
- ): QueryMetadata | undefined {
473
- if (raw === undefined || raw === null) return undefined;
474
- const problems = queryMetadataViolations(raw);
475
- if (problems.length > 0) throw new Error(problems.join("; "));
476
- const meta = raw as QueryMetadata;
477
- return Object.keys(meta).length > 0 ? meta : undefined;
478
- }
479
-
480
- /** Read a `queryClass`, rejecting an unrecognized value. @throws {Error} */
481
- export function parseQueryClass(raw: unknown): QueryClass | undefined {
482
- if (raw === undefined || raw === null) return undefined;
483
- if (
484
- typeof raw === "string" &&
485
- (QUERY_CLASSES as readonly string[]).includes(raw)
486
- ) {
487
- return raw as QueryClass;
488
- }
489
- throw new Error(
490
- `queryClass must be one of ${QUERY_CLASSES.join(" | ")} (got ${JSON.stringify(raw)})`,
491
- );
492
- }
@@ -1,149 +0,0 @@
1
- import { afterEach, beforeEach, describe, expect, it } from "bun:test";
2
- import * as fs from "fs/promises";
3
- import * as os from "os";
4
- import * as path from "path";
5
-
6
- import { Environment } from "./environment";
7
-
8
- /**
9
- * Query metadata is observability, and it must never be identity.
10
- *
11
- * `sourceEntityId` is a content address: `makeBuildId(connectionDigest,
12
- * getSQL())`. If a tag ever reached either input, declaring one would
13
- * re-address every persist source in the package — the manifest lookup would
14
- * miss, every source would rebuild into a new physical table, and the tables
15
- * built before the change would be orphaned at rest. Silent, expensive, and
16
- * discovered from a warehouse bill.
17
- *
18
- * It holds by construction today: `queryMetadata` exists only on Malloy's
19
- * `RunSQLOptions` (the dispatch path), never on the compile options `getSQL`
20
- * takes, and every connector's `getDigest()` enumerates connection-shape fields.
21
- * Nothing pins it, which is what this file is for — the invariant is cheap to
22
- * assert and its failure mode is the worst on this surface.
23
- *
24
- * Deliberately end-to-end over a real `Environment`/`Package` rather than the
25
- * fake source fixture: `fakeSource.makeBuildId` returns a constant, so a
26
- * fixture-based version of this test would pass no matter what leaked. The
27
- * three layers exercised here are the ones that ride in the model text, where a
28
- * leak is actually plausible — the manifest block, the model-file `##`
29
- * envelope, and the source's own `#@ persist`.
30
- */
31
- describe("query metadata is excluded from persist identity", () => {
32
- let rootDir: string;
33
- let envPath: string;
34
-
35
- /** The one persist source both variants declare, byte-identical but for the tags. */
36
- const model = (tagged: boolean) =>
37
- [
38
- "##! experimental.persistence",
39
- ...(tagged
40
- ? ['## materialization.queryMetadata.surface="marts"']
41
- : []),
42
- "",
43
- `source: raw is duckdb.sql("SELECT 1 as id, 'a' as category")`,
44
- "",
45
- tagged
46
- ? '#@ persist name="rollup" queryMetadata.tier="platinum"'
47
- : '#@ persist name="rollup"',
48
- "source: rollup is raw -> {",
49
- " group_by: category",
50
- " aggregate: n is count()",
51
- "}",
52
- "",
53
- ].join("\n");
54
-
55
- const manifest = (tagged: boolean) => ({
56
- name: "pkg",
57
- description: "fixture",
58
- materialization: {
59
- scope: "version",
60
- ...(tagged
61
- ? { queryMetadata: { team: "finance", tier: "bronze" } }
62
- : {}),
63
- },
64
- });
65
-
66
- /**
67
- * Load a fresh package from a fresh environment and return the entity id its
68
- * build plan assigned to `rollup`. A new environment per variant so nothing
69
- * is served from the previous compile.
70
- */
71
- async function entityIdFor(tagged: boolean): Promise<string> {
72
- const dir = path.join(envPath, "pkg");
73
- await fs.mkdir(dir, { recursive: true });
74
- await fs.writeFile(
75
- path.join(dir, "publisher.json"),
76
- JSON.stringify(manifest(tagged)),
77
- );
78
- await fs.writeFile(path.join(dir, "model.malloy"), model(tagged));
79
-
80
- const env = await Environment.create("testEnv", envPath, []);
81
- await env.addPackage("pkg");
82
- const pkg = await env.getPackage("pkg", false);
83
- const plan = pkg.getBuildPlan();
84
- const sources = Object.values(plan?.sources ?? {});
85
- // Guard the guard: an empty plan would make the comparison below vacuous,
86
- // and a `#@ persist` Malloy declines to treat as a build root yields
87
- // exactly that.
88
- expect(sources).toHaveLength(1);
89
- return sources[0].sourceEntityId as string;
90
- }
91
-
92
- beforeEach(async () => {
93
- rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-qm-id-"));
94
- envPath = path.join(rootDir, "env");
95
- await fs.mkdir(envPath, { recursive: true });
96
- });
97
-
98
- afterEach(async () => {
99
- await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
100
- });
101
-
102
- it(
103
- "declaring metadata at every model-side layer does not re-address the source",
104
- async () => {
105
- const bare = await entityIdFor(false);
106
- await fs.rm(path.join(envPath, "pkg"), {
107
- recursive: true,
108
- force: true,
109
- });
110
- const tagged = await entityIdFor(true);
111
-
112
- // Byte-identical, not merely "both defined": this is a content address,
113
- // and a differing id means every table built under the old one is
114
- // orphaned the moment someone adds a tag.
115
- expect(tagged).toBe(bare);
116
- },
117
- { timeout: 30000 },
118
- );
119
-
120
- it(
121
- "the resolved metadata really did reach the plan",
122
- async () => {
123
- // Without this the test above passes just as well when the tags are
124
- // being dropped on the floor — the failure mode that would make it
125
- // look like the invariant holds while nothing is being tagged at all.
126
- const dir = path.join(envPath, "pkg");
127
- await fs.mkdir(dir, { recursive: true });
128
- await fs.writeFile(
129
- path.join(dir, "publisher.json"),
130
- JSON.stringify(manifest(true)),
131
- );
132
- await fs.writeFile(path.join(dir, "model.malloy"), model(true));
133
-
134
- const env = await Environment.create("testEnv", envPath, []);
135
- await env.addPackage("pkg");
136
- const pkg = await env.getPackage("pkg", false);
137
- const source = Object.values(pkg.getBuildPlan()?.sources ?? {})[0];
138
-
139
- expect(source.queryMetadata).toMatchObject({
140
- team: "finance",
141
- surface: "marts",
142
- // Most specific wins: the source's own `#@ persist` over the
143
- // manifest's `tier`.
144
- tier: "platinum",
145
- });
146
- },
147
- { timeout: 30000 },
148
- );
149
- });