@malloy-publisher/server 0.0.235 → 0.0.236

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (301) hide show
  1. package/dist/app/api-doc.yaml +47 -17
  2. package/dist/package_load_worker.mjs +8 -1
  3. package/dist/server.mjs +471 -54
  4. package/package.json +5 -2
  5. package/.eslintrc.json +0 -44
  6. package/.prettierignore +0 -1
  7. package/.prettierrc +0 -4
  8. package/build.ts +0 -97
  9. package/eslint.config.mjs +0 -76
  10. package/k6-tests/README.md +0 -144
  11. package/k6-tests/bun.lock +0 -797
  12. package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
  13. package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
  14. package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
  15. package/k6-tests/load-test/load-test-crud.ts +0 -142
  16. package/k6-tests/load-test/load-test.ts +0 -784
  17. package/k6-tests/package.json +0 -27
  18. package/k6-tests/scripts/run-with-otel.sh +0 -50
  19. package/k6-tests/smoke-test/smoke-test.ts +0 -158
  20. package/k6-tests/tsconfig.json +0 -29
  21. package/k6-tests/utils/client_factory.ts +0 -128
  22. package/k6-tests/utils/common.ts +0 -489
  23. package/k6-tests/utils/logger.ts +0 -62
  24. package/k6-tests/utils/types.d.ts +0 -35
  25. package/malloy_mcp.dxt +0 -0
  26. package/publisher.config.example.bigquery.json +0 -33
  27. package/publisher.config.example.duckdb.json +0 -23
  28. package/publisher.config.json +0 -23
  29. package/scripts/bake-duckdb-extensions.js +0 -110
  30. package/src/config.spec.ts +0 -1535
  31. package/src/config.theme.spec.ts +0 -178
  32. package/src/config.ts +0 -1142
  33. package/src/constants.ts +0 -105
  34. package/src/controller/compile.controller.ts +0 -40
  35. package/src/controller/connection.controller.spec.ts +0 -849
  36. package/src/controller/connection.controller.ts +0 -901
  37. package/src/controller/database.controller.ts +0 -24
  38. package/src/controller/materialization.controller.spec.ts +0 -293
  39. package/src/controller/materialization.controller.ts +0 -319
  40. package/src/controller/model.controller.ts +0 -164
  41. package/src/controller/package.controller.spec.ts +0 -273
  42. package/src/controller/package.controller.ts +0 -306
  43. package/src/controller/query.controller.ts +0 -152
  44. package/src/controller/theme.controller.ts +0 -83
  45. package/src/controller/watch-mode.controller.ts +0 -243
  46. package/src/data_styles.ts +0 -71
  47. package/src/default-publisher.config.json +0 -23
  48. package/src/ducklake_version.spec.ts +0 -163
  49. package/src/ducklake_version.ts +0 -153
  50. package/src/errors.spec.ts +0 -119
  51. package/src/errors.ts +0 -237
  52. package/src/filter_deprecation.spec.ts +0 -64
  53. package/src/filter_deprecation.ts +0 -42
  54. package/src/health.spec.ts +0 -123
  55. package/src/health.ts +0 -233
  56. package/src/heap_check.spec.ts +0 -144
  57. package/src/heap_check.ts +0 -144
  58. package/src/instrumentation.ts +0 -214
  59. package/src/json_utils.spec.ts +0 -51
  60. package/src/json_utils.ts +0 -33
  61. package/src/logger.spec.ts +0 -210
  62. package/src/logger.ts +0 -251
  63. package/src/malloy_pin_prereqs.spec.ts +0 -25
  64. package/src/materialization_metrics.spec.ts +0 -169
  65. package/src/materialization_metrics.ts +0 -346
  66. package/src/mcp/error_messages.spec.ts +0 -35
  67. package/src/mcp/error_messages.ts +0 -183
  68. package/src/mcp/handler_utils.spec.ts +0 -108
  69. package/src/mcp/handler_utils.ts +0 -251
  70. package/src/mcp/mcp_constants.ts +0 -6
  71. package/src/mcp/query_envelope.spec.ts +0 -229
  72. package/src/mcp/query_envelope.ts +0 -240
  73. package/src/mcp/server.protocol.spec.ts +0 -250
  74. package/src/mcp/server.ts +0 -87
  75. package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
  76. package/src/mcp/skills/build_skills_bundle.ts +0 -176
  77. package/src/mcp/skills/skills_bundle.json +0 -1
  78. package/src/mcp/skills/skills_bundle.spec.ts +0 -171
  79. package/src/mcp/tool_response.spec.ts +0 -108
  80. package/src/mcp/tool_response.ts +0 -138
  81. package/src/mcp/tools/compile_tool.spec.ts +0 -315
  82. package/src/mcp/tools/compile_tool.ts +0 -208
  83. package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
  84. package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
  85. package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
  86. package/src/mcp/tools/docs_search_tool.ts +0 -128
  87. package/src/mcp/tools/embedding_index.spec.ts +0 -1236
  88. package/src/mcp/tools/embedding_index.ts +0 -808
  89. package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
  90. package/src/mcp/tools/execute_query_tool.ts +0 -316
  91. package/src/mcp/tools/get_context_eval.ts +0 -275
  92. package/src/mcp/tools/get_context_tool.spec.ts +0 -592
  93. package/src/mcp/tools/get_context_tool.ts +0 -585
  94. package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
  95. package/src/mcp/tools/reload_package_tool.ts +0 -132
  96. package/src/mcp_config.spec.ts +0 -919
  97. package/src/mcp_config.ts +0 -425
  98. package/src/oom_guards.integration.spec.ts +0 -269
  99. package/src/package_load/package_load_pool.spec.ts +0 -252
  100. package/src/package_load/package_load_pool.ts +0 -931
  101. package/src/package_load/package_load_worker.ts +0 -1034
  102. package/src/package_load/protocol.ts +0 -364
  103. package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
  104. package/src/package_load/rpc_wait_accountant.ts +0 -76
  105. package/src/package_load_metrics.spec.ts +0 -114
  106. package/src/package_load_metrics.ts +0 -127
  107. package/src/path_safety.spec.ts +0 -158
  108. package/src/path_safety.ts +0 -146
  109. package/src/pg_helpers.spec.ts +0 -223
  110. package/src/pg_helpers.ts +0 -52
  111. package/src/query_cap_metrics.spec.ts +0 -89
  112. package/src/query_cap_metrics.ts +0 -116
  113. package/src/query_concurrency.spec.ts +0 -247
  114. package/src/query_concurrency.ts +0 -237
  115. package/src/query_metadata_metrics.ts +0 -49
  116. package/src/query_param_utils.ts +0 -29
  117. package/src/query_timeout.spec.ts +0 -224
  118. package/src/query_timeout.ts +0 -179
  119. package/src/runtime/publisher.js +0 -328
  120. package/src/server-old.ts +0 -988
  121. package/src/server.ts +0 -2059
  122. package/src/service/annotations.spec.ts +0 -118
  123. package/src/service/annotations.ts +0 -91
  124. package/src/service/authorize.spec.ts +0 -154
  125. package/src/service/authorize.ts +0 -498
  126. package/src/service/authorize_integration.spec.ts +0 -2041
  127. package/src/service/build_plan.spec.ts +0 -685
  128. package/src/service/build_plan.ts +0 -797
  129. package/src/service/compile_authorize.spec.ts +0 -85
  130. package/src/service/compile_fragment_techniques.spec.ts +0 -156
  131. package/src/service/connection.spec.ts +0 -2360
  132. package/src/service/connection.ts +0 -2187
  133. package/src/service/connection_config.spec.ts +0 -734
  134. package/src/service/connection_config.ts +0 -899
  135. package/src/service/connection_federation.spec.ts +0 -184
  136. package/src/service/connection_fingerprint.spec.ts +0 -102
  137. package/src/service/connection_service.spec.ts +0 -711
  138. package/src/service/connection_service.ts +0 -291
  139. package/src/service/cron_evaluator.spec.ts +0 -79
  140. package/src/service/cron_evaluator.ts +0 -105
  141. package/src/service/db_utils.spec.ts +0 -1085
  142. package/src/service/db_utils.ts +0 -1502
  143. package/src/service/duckdb_instance_isolation.spec.ts +0 -137
  144. package/src/service/ducklake_lazy_attach.spec.ts +0 -110
  145. package/src/service/embedding_provider.spec.ts +0 -329
  146. package/src/service/embedding_provider.ts +0 -236
  147. package/src/service/environment.ts +0 -2191
  148. package/src/service/environment_admission.spec.ts +0 -344
  149. package/src/service/environment_compile.spec.ts +0 -197
  150. package/src/service/environment_store.spec.ts +0 -2274
  151. package/src/service/environment_store.ts +0 -2434
  152. package/src/service/environment_store_anchoring.spec.ts +0 -107
  153. package/src/service/environment_store_clone.spec.ts +0 -350
  154. package/src/service/explore_visibility.spec.ts +0 -434
  155. package/src/service/exports_probe.spec.ts +0 -107
  156. package/src/service/extension_fetch_policy.spec.ts +0 -256
  157. package/src/service/filter.spec.ts +0 -458
  158. package/src/service/filter.ts +0 -341
  159. package/src/service/filter_bypass.spec.ts +0 -418
  160. package/src/service/filter_integration.spec.ts +0 -1004
  161. package/src/service/freshness.spec.ts +0 -183
  162. package/src/service/freshness.ts +0 -112
  163. package/src/service/gcs_s3_utils.ts +0 -427
  164. package/src/service/given.ts +0 -105
  165. package/src/service/givens_integration.spec.ts +0 -219
  166. package/src/service/manifest_loader.spec.ts +0 -189
  167. package/src/service/manifest_loader.ts +0 -156
  168. package/src/service/materialization_build_session.spec.ts +0 -435
  169. package/src/service/materialization_build_session.ts +0 -681
  170. package/src/service/materialization_config_validation.spec.ts +0 -99
  171. package/src/service/materialization_config_validation.ts +0 -120
  172. package/src/service/materialization_eligibility.spec.ts +0 -158
  173. package/src/service/materialization_eligibility.ts +0 -305
  174. package/src/service/materialization_schedule_surface.spec.ts +0 -256
  175. package/src/service/materialization_scheduler.spec.ts +0 -289
  176. package/src/service/materialization_scheduler.ts +0 -304
  177. package/src/service/materialization_serve_transform.spec.ts +0 -1003
  178. package/src/service/materialization_serve_transform.ts +0 -779
  179. package/src/service/materialization_service.spec.ts +0 -2373
  180. package/src/service/materialization_service.ts +0 -2342
  181. package/src/service/materialization_test_fixtures.ts +0 -249
  182. package/src/service/model.spec.ts +0 -1157
  183. package/src/service/model.ts +0 -3370
  184. package/src/service/model_limits.spec.ts +0 -209
  185. package/src/service/model_limits.ts +0 -131
  186. package/src/service/model_storage_serve.spec.ts +0 -193
  187. package/src/service/model_storage_serve_joins.spec.ts +0 -193
  188. package/src/service/package.spec.ts +0 -612
  189. package/src/service/package.ts +0 -1799
  190. package/src/service/package_manifest.spec.ts +0 -230
  191. package/src/service/package_manifest.ts +0 -241
  192. package/src/service/package_memory_governor.spec.ts +0 -173
  193. package/src/service/package_memory_governor.ts +0 -233
  194. package/src/service/package_quote_bound_tables.spec.ts +0 -171
  195. package/src/service/package_race.spec.ts +0 -208
  196. package/src/service/package_reload_safety.spec.ts +0 -193
  197. package/src/service/package_rollback.spec.ts +0 -190
  198. package/src/service/package_worker_path.spec.ts +0 -645
  199. package/src/service/path_injection.spec.ts +0 -39
  200. package/src/service/persist_annotation_validation.spec.ts +0 -89
  201. package/src/service/persist_annotation_validation.ts +0 -59
  202. package/src/service/persistence_policy.spec.ts +0 -374
  203. package/src/service/proxy.spec.ts +0 -414
  204. package/src/service/proxy.ts +0 -248
  205. package/src/service/query_boundary.spec.ts +0 -470
  206. package/src/service/query_metadata.spec.ts +0 -408
  207. package/src/service/query_metadata.ts +0 -492
  208. package/src/service/query_metadata_identity.spec.ts +0 -149
  209. package/src/service/query_text.spec.ts +0 -79
  210. package/src/service/query_text.ts +0 -43
  211. package/src/service/quoting.spec.ts +0 -135
  212. package/src/service/quoting.ts +0 -78
  213. package/src/service/resolve_environment.ts +0 -15
  214. package/src/service/restricted_mode.spec.ts +0 -299
  215. package/src/service/source_extraction.ts +0 -225
  216. package/src/service/theme_store.ts +0 -199
  217. package/src/storage/DatabaseInterface.ts +0 -198
  218. package/src/storage/StorageManager.ts +0 -113
  219. package/src/storage/duckdb/ConnectionRepository.ts +0 -143
  220. package/src/storage/duckdb/DuckDBConnection.ts +0 -123
  221. package/src/storage/duckdb/DuckDBRepository.ts +0 -208
  222. package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
  223. package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
  224. package/src/storage/duckdb/MaterializationRepository.ts +0 -336
  225. package/src/storage/duckdb/PackageRepository.ts +0 -135
  226. package/src/storage/duckdb/schema.ts +0 -229
  227. package/src/stream_helpers.spec.ts +0 -280
  228. package/src/stream_helpers.ts +0 -162
  229. package/src/telemetry.ts +0 -20
  230. package/src/test_helpers/metrics_harness.ts +0 -166
  231. package/src/theme_key_parity.spec.ts +0 -57
  232. package/src/utils.ts +0 -29
  233. package/tests/fixtures/authorize-compile/model.malloy +0 -9
  234. package/tests/fixtures/authorize-compile/publisher.json +0 -4
  235. package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
  236. package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
  237. package/tests/fixtures/html-pages-test/data.csv +0 -3
  238. package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
  239. package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
  240. package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
  241. package/tests/fixtures/html-pages-test/public/data.json +0 -1
  242. package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
  243. package/tests/fixtures/html-pages-test/public/index.html +0 -9
  244. package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
  245. package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
  246. package/tests/fixtures/html-pages-test/public/slides.html +0 -12
  247. package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
  248. package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
  249. package/tests/fixtures/html-pages-test/publisher.json +0 -5
  250. package/tests/fixtures/html-pages-test/report.malloy +0 -1
  251. package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
  252. package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
  253. package/tests/fixtures/persist-multi-level/publisher.json +0 -5
  254. package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
  255. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
  256. package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
  257. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
  258. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
  259. package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
  260. package/tests/fixtures/persist-test/data/orders.csv +0 -5
  261. package/tests/fixtures/persist-test/persist_test.malloy +0 -11
  262. package/tests/fixtures/persist-test/publisher.json +0 -5
  263. package/tests/fixtures/publisher.config.json +0 -15
  264. package/tests/fixtures/query-givens/data/orders.csv +0 -7
  265. package/tests/fixtures/query-givens/model.malloy +0 -34
  266. package/tests/fixtures/query-givens/publisher.json +0 -5
  267. package/tests/fixtures/xlsx/database.xlsx +0 -0
  268. package/tests/harness/mcp_test_setup.ts +0 -332
  269. package/tests/harness/rest_e2e.ts +0 -68
  270. package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
  271. package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
  272. package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
  273. package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
  274. package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
  275. package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
  276. package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
  277. package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
  278. package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
  279. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
  280. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
  281. package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
  282. package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
  283. package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
  284. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
  285. package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
  286. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
  287. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
  288. package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
  289. package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
  290. package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
  291. package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
  292. package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
  293. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
  294. package/tests/unit/controller/theme.controller.spec.ts +0 -141
  295. package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
  296. package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
  297. package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
  298. package/tests/unit/duckdb/repositories.test.ts +0 -208
  299. package/tests/unit/ducklake/ducklake.test.ts +0 -952
  300. package/tests/unit/service/theme_store.spec.ts +0 -139
  301. package/tsconfig.json +0 -32
@@ -1,240 +0,0 @@
1
- import type * as Malloy from "@malloydata/malloy-interfaces";
2
- import { bigIntReplacer } from "../json_utils";
3
- import type { QueryRowLimitSource } from "../service/model_limits";
4
-
5
- /**
6
- * The agent-facing shape for a query result, matched to what Credible's
7
- * `execute_query` returns.
8
- *
9
- * Matching matters for a specific workflow: a data app is authored locally
10
- * against Publisher and then served through Credible. An agent that sees one
11
- * response shape while developing and a different one in production has to
12
- * learn both, and a shared skill cannot describe either without forking. So the
13
- * field names here are Credible's, deliberately, including the leading
14
- * underscores and the mixed casing of `renderLogErrors`.
15
- *
16
- * Also following Credible: the truncation fields appear only when truncation
17
- * happened, rather than always carrying `false`.
18
- *
19
- * The two `_limit_...` fields are new to both products. Three separate things
20
- * can shorten a result and only one of them was ever reported:
21
- *
22
- * 1. The query row cap, pushed into the SQL. A query with no `limit:` of its
23
- * own gets DEFAULT_QUERY_ROW_LIMIT (1000) rows, which is far under
24
- * PUBLISHER_MAX_QUERY_ROWS, so nothing raises and nothing warns. An agent
25
- * reports statistics on a silent sample. This is why `_limit_hit` exists,
26
- * and unlike the others it is not derivable by a client: the cap depends on
27
- * server config and on the query's own LIMIT.
28
- * 2. The hard ceiling (maxRows / maxBytes), which throws 413 and is loud.
29
- * 3. The payload cap below, which degrades to a truncated result plus a
30
- * warning instead of overflowing the client's per-result limit.
31
- *
32
- * `_limit_hit` is a bound, not a total. The server cannot know the true row
33
- * count, because the database applied the cap. Landing exactly on the limit is
34
- * the only evidence available that rows were left behind.
35
- *
36
- * It is reported only when the cap was the server default, which `_limit_source`
37
- * names. `resolveModelQueryRowLimit` folds the query's own `limit:`/`top:` into
38
- * the same number, so equality also holds every time an author limited the query
39
- * on purpose: three of the eight views in the bundled storefront example use
40
- * `top:`, and each would otherwise report its complete answer as cut off. The
41
- * silent sample this exists to catch only ever happens under the default.
42
- */
43
- export interface QueryEnvelope {
44
- rows: unknown;
45
- /** Malloy metadata the flat rows drop: field types, render tags, timezone. */
46
- _meta: {
47
- schema: Malloy.Schema;
48
- annotations: Malloy.Annotation[];
49
- connection_name: string;
50
- model_annotations?: Malloy.Annotation[];
51
- query_timezone?: string;
52
- source_annotations?: Malloy.Annotation[];
53
- };
54
- /** The cap pushed into the SQL: the query's own LIMIT, else the server default. */
55
- _query_row_limit: number;
56
- /** Which of those two the cap came from. */
57
- _limit_source: QueryRowLimitSource;
58
- /**
59
- * Row count equals the cap AND the cap was the server default, so rows were
60
- * almost certainly left behind. A deliberate `limit:`/`top:` that returns
61
- * exactly what it asked for is a complete answer and does not set this.
62
- */
63
- _limit_hit: boolean;
64
- /** Present only when the payload cap dropped rows. */
65
- _rows_truncated?: boolean;
66
- _total_rows?: number;
67
- _returned_rows?: number;
68
- /**
69
- * The `query_id` property attached to this query's statements: the join key
70
- * into the backend's own query record (`QUERY_HISTORY`, `JOBS.labels`, the
71
- * statement text). Publisher-side and new to both products, like the two
72
- * `_limit_...` fields. Absent when nothing was attached, which is every query
73
- * on a deployment that has not enabled query metadata.
74
- */
75
- _query_id?: string;
76
- warning?: string;
77
- renderLogErrors?: string[];
78
- }
79
-
80
- /**
81
- * Cap on the serialized envelope, in characters. Same value Credible uses.
82
- *
83
- * Host-loop MCP clients enforce a per-tool-result ceiling of roughly 25k tokens;
84
- * past it the result is spilled to disk or rejected outright, and the model then
85
- * struggles to recover it. Chars stand in for tokens at about 4:1, with headroom
86
- * for the envelope itself.
87
- */
88
- export const MAX_RESULT_CHARS = 90_000;
89
-
90
- function serialize(envelope: QueryEnvelope): string {
91
- return JSON.stringify(envelope, bigIntReplacer, 2);
92
- }
93
-
94
- /**
95
- * Build the envelope, applying the payload cap.
96
- *
97
- * @param rows compactResult: flat row objects, straight from the driver.
98
- * @param rowLimit the cap pushed into the SQL.
99
- * @param result the full Malloy result, read only for its metadata.
100
- */
101
- export function buildQueryEnvelope(
102
- rows: unknown,
103
- rowLimit: number,
104
- result: Malloy.Result,
105
- renderLogErrors: string[] = [],
106
- limit = MAX_RESULT_CHARS,
107
- rowLimitSource: QueryRowLimitSource = "server_default",
108
- queryCorrelationId: string | null = null,
109
- ): QueryEnvelope {
110
- const rowCount = Array.isArray(rows) ? rows.length : 0;
111
- // Equality, not >=: the cap is pushed into the SQL, so the database cannot
112
- // return more than it. Landing exactly on it is the signal.
113
- //
114
- // Restricted to the server default on purpose. resolveModelQueryRowLimit
115
- // folds the query's own limit:/top: into the same cap, so equality also holds
116
- // every time an author deliberately limited the query: a modelled `top: 10`
117
- // view returning its 10 rows would otherwise be reported as cut off, and the
118
- // Contract rule would tell an agent that a top-N is "not the answer". Only
119
- // the silently-applied default is evidence that rows were left behind.
120
- const limitHit =
121
- rowLimitSource === "server_default" &&
122
- rowLimit > 0 &&
123
- rowCount === rowLimit;
124
-
125
- const envelope: QueryEnvelope = {
126
- rows,
127
- _meta: {
128
- schema: result.schema,
129
- annotations: result.annotations ?? [],
130
- connection_name: result.connection_name,
131
- ...(result.model_annotations !== undefined && {
132
- model_annotations: result.model_annotations,
133
- }),
134
- ...(result.query_timezone !== undefined && {
135
- query_timezone: result.query_timezone,
136
- }),
137
- ...(result.source_annotations !== undefined && {
138
- source_annotations: result.source_annotations,
139
- }),
140
- },
141
- _query_row_limit: rowLimit,
142
- _limit_source: rowLimitSource,
143
- _limit_hit: limitHit,
144
- ...(queryCorrelationId !== null && { _query_id: queryCorrelationId }),
145
- ...(renderLogErrors.length > 0 && { renderLogErrors }),
146
- };
147
-
148
- if (limitHit) {
149
- envelope.warning =
150
- `Returned exactly ${rowLimit} rows, the row limit applied to this query, so there are probably more. ` +
151
- `This is not a complete result: add an explicit limit, aggregate, or filter rather than reporting these rows as the whole set.`;
152
- }
153
-
154
- return fitToBudget(envelope, limit);
155
- }
156
-
157
- /**
158
- * Drop rows until the serialized envelope fits, by binary search on the row
159
- * count.
160
- *
161
- * The marker fields are set BEFORE the search, so their serialized size counts
162
- * against the limit. Adding them afterwards pushes the payload back over the cap
163
- * the truncation existed to respect. `_returned_rows` is measured at `total`
164
- * first, whose digit width is at least that of any value it ends up holding, so
165
- * the finished payload stays under the limit. Credible's `_truncate_rows` does
166
- * the same thing for the same reason.
167
- *
168
- * The search can land on zero rows, which needs its own wording rather than
169
- * "Showing 0 of N rows": that reads as an empty result set, and an agent will
170
- * report "no rows matched" for what is really one row too large to send. It is
171
- * reachable, because the hard ceiling upstream is maxBytes (50MB by default), so
172
- * a single row carrying a large text or JSON column passes
173
- * assertWithinModelResponseLimits and arrives here. Both wordings are measured,
174
- * and the search uses whichever is longer, so swapping one for the other after
175
- * the fact cannot push the payload back over the cap.
176
- *
177
- * One case cannot be fixed by dropping rows: when the envelope minus its rows
178
- * already exceeds the limit (a very wide schema in `_meta`). The result is then
179
- * returned over-limit rather than emptied further, and the warning says so.
180
- */
181
- function fitToBudget(envelope: QueryEnvelope, limit: number): QueryEnvelope {
182
- if (serialize(envelope).length <= limit) return envelope;
183
-
184
- const rows = envelope.rows;
185
- if (!Array.isArray(rows) || rows.length === 0) return envelope;
186
- const total = rows.length;
187
-
188
- const truncating: QueryEnvelope = {
189
- ...envelope,
190
- _rows_truncated: true,
191
- _total_rows: total,
192
- _returned_rows: total,
193
- };
194
- // The size warning joins any limit warning already present, and is included
195
- // in the measurement for the same reason as the counts.
196
- const sizeWarning = (kept: number) =>
197
- `Showing ${kept} of ${total} rows; the rest were dropped to fit the result size limit. Narrow the query rather than paging through it.`;
198
- const noneFitWarning =
199
- `No rows fit the result size limit: the first of ${total} rows is too large to send on its own. ` +
200
- `This is NOT an empty result and does not mean nothing matched. Select fewer columns, or truncate the oversized field, then run it again.`;
201
- // Measure with whichever wording is longer, so the one actually returned is
202
- // never bigger than the one the search sized the payload against.
203
- const measured =
204
- noneFitWarning.length > sizeWarning(total).length
205
- ? noneFitWarning
206
- : sizeWarning(total);
207
- truncating.warning = [envelope.warning, measured].filter(Boolean).join(" ");
208
-
209
- let low = 0;
210
- let high = total;
211
- let best = 0;
212
- while (low <= high) {
213
- const mid = Math.floor((low + high) / 2);
214
- if (
215
- serialize({ ...truncating, rows: rows.slice(0, mid) }).length <= limit
216
- ) {
217
- best = mid;
218
- low = mid + 1;
219
- } else {
220
- high = mid - 1;
221
- }
222
- }
223
-
224
- return {
225
- ...truncating,
226
- rows: rows.slice(0, best),
227
- _returned_rows: best,
228
- warning: [
229
- envelope.warning,
230
- best === 0 ? noneFitWarning : sizeWarning(best),
231
- ]
232
- .filter(Boolean)
233
- .join(" "),
234
- };
235
- }
236
-
237
- /** Serialize an envelope for transport, BigInt-safe. */
238
- export function serializeEnvelope(envelope: QueryEnvelope): string {
239
- return serialize(envelope);
240
- }
@@ -1,250 +0,0 @@
1
- import { beforeAll, describe, expect, it } from "bun:test";
2
- import { Client } from "@modelcontextprotocol/sdk/client/index.js";
3
- import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
4
- import { initializeMcpServer } from "./server";
5
- import { RELOAD_FAILURE_IS_SAFE } from "./tools/reload_package_tool";
6
- import type { EnvironmentStore } from "../service/environment_store";
7
-
8
- /**
9
- * End-to-end coverage of the unified MCP server over the real MCP protocol,
10
- * using the SDK's in-memory transport (no HTTP, no network, no DuckDB).
11
- * Exercises tool registration (including the agent retrieval tools that now
12
- * live on the same server), the dual-channel skill prompts, a real tool call
13
- * (searchDocs over the bundled index), and the getContext error path.
14
- */
15
- describe("MCP server over the MCP protocol (in-memory)", () => {
16
- let client: Client;
17
-
18
- beforeAll(async () => {
19
- // searchDocs does not touch the store, and the tools' error paths only
20
- // need getEnvironment to reject. The one exception is `compiles-badly`,
21
- // which resolves to a compileSource returning an error diagnostic: that
22
- // is malloy_compile's own isError path (a bad Malloy snippet rather than
23
- // a thrown error), and it is only reachable with an environment that
24
- // exists.
25
- const stubStore = {
26
- getEnvironment: async (name: string) => {
27
- if (name === "compiles-badly") {
28
- return {
29
- compileSource: async () => ({
30
- problems: [
31
- {
32
- severity: "error",
33
- message: "'nope' is not defined",
34
- code: "field-not-found",
35
- at: {
36
- range: {
37
- start: { line: 2, character: 3 },
38
- end: { line: 2, character: 7 },
39
- },
40
- },
41
- },
42
- ],
43
- }),
44
- };
45
- }
46
- throw new Error(`Environment not found: ${name}`);
47
- },
48
- } as unknown as EnvironmentStore;
49
-
50
- const server = initializeMcpServer(stubStore);
51
- const [clientTransport, serverTransport] =
52
- InMemoryTransport.createLinkedPair();
53
- await server.connect(serverTransport);
54
- client = new Client({ name: "mcp-protocol-test", version: "0.0.0" });
55
- await client.connect(clientTransport);
56
- });
57
-
58
- it("exposes the agent retrieval tools alongside the core tools", async () => {
59
- const { tools } = await client.listTools();
60
- const names = new Set(tools.map((t) => t.name));
61
- expect(names.has("malloy_getContext")).toBe(true);
62
- expect(names.has("malloy_searchDocs")).toBe(true);
63
- expect(names.has("malloy_executeQuery")).toBe(true);
64
- expect(names.has("malloy_compile")).toBe(true);
65
- expect(names.has("malloy_reloadPackage")).toBe(true);
66
- });
67
-
68
- /**
69
- * Tool descriptions are truncated by some clients. `malloy_getContext`'s was
70
- * observed arriving cut off mid-sentence at 2271 characters, and what a tail
71
- * cut removes is whatever the description put last.
72
- *
73
- * Two defenses, and the ordering one matters more. No single number is
74
- * portable, since the cap belongs to the client and is not published; this
75
- * budget only stops silent regrowth past the one length we have watched fail.
76
- * The real rule is that contract rules come BEFORE the reference material, so
77
- * a cut costs an agent the worked example rather than the invariants it
78
- * cannot self-correct without. See docs/agent-skills/tool-description-template.md.
79
- */
80
- it("keeps every tool description under the truncation budget", async () => {
81
- const BUDGET = 2000;
82
- const { tools } = await client.listTools();
83
- const oversized = tools
84
- .filter((t) => (t.description ?? "").length > BUDGET)
85
- .map((t) => `${t.name} (${t.description?.length})`);
86
- expect(oversized).toEqual([]);
87
- });
88
-
89
- it("puts contract rules ahead of the reference sections", async () => {
90
- // The section a tail-truncating client drops must never be the invariants.
91
- const { tools } = await client.listTools();
92
- for (const tool of tools) {
93
- const description = tool.description ?? "";
94
- const contract = description.indexOf("## Contract rules");
95
- if (contract === -1) continue;
96
- for (const later of ["## Response", "## Worked example"]) {
97
- const index = description.indexOf(later);
98
- if (index !== -1) {
99
- expect(contract).toBeLessThan(index);
100
- }
101
- }
102
- }
103
- });
104
-
105
- it("exposes the skill set as dual-channel prompts", async () => {
106
- const { prompts } = await client.listPrompts();
107
- expect(prompts.length).toBeGreaterThanOrEqual(24);
108
- expect(prompts.some((p) => p.name === "malloy-analysis")).toBe(true);
109
- });
110
-
111
- it("delivers orientation instructions to the connecting client", () => {
112
- const instructions = client.getInstructions();
113
- expect(instructions).toBeDefined();
114
- expect(instructions).toContain("malloy_getContext");
115
- // Pin the shared fragment, not just that instructions exist. The whole
116
- // point of exporting it is that this surface and the tool description
117
- // cannot drift, and they have drifted twice. Without this, deleting the
118
- // interpolation leaves the suite green and the drift comes back.
119
- expect(instructions).toContain(RELOAD_FAILURE_IS_SAFE);
120
- });
121
-
122
- it("malloy_searchDocs returns relevant docs over the protocol", async () => {
123
- const res = await client.callTool({
124
- name: "malloy_searchDocs",
125
- arguments: { query: "window functions" },
126
- });
127
- const content = res.content as Array<{ resource?: { text?: string } }>;
128
- const results = JSON.parse(
129
- content?.[0]?.resource?.text ?? "[]",
130
- ) as Array<{
131
- url: string;
132
- }>;
133
- expect(results.length).toBeGreaterThan(0);
134
- expect(results[0].url.length).toBeGreaterThan(0);
135
- });
136
-
137
- /**
138
- * Assert the tool's OWN error payload, not just isError. The SDK sets
139
- * isError for any uncaught throw, so `expect(res.isError).toBe(true)` passes
140
- * even with the tool's catch deleted entirely. Only these tools emit a
141
- * resource block carrying structured JSON; a raw throw yields the SDK's text
142
- * content instead, so this shape is what actually pins the catch.
143
- */
144
- function expectToolErrorPayload(result: unknown): {
145
- error: string;
146
- suggestions: string[];
147
- } {
148
- const res = result as {
149
- isError?: boolean;
150
- content?: Array<{
151
- type?: string;
152
- text?: string;
153
- resource?: { text?: string; mimeType?: string };
154
- }>;
155
- };
156
- expect(res.isError).toBe(true);
157
- expect(res.content?.[0]?.type).toBe("resource");
158
- expect(res.content?.[0]?.resource?.mimeType).toBe("application/json");
159
- const payload = JSON.parse(res.content?.[0]?.resource?.text ?? "{}");
160
- expect(typeof payload.error).toBe("string");
161
- expect(Array.isArray(payload.suggestions)).toBe(true);
162
-
163
- // The structured payload alone is invisible to a client that renders
164
- // only text blocks on an error, which is how a real diagnostic ends up
165
- // reported as a bare "Unknown error". Every error must also say it in
166
- // plain text.
167
- const textBlock = res.content?.find((b) => b.type === "text");
168
- expect(textBlock?.text).toContain(payload.error);
169
-
170
- return payload;
171
- }
172
-
173
- it("malloy_getContext returns its own error payload over the protocol", async () => {
174
- const res = await client.callTool({
175
- name: "malloy_getContext",
176
- arguments: {
177
- environmentName: "nope",
178
- packageName: "nope",
179
- query: "x",
180
- },
181
- });
182
- expect(expectToolErrorPayload(res).error).toContain("nope");
183
- // getContext answers with `results` at every tier, so an error keeps the
184
- // key rather than making callers branch on success first.
185
- const payload = JSON.parse(
186
- (res.content as Array<{ resource?: { text?: string } }>)[0]?.resource
187
- ?.text ?? "{}",
188
- );
189
- expect(payload.results).toEqual([]);
190
- });
191
-
192
- it("malloy_compile returns its own error payload over the protocol", async () => {
193
- const res = await client.callTool({
194
- name: "malloy_compile",
195
- arguments: {
196
- environmentName: "nope",
197
- packageName: "nope",
198
- modelPath: "x.malloy",
199
- source: "run: x -> { aggregate: c is count() }",
200
- },
201
- });
202
- expect(expectToolErrorPayload(res).error).toContain("nope");
203
- });
204
-
205
- it("malloy_compile states a compile diagnostic in text over the protocol", async () => {
206
- // The diagnostics path keeps its {status, diagnostics} payload rather
207
- // than the {error, suggestions} of the thrown-error path, so it cannot go
208
- // through expectToolErrorPayload. What it shares is the invariant that
209
- // matters: an isError result is never resource-only. Asserted over the
210
- // real transport because a client that renders only text is exactly the
211
- // one this was invisible to.
212
- const res = await client.callTool({
213
- name: "malloy_compile",
214
- arguments: {
215
- environmentName: "compiles-badly",
216
- packageName: "p",
217
- modelPath: "m.malloy",
218
- source: "run: nope -> { aggregate: c is count() }",
219
- },
220
- });
221
- const content = res.content as Array<{
222
- type?: string;
223
- text?: string;
224
- resource?: { text?: string; mimeType?: string };
225
- }>;
226
- expect(res.isError).toBe(true);
227
- expect(content[0]?.type).toBe("resource");
228
- expect(content[0]?.resource?.mimeType).toBe("application/json");
229
- expect(JSON.parse(content[0]?.resource?.text ?? "{}").status).toBe(
230
- "error",
231
- );
232
- const textBlock = content.find((b) => b.type === "text");
233
- expect(textBlock?.text).toContain("'nope' is not defined");
234
- expect(textBlock?.text).toContain("field-not-found");
235
- });
236
-
237
- it("malloy_reloadPackage returns its own error payload over the protocol", async () => {
238
- const res = await client.callTool({
239
- name: "malloy_reloadPackage",
240
- arguments: { environmentName: "nope", packageName: "nope" },
241
- });
242
- expect(expectToolErrorPayload(res).error).toContain("nope");
243
- });
244
-
245
- it("a skill prompt returns its body", async () => {
246
- const res = await client.getPrompt({ name: "malloy-analysis" });
247
- const content = res.messages[0].content as { text?: string };
248
- expect((content.text ?? "").length).toBeGreaterThan(200);
249
- });
250
- });
package/src/mcp/server.ts DELETED
@@ -1,87 +0,0 @@
1
- import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
2
- import { EnvironmentStore } from "../service/environment_store";
3
-
4
- import { formatDuration, logger } from "../logger";
5
- import { registerCompileTool } from "./tools/compile_tool";
6
- import { registerDocsSearchTool } from "./tools/docs_search_tool";
7
- import { registerExecuteQueryTool } from "./tools/execute_query_tool";
8
- import { registerGetContextTool } from "./tools/get_context_tool";
9
- import {
10
- registerReloadPackageTool,
11
- RELOAD_FAILURE_IS_SAFE,
12
- } from "./tools/reload_package_tool";
13
- import skillsBundle from "./skills/skills_bundle.json";
14
-
15
- export const testServerInfo = {
16
- name: "malloy-publisher-mcp-server",
17
- version: "0.0.1",
18
- displayName: "Malloy Publisher MCP Server",
19
- description: "Provides access to Malloy models and query execution via MCP.",
20
- };
21
-
22
- // Build-time bundle of the agent skills (see skills/build_skills_bundle.ts),
23
- // exposed as MCP prompts for the dual-channel delivery below.
24
- const AGENT_SKILLS = (
25
- skillsBundle as {
26
- skills: { name: string; description: string; body: string }[];
27
- }
28
- ).skills;
29
-
30
- // Orientation delivered to any connecting MCP client via the initialize
31
- // response (the on-the-wire analog of AGENTS.md), so an agent that reached the
32
- // server over npx without cloning the repo still knows how to work. Kept
33
- // workflow-focused rather than a tool catalog, which the client already gets
34
- // from listTools, so it does not drift as the tool set changes.
35
- // SECURITY: this is delivered pre-authorization to any connecting client, so it
36
- // must stay STATIC. It may interpolate module-scope string LITERALS (see
37
- // RELOAD_FAILURE_IS_SAFE, shared with the tool description so the two cannot
38
- // drift). Not "constants": `const x = process.env.FOO` is a module-scope
39
- // constant and would put deployment config into an unauthenticated string. It
40
- // must never interpolate environment, package, connection, or request data.
41
- const MCP_INSTRUCTIONS = `Malloy Publisher serves one or more Malloy semantic-model packages, so you can discover what data exists and answer questions against it, grounded in the names the model actually defines.
42
-
43
- Start with malloy_getContext. Call it with no arguments to list the environments (each with its packages), with an environment to list its packages, with a package to list its sources, and with a package plus a plain-English question to get the sources, views, and fields most relevant to it. Use the names it returns verbatim and do not guess. Then run a query with malloy_executeQuery. To change a model: validate the edit with malloy_compile, save it, then call malloy_reloadPackage so the new sources and views become queryable by name without restarting the server. ${RELOAD_FAILURE_IS_SAFE}
44
-
45
- Task-specific guidance is served as prompts you can fetch by name: malloy-getting-started to begin, malloy-modeling to build or change a model, malloy-analysis to explore and answer questions, and malloy-review to check correctness.
46
-
47
- Results and any charts render in the Publisher web UI on the REST port (4000 by default).`;
48
-
49
- export function initializeMcpServer(
50
- environmentStore: EnvironmentStore,
51
- ): McpServer {
52
- logger.info("[MCP Init] Starting initializeMcpServer...");
53
- const startTime = performance.now();
54
-
55
- const mcpServer = new McpServer(testServerInfo, {
56
- instructions: MCP_INSTRUCTIONS,
57
- });
58
-
59
- registerExecuteQueryTool(mcpServer, environmentStore);
60
- registerGetContextTool(mcpServer, environmentStore);
61
- registerDocsSearchTool(mcpServer, environmentStore);
62
- registerCompileTool(mcpServer, environmentStore);
63
- registerReloadPackageTool(mcpServer, environmentStore);
64
-
65
- // Dual-channel: also expose each skill as an MCP prompt, so hosts that ingest
66
- // MCP but do not load skill files (e.g. Codex, ChatGPT, Cursor) can pull the
67
- // same guidance. Skill-aware hosts (Claude Code/Desktop) use the native skill
68
- // files. Note: MCP prompts are on-demand, so always-on "prevention" skills
69
- // become on-demand here; see docs/agent-skills/design-principles.md.
70
- for (const skill of AGENT_SKILLS) {
71
- mcpServer.prompt(skill.name, skill.description, () => ({
72
- messages: [
73
- {
74
- role: "user" as const,
75
- content: { type: "text" as const, text: skill.body },
76
- },
77
- ],
78
- }));
79
- }
80
-
81
- const endTime = performance.now();
82
- logger.info(`[MCP Init] Finished initializeMcpServer`, {
83
- duration: formatDuration(endTime - startTime),
84
- });
85
-
86
- return mcpServer;
87
- }
@@ -1,51 +0,0 @@
1
- import { describe, expect, it } from "bun:test";
2
- import { parseSkill, unquote } from "./build_skills_bundle";
3
-
4
- describe("build_skills_bundle parseSkill", () => {
5
- it("parses name, description, and body from frontmatter", () => {
6
- const md =
7
- "---\nname: malloy-queries\ndescription: Query patterns.\n---\n# Body\n\nText.";
8
- const s = parseSkill(md, "dir");
9
- expect(s.name).toBe("malloy-queries");
10
- expect(s.description).toBe("Query patterns.");
11
- expect(s.body).toBe("# Body\n\nText.");
12
- });
13
-
14
- it("keeps colons in the description (line-based, not split on colon)", () => {
15
- const md =
16
- "---\nname: x\ndescription: Use when: a thing happens.\n---\nbody";
17
- expect(parseSkill(md, "dir").description).toBe(
18
- "Use when: a thing happens.",
19
- );
20
- });
21
-
22
- it("tolerates CRLF line endings", () => {
23
- const md = "---\r\nname: x\r\ndescription: d\r\n---\r\nbody line";
24
- const s = parseSkill(md, "dir");
25
- expect(s.name).toBe("x");
26
- expect(s.description).toBe("d");
27
- expect(s.body).toBe("body line");
28
- });
29
-
30
- it("falls back to the dir name and empty description when frontmatter is missing", () => {
31
- const s = parseSkill("# Just a body, no frontmatter", "fallback-name");
32
- expect(s.name).toBe("fallback-name");
33
- expect(s.description).toBe("");
34
- expect(s.body).toBe("# Just a body, no frontmatter");
35
- });
36
-
37
- it("strips surrounding quotes from values", () => {
38
- const md = "---\nname: \"quoted-name\"\ndescription: 'q'\n---\nb";
39
- const s = parseSkill(md, "dir");
40
- expect(s.name).toBe("quoted-name");
41
- expect(s.description).toBe("q");
42
- });
43
- });
44
-
45
- describe("build_skills_bundle unquote", () => {
46
- it("removes a single layer of surrounding quotes", () => {
47
- expect(unquote('"hi"')).toBe("hi");
48
- expect(unquote("'hi'")).toBe("hi");
49
- expect(unquote("plain")).toBe("plain");
50
- });
51
- });