@malloy-publisher/server 0.0.235 → 0.0.236

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (301) hide show
  1. package/dist/app/api-doc.yaml +47 -17
  2. package/dist/package_load_worker.mjs +8 -1
  3. package/dist/server.mjs +471 -54
  4. package/package.json +5 -2
  5. package/.eslintrc.json +0 -44
  6. package/.prettierignore +0 -1
  7. package/.prettierrc +0 -4
  8. package/build.ts +0 -97
  9. package/eslint.config.mjs +0 -76
  10. package/k6-tests/README.md +0 -144
  11. package/k6-tests/bun.lock +0 -797
  12. package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
  13. package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
  14. package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
  15. package/k6-tests/load-test/load-test-crud.ts +0 -142
  16. package/k6-tests/load-test/load-test.ts +0 -784
  17. package/k6-tests/package.json +0 -27
  18. package/k6-tests/scripts/run-with-otel.sh +0 -50
  19. package/k6-tests/smoke-test/smoke-test.ts +0 -158
  20. package/k6-tests/tsconfig.json +0 -29
  21. package/k6-tests/utils/client_factory.ts +0 -128
  22. package/k6-tests/utils/common.ts +0 -489
  23. package/k6-tests/utils/logger.ts +0 -62
  24. package/k6-tests/utils/types.d.ts +0 -35
  25. package/malloy_mcp.dxt +0 -0
  26. package/publisher.config.example.bigquery.json +0 -33
  27. package/publisher.config.example.duckdb.json +0 -23
  28. package/publisher.config.json +0 -23
  29. package/scripts/bake-duckdb-extensions.js +0 -110
  30. package/src/config.spec.ts +0 -1535
  31. package/src/config.theme.spec.ts +0 -178
  32. package/src/config.ts +0 -1142
  33. package/src/constants.ts +0 -105
  34. package/src/controller/compile.controller.ts +0 -40
  35. package/src/controller/connection.controller.spec.ts +0 -849
  36. package/src/controller/connection.controller.ts +0 -901
  37. package/src/controller/database.controller.ts +0 -24
  38. package/src/controller/materialization.controller.spec.ts +0 -293
  39. package/src/controller/materialization.controller.ts +0 -319
  40. package/src/controller/model.controller.ts +0 -164
  41. package/src/controller/package.controller.spec.ts +0 -273
  42. package/src/controller/package.controller.ts +0 -306
  43. package/src/controller/query.controller.ts +0 -152
  44. package/src/controller/theme.controller.ts +0 -83
  45. package/src/controller/watch-mode.controller.ts +0 -243
  46. package/src/data_styles.ts +0 -71
  47. package/src/default-publisher.config.json +0 -23
  48. package/src/ducklake_version.spec.ts +0 -163
  49. package/src/ducklake_version.ts +0 -153
  50. package/src/errors.spec.ts +0 -119
  51. package/src/errors.ts +0 -237
  52. package/src/filter_deprecation.spec.ts +0 -64
  53. package/src/filter_deprecation.ts +0 -42
  54. package/src/health.spec.ts +0 -123
  55. package/src/health.ts +0 -233
  56. package/src/heap_check.spec.ts +0 -144
  57. package/src/heap_check.ts +0 -144
  58. package/src/instrumentation.ts +0 -214
  59. package/src/json_utils.spec.ts +0 -51
  60. package/src/json_utils.ts +0 -33
  61. package/src/logger.spec.ts +0 -210
  62. package/src/logger.ts +0 -251
  63. package/src/malloy_pin_prereqs.spec.ts +0 -25
  64. package/src/materialization_metrics.spec.ts +0 -169
  65. package/src/materialization_metrics.ts +0 -346
  66. package/src/mcp/error_messages.spec.ts +0 -35
  67. package/src/mcp/error_messages.ts +0 -183
  68. package/src/mcp/handler_utils.spec.ts +0 -108
  69. package/src/mcp/handler_utils.ts +0 -251
  70. package/src/mcp/mcp_constants.ts +0 -6
  71. package/src/mcp/query_envelope.spec.ts +0 -229
  72. package/src/mcp/query_envelope.ts +0 -240
  73. package/src/mcp/server.protocol.spec.ts +0 -250
  74. package/src/mcp/server.ts +0 -87
  75. package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
  76. package/src/mcp/skills/build_skills_bundle.ts +0 -176
  77. package/src/mcp/skills/skills_bundle.json +0 -1
  78. package/src/mcp/skills/skills_bundle.spec.ts +0 -171
  79. package/src/mcp/tool_response.spec.ts +0 -108
  80. package/src/mcp/tool_response.ts +0 -138
  81. package/src/mcp/tools/compile_tool.spec.ts +0 -315
  82. package/src/mcp/tools/compile_tool.ts +0 -208
  83. package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
  84. package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
  85. package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
  86. package/src/mcp/tools/docs_search_tool.ts +0 -128
  87. package/src/mcp/tools/embedding_index.spec.ts +0 -1236
  88. package/src/mcp/tools/embedding_index.ts +0 -808
  89. package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
  90. package/src/mcp/tools/execute_query_tool.ts +0 -316
  91. package/src/mcp/tools/get_context_eval.ts +0 -275
  92. package/src/mcp/tools/get_context_tool.spec.ts +0 -592
  93. package/src/mcp/tools/get_context_tool.ts +0 -585
  94. package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
  95. package/src/mcp/tools/reload_package_tool.ts +0 -132
  96. package/src/mcp_config.spec.ts +0 -919
  97. package/src/mcp_config.ts +0 -425
  98. package/src/oom_guards.integration.spec.ts +0 -269
  99. package/src/package_load/package_load_pool.spec.ts +0 -252
  100. package/src/package_load/package_load_pool.ts +0 -931
  101. package/src/package_load/package_load_worker.ts +0 -1034
  102. package/src/package_load/protocol.ts +0 -364
  103. package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
  104. package/src/package_load/rpc_wait_accountant.ts +0 -76
  105. package/src/package_load_metrics.spec.ts +0 -114
  106. package/src/package_load_metrics.ts +0 -127
  107. package/src/path_safety.spec.ts +0 -158
  108. package/src/path_safety.ts +0 -146
  109. package/src/pg_helpers.spec.ts +0 -223
  110. package/src/pg_helpers.ts +0 -52
  111. package/src/query_cap_metrics.spec.ts +0 -89
  112. package/src/query_cap_metrics.ts +0 -116
  113. package/src/query_concurrency.spec.ts +0 -247
  114. package/src/query_concurrency.ts +0 -237
  115. package/src/query_metadata_metrics.ts +0 -49
  116. package/src/query_param_utils.ts +0 -29
  117. package/src/query_timeout.spec.ts +0 -224
  118. package/src/query_timeout.ts +0 -179
  119. package/src/runtime/publisher.js +0 -328
  120. package/src/server-old.ts +0 -988
  121. package/src/server.ts +0 -2059
  122. package/src/service/annotations.spec.ts +0 -118
  123. package/src/service/annotations.ts +0 -91
  124. package/src/service/authorize.spec.ts +0 -154
  125. package/src/service/authorize.ts +0 -498
  126. package/src/service/authorize_integration.spec.ts +0 -2041
  127. package/src/service/build_plan.spec.ts +0 -685
  128. package/src/service/build_plan.ts +0 -797
  129. package/src/service/compile_authorize.spec.ts +0 -85
  130. package/src/service/compile_fragment_techniques.spec.ts +0 -156
  131. package/src/service/connection.spec.ts +0 -2360
  132. package/src/service/connection.ts +0 -2187
  133. package/src/service/connection_config.spec.ts +0 -734
  134. package/src/service/connection_config.ts +0 -899
  135. package/src/service/connection_federation.spec.ts +0 -184
  136. package/src/service/connection_fingerprint.spec.ts +0 -102
  137. package/src/service/connection_service.spec.ts +0 -711
  138. package/src/service/connection_service.ts +0 -291
  139. package/src/service/cron_evaluator.spec.ts +0 -79
  140. package/src/service/cron_evaluator.ts +0 -105
  141. package/src/service/db_utils.spec.ts +0 -1085
  142. package/src/service/db_utils.ts +0 -1502
  143. package/src/service/duckdb_instance_isolation.spec.ts +0 -137
  144. package/src/service/ducklake_lazy_attach.spec.ts +0 -110
  145. package/src/service/embedding_provider.spec.ts +0 -329
  146. package/src/service/embedding_provider.ts +0 -236
  147. package/src/service/environment.ts +0 -2191
  148. package/src/service/environment_admission.spec.ts +0 -344
  149. package/src/service/environment_compile.spec.ts +0 -197
  150. package/src/service/environment_store.spec.ts +0 -2274
  151. package/src/service/environment_store.ts +0 -2434
  152. package/src/service/environment_store_anchoring.spec.ts +0 -107
  153. package/src/service/environment_store_clone.spec.ts +0 -350
  154. package/src/service/explore_visibility.spec.ts +0 -434
  155. package/src/service/exports_probe.spec.ts +0 -107
  156. package/src/service/extension_fetch_policy.spec.ts +0 -256
  157. package/src/service/filter.spec.ts +0 -458
  158. package/src/service/filter.ts +0 -341
  159. package/src/service/filter_bypass.spec.ts +0 -418
  160. package/src/service/filter_integration.spec.ts +0 -1004
  161. package/src/service/freshness.spec.ts +0 -183
  162. package/src/service/freshness.ts +0 -112
  163. package/src/service/gcs_s3_utils.ts +0 -427
  164. package/src/service/given.ts +0 -105
  165. package/src/service/givens_integration.spec.ts +0 -219
  166. package/src/service/manifest_loader.spec.ts +0 -189
  167. package/src/service/manifest_loader.ts +0 -156
  168. package/src/service/materialization_build_session.spec.ts +0 -435
  169. package/src/service/materialization_build_session.ts +0 -681
  170. package/src/service/materialization_config_validation.spec.ts +0 -99
  171. package/src/service/materialization_config_validation.ts +0 -120
  172. package/src/service/materialization_eligibility.spec.ts +0 -158
  173. package/src/service/materialization_eligibility.ts +0 -305
  174. package/src/service/materialization_schedule_surface.spec.ts +0 -256
  175. package/src/service/materialization_scheduler.spec.ts +0 -289
  176. package/src/service/materialization_scheduler.ts +0 -304
  177. package/src/service/materialization_serve_transform.spec.ts +0 -1003
  178. package/src/service/materialization_serve_transform.ts +0 -779
  179. package/src/service/materialization_service.spec.ts +0 -2373
  180. package/src/service/materialization_service.ts +0 -2342
  181. package/src/service/materialization_test_fixtures.ts +0 -249
  182. package/src/service/model.spec.ts +0 -1157
  183. package/src/service/model.ts +0 -3370
  184. package/src/service/model_limits.spec.ts +0 -209
  185. package/src/service/model_limits.ts +0 -131
  186. package/src/service/model_storage_serve.spec.ts +0 -193
  187. package/src/service/model_storage_serve_joins.spec.ts +0 -193
  188. package/src/service/package.spec.ts +0 -612
  189. package/src/service/package.ts +0 -1799
  190. package/src/service/package_manifest.spec.ts +0 -230
  191. package/src/service/package_manifest.ts +0 -241
  192. package/src/service/package_memory_governor.spec.ts +0 -173
  193. package/src/service/package_memory_governor.ts +0 -233
  194. package/src/service/package_quote_bound_tables.spec.ts +0 -171
  195. package/src/service/package_race.spec.ts +0 -208
  196. package/src/service/package_reload_safety.spec.ts +0 -193
  197. package/src/service/package_rollback.spec.ts +0 -190
  198. package/src/service/package_worker_path.spec.ts +0 -645
  199. package/src/service/path_injection.spec.ts +0 -39
  200. package/src/service/persist_annotation_validation.spec.ts +0 -89
  201. package/src/service/persist_annotation_validation.ts +0 -59
  202. package/src/service/persistence_policy.spec.ts +0 -374
  203. package/src/service/proxy.spec.ts +0 -414
  204. package/src/service/proxy.ts +0 -248
  205. package/src/service/query_boundary.spec.ts +0 -470
  206. package/src/service/query_metadata.spec.ts +0 -408
  207. package/src/service/query_metadata.ts +0 -492
  208. package/src/service/query_metadata_identity.spec.ts +0 -149
  209. package/src/service/query_text.spec.ts +0 -79
  210. package/src/service/query_text.ts +0 -43
  211. package/src/service/quoting.spec.ts +0 -135
  212. package/src/service/quoting.ts +0 -78
  213. package/src/service/resolve_environment.ts +0 -15
  214. package/src/service/restricted_mode.spec.ts +0 -299
  215. package/src/service/source_extraction.ts +0 -225
  216. package/src/service/theme_store.ts +0 -199
  217. package/src/storage/DatabaseInterface.ts +0 -198
  218. package/src/storage/StorageManager.ts +0 -113
  219. package/src/storage/duckdb/ConnectionRepository.ts +0 -143
  220. package/src/storage/duckdb/DuckDBConnection.ts +0 -123
  221. package/src/storage/duckdb/DuckDBRepository.ts +0 -208
  222. package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
  223. package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
  224. package/src/storage/duckdb/MaterializationRepository.ts +0 -336
  225. package/src/storage/duckdb/PackageRepository.ts +0 -135
  226. package/src/storage/duckdb/schema.ts +0 -229
  227. package/src/stream_helpers.spec.ts +0 -280
  228. package/src/stream_helpers.ts +0 -162
  229. package/src/telemetry.ts +0 -20
  230. package/src/test_helpers/metrics_harness.ts +0 -166
  231. package/src/theme_key_parity.spec.ts +0 -57
  232. package/src/utils.ts +0 -29
  233. package/tests/fixtures/authorize-compile/model.malloy +0 -9
  234. package/tests/fixtures/authorize-compile/publisher.json +0 -4
  235. package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
  236. package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
  237. package/tests/fixtures/html-pages-test/data.csv +0 -3
  238. package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
  239. package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
  240. package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
  241. package/tests/fixtures/html-pages-test/public/data.json +0 -1
  242. package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
  243. package/tests/fixtures/html-pages-test/public/index.html +0 -9
  244. package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
  245. package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
  246. package/tests/fixtures/html-pages-test/public/slides.html +0 -12
  247. package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
  248. package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
  249. package/tests/fixtures/html-pages-test/publisher.json +0 -5
  250. package/tests/fixtures/html-pages-test/report.malloy +0 -1
  251. package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
  252. package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
  253. package/tests/fixtures/persist-multi-level/publisher.json +0 -5
  254. package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
  255. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
  256. package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
  257. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
  258. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
  259. package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
  260. package/tests/fixtures/persist-test/data/orders.csv +0 -5
  261. package/tests/fixtures/persist-test/persist_test.malloy +0 -11
  262. package/tests/fixtures/persist-test/publisher.json +0 -5
  263. package/tests/fixtures/publisher.config.json +0 -15
  264. package/tests/fixtures/query-givens/data/orders.csv +0 -7
  265. package/tests/fixtures/query-givens/model.malloy +0 -34
  266. package/tests/fixtures/query-givens/publisher.json +0 -5
  267. package/tests/fixtures/xlsx/database.xlsx +0 -0
  268. package/tests/harness/mcp_test_setup.ts +0 -332
  269. package/tests/harness/rest_e2e.ts +0 -68
  270. package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
  271. package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
  272. package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
  273. package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
  274. package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
  275. package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
  276. package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
  277. package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
  278. package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
  279. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
  280. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
  281. package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
  282. package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
  283. package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
  284. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
  285. package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
  286. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
  287. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
  288. package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
  289. package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
  290. package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
  291. package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
  292. package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
  293. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
  294. package/tests/unit/controller/theme.controller.spec.ts +0 -141
  295. package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
  296. package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
  297. package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
  298. package/tests/unit/duckdb/repositories.test.ts +0 -208
  299. package/tests/unit/ducklake/ducklake.test.ts +0 -952
  300. package/tests/unit/service/theme_store.spec.ts +0 -139
  301. package/tsconfig.json +0 -32
@@ -1,645 +0,0 @@
1
- /**
2
- * Integration test: exercise `Package.create` with the package-load
3
- * worker pool enabled (PACKAGE_LOAD_WORKERS=1).
4
- *
5
- * Validates that the worker-load path:
6
- * - reads the manifest, probes embedded databases, and compiles
7
- * every model in a single off-thread job
8
- * - produces a live `Package` whose `Model`s have populated
9
- * `modelDef` / `sources` / `queries`
10
- * - hydrates the `ModelMaterializer` from `modelDef` on first
11
- * query (no recompile) — verified end-to-end by running a
12
- * query through the resulting Model and getting a result
13
- *
14
- * Kept separate from `package.spec.ts` so the existing tests keep
15
- * running on the in-process path without paying worker startup cost.
16
- *
17
- * Pool reuse strategy: one `PackageLoadPool` shared across all
18
- * cases in this file. Spawning a fresh worker per test crashes Bun
19
- * (segfault) because DuckDB's native bindings don't tolerate being
20
- * loaded concurrently into multiple worker isolates of the same Bun
21
- * process. Production uses one pool; this matches.
22
- */
23
- import {
24
- afterAll,
25
- afterEach,
26
- beforeAll,
27
- beforeEach,
28
- describe,
29
- expect,
30
- it,
31
- spyOn,
32
- } from "bun:test";
33
- import * as fs from "fs";
34
- import * as os from "os";
35
- import * as path from "path";
36
- import { logger } from "../logger";
37
- import {
38
- PackageLoadPool,
39
- __setPackageLoadPoolForTests,
40
- } from "../package_load/package_load_pool";
41
- import { Package } from "./package";
42
-
43
- const ORIGINAL_ENV = process.env.PACKAGE_LOAD_WORKERS;
44
-
45
- describe("Package.create via worker pool", () => {
46
- let tempDir: string;
47
- let pool: PackageLoadPool;
48
-
49
- beforeAll(async () => {
50
- process.env.PACKAGE_LOAD_WORKERS = "1";
51
- pool = new PackageLoadPool(1);
52
- // Wire our pool into the module-level singleton so Package.create
53
- // picks it up via getPackageLoadPool().
54
- await __setPackageLoadPoolForTests(pool);
55
- });
56
-
57
- afterAll(async () => {
58
- await __setPackageLoadPoolForTests(null);
59
- if (ORIGINAL_ENV === undefined) {
60
- delete process.env.PACKAGE_LOAD_WORKERS;
61
- } else {
62
- process.env.PACKAGE_LOAD_WORKERS = ORIGINAL_ENV;
63
- }
64
- });
65
-
66
- beforeEach(() => {
67
- tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "publisher-pkg-worker-"));
68
- });
69
-
70
- afterEach(() => {
71
- if (tempDir) {
72
- // tempDir gets wiped by Package.create on failure (it's the
73
- // staging-cleanup path); ignore ENOENT here.
74
- try {
75
- fs.rmSync(tempDir, { recursive: true, force: true });
76
- } catch {
77
- /* already gone */
78
- }
79
- tempDir = "";
80
- }
81
- });
82
-
83
- async function makeMalloyConfig(): Promise<{
84
- malloyConfig: import("@malloydata/malloy").MalloyConfig;
85
- duckdb: { close: () => Promise<void> };
86
- }> {
87
- const { MalloyConfig, FixedConnectionMap } = await import(
88
- "@malloydata/malloy"
89
- );
90
- const { DuckDBConnection } = await import("@malloydata/db-duckdb");
91
- const duckdb = new DuckDBConnection("duckdb", ":memory:");
92
- const connections = new FixedConnectionMap(
93
- new Map([["duckdb", duckdb]]),
94
- "duckdb",
95
- );
96
- const malloyConfig = new MalloyConfig({ connections: {} });
97
- malloyConfig.wrapConnections(() => connections);
98
- return { malloyConfig, duckdb };
99
- }
100
-
101
- function writeManifest(): void {
102
- fs.writeFileSync(
103
- path.join(tempDir, "publisher.json"),
104
- JSON.stringify({ name: "pkg", description: "test package" }),
105
- );
106
- }
107
-
108
- it("loads a package end-to-end and serves a query through the hydrated materializer", async () => {
109
- writeManifest();
110
- // Define `total_v` as a *view* on the source so the query
111
- // builder's `run: nums -> total_v` form resolves. (Top-level
112
- // queries take the `run: total_q` form — orthogonal path that
113
- // the in-process tests already cover.)
114
- fs.writeFileSync(
115
- path.join(tempDir, "trivial.malloy"),
116
- `source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
117
- measure: total is a.sum()
118
- view: total_v is { aggregate: total }
119
- }`,
120
- );
121
-
122
- const { malloyConfig, duckdb } = await makeMalloyConfig();
123
- try {
124
- const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
125
- expect(pkg).toBeInstanceOf(Package);
126
- expect(pkg.getModelPaths()).toEqual(["trivial.malloy"]);
127
-
128
- const model = pkg.getModel("trivial.malloy");
129
- expect(model).toBeDefined();
130
- const apiModel = (await model!.getModel()) as {
131
- modelDef?: string;
132
- sources?: { name?: string }[];
133
- };
134
- expect(apiModel.modelDef).toBeDefined();
135
- expect(apiModel.sources?.[0]?.name).toBe("nums");
136
-
137
- // First query against the package — hydrates the
138
- // ModelMaterializer from the worker's modelDef without a
139
- // recompile, then runs the SQL against the *main thread's*
140
- // DuckDB connection (the only one with the in-memory `nums`
141
- // source loaded via duckdb.sql()).
142
- const { result } = await model!.getQueryResults(
143
- "nums",
144
- "total_v",
145
- undefined,
146
- );
147
- expect(result.data).toBeDefined();
148
- } finally {
149
- await duckdb.close();
150
- }
151
- });
152
-
153
- it("propagates a per-model compile failure as a thrown error from Package.create", async () => {
154
- writeManifest();
155
- fs.writeFileSync(
156
- path.join(tempDir, "broken.malloy"),
157
- `source: bad is duckdb.sql("select 1 as a") extend {
158
- measure: oops is THIS_FUNC_DOES_NOT_EXIST(a)
159
- }`,
160
- );
161
-
162
- const { malloyConfig, duckdb } = await makeMalloyConfig();
163
- try {
164
- await expect(
165
- Package.create("env", "pkg", tempDir, malloyConfig),
166
- ).rejects.toBeInstanceOf(Error);
167
- } finally {
168
- await duckdb.close();
169
- }
170
- });
171
-
172
- it("validates and surfaces a valid #(authorize) model through the worker", async () => {
173
- writeManifest();
174
- fs.writeFileSync(
175
- path.join(tempDir, "gated.malloy"),
176
- `##! experimental.givens
177
-
178
- given:
179
- ROLE :: string
180
-
181
- #(authorize) "$ROLE = 'analyst'"
182
- source: gated is duckdb.sql("select 1 as id")`,
183
- );
184
-
185
- const { malloyConfig, duckdb } = await makeMalloyConfig();
186
- try {
187
- const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
188
- const model = pkg.getModel("gated.malloy");
189
- const apiModel = (await model!.getModel()) as {
190
- sources?: { name?: string; authorize?: string[] }[];
191
- };
192
- // The worker compiled the authorize probe (no throw) and surfaced the
193
- // effective expression list — proves worker-path validation runs.
194
- expect(apiModel.sources?.[0]?.authorize).toEqual([
195
- "$ROLE = 'analyst'",
196
- ]);
197
- expect(model!.getAuthorize("gated")).toEqual(["$ROLE = 'analyst'"]);
198
- } finally {
199
- await duckdb.close();
200
- }
201
- });
202
-
203
- it("rejects a package whose #(authorize) references an unknown given (worker validation)", async () => {
204
- writeManifest();
205
- fs.writeFileSync(
206
- path.join(tempDir, "badgate.malloy"),
207
- `##! experimental.givens
208
-
209
- given:
210
- ROLE :: string
211
-
212
- #(authorize) "$NOPE = 'x'"
213
- source: gated is duckdb.sql("select 1 as id")`,
214
- );
215
-
216
- const { ModelCompilationError } = await import("../errors");
217
- const { malloyConfig, duckdb } = await makeMalloyConfig();
218
- try {
219
- // Must surface as a 424 ModelCompilationError across the worker
220
- // boundary, not a generic 500 — the worker serializes it with
221
- // isCompilationError so the main thread re-wraps it.
222
- await expect(
223
- Package.create("env", "pkg", tempDir, malloyConfig),
224
- ).rejects.toBeInstanceOf(ModelCompilationError);
225
- } finally {
226
- await duckdb.close();
227
- }
228
- });
229
-
230
- it("validates a valid #(authorize) source in a .malloynb notebook through the worker", async () => {
231
- writeManifest();
232
- fs.writeFileSync(
233
- path.join(tempDir, "gated.malloynb"),
234
- `>>>markdown
235
- # Gated notebook
236
-
237
- >>>malloy
238
- ##! experimental.givens
239
-
240
- given:
241
- ROLE :: string
242
-
243
- #(authorize) "$ROLE = 'analyst'"
244
- source: gated is duckdb.sql("select 1 as id")`,
245
- );
246
-
247
- const { malloyConfig, duckdb } = await makeMalloyConfig();
248
- try {
249
- const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
250
- const model = pkg.getModel("gated.malloynb");
251
- // compileNotebookModel ran authorize validation (no throw) and
252
- // surfaced the gate — the notebook compile path was previously
253
- // unexercised by tests.
254
- expect(model!.getAuthorize("gated")).toEqual(["$ROLE = 'analyst'"]);
255
- } finally {
256
- await duckdb.close();
257
- }
258
- });
259
-
260
- it("rejects a .malloynb notebook whose #(authorize) references an unknown given", async () => {
261
- writeManifest();
262
- fs.writeFileSync(
263
- path.join(tempDir, "badgate.malloynb"),
264
- `>>>malloy
265
- ##! experimental.givens
266
-
267
- given:
268
- ROLE :: string
269
-
270
- #(authorize) "$NOPE = 'x'"
271
- source: gated is duckdb.sql("select 1 as id")`,
272
- );
273
-
274
- const { ModelCompilationError } = await import("../errors");
275
- const { malloyConfig, duckdb } = await makeMalloyConfig();
276
- try {
277
- await expect(
278
- Package.create("env", "pkg", tempDir, malloyConfig),
279
- ).rejects.toBeInstanceOf(ModelCompilationError);
280
- } finally {
281
- await duckdb.close();
282
- }
283
- });
284
-
285
- it("logs a warning for a package whose view carries an invalid renderer tag", async () => {
286
- writeManifest();
287
- // `# big_value { sparkline=... }` is a child-only renderer config placed on
288
- // the view itself, with no activating big_value. The renderer declines to
289
- // match it. Render-tag validation runs on the main thread after the worker
290
- // hydrates the model. The misconfiguration is logged as a warning naming
291
- // the offending view; it does not fail the package load.
292
- fs.writeFileSync(
293
- path.join(tempDir, "bad_render.malloy"),
294
- `source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
295
- measure: total is a.sum()
296
- # big_value { sparkline=trend }
297
- view: card is {
298
- aggregate: total
299
- nest:
300
- # line_chart { size=spark }
301
- trend is { group_by: a; aggregate: total }
302
- }
303
- }`,
304
- );
305
-
306
- const warnSpy = spyOn(logger, "warn");
307
- const { malloyConfig, duckdb } = await makeMalloyConfig();
308
- try {
309
- const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
310
- expect(pkg.getModelPaths()).toEqual(["bad_render.malloy"]);
311
- const warnings = warnSpy.mock.calls.map((c) => String(c[0]));
312
- expect(
313
- warnings.some(
314
- (m) =>
315
- m.includes("Invalid renderer configuration") &&
316
- m.includes("nums -> card"),
317
- ),
318
- ).toBe(true);
319
- // The same finding rides the package response (non-fatal) so the
320
- // control plane / UI can surface it without scraping logs.
321
- const responseWarnings = pkg.getPackageMetadata().warnings ?? [];
322
- expect(
323
- responseWarnings.some(
324
- (w) =>
325
- w.model === "bad_render.malloy" &&
326
- w.target === "nums -> card" &&
327
- w.severity === "error",
328
- ),
329
- ).toBe(true);
330
- } finally {
331
- warnSpy.mockRestore();
332
- await duckdb.close();
333
- }
334
- });
335
-
336
- it("loads a package whose view renderer tags are valid", async () => {
337
- writeManifest();
338
- fs.writeFileSync(
339
- path.join(tempDir, "good_render.malloy"),
340
- `source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
341
- measure: total is a.sum()
342
- # bar_chart
343
- view: chart is { group_by: a; aggregate: total }
344
- }`,
345
- );
346
-
347
- const { malloyConfig, duckdb } = await makeMalloyConfig();
348
- try {
349
- const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
350
- expect(pkg.getModelPaths()).toEqual(["good_render.malloy"]);
351
- // A clean package omits the warnings field entirely.
352
- expect(pkg.getPackageMetadata().warnings).toBeUndefined();
353
- } finally {
354
- await duckdb.close();
355
- }
356
- });
357
-
358
- it("logs a warning for a backtick-quoted (hyphenated) source with a bad view render tag", async () => {
359
- writeManifest();
360
- // A source whose name needs Malloy backtick-quoting (here, a hyphen). The
361
- // validation target must quote the identifier; otherwise `run: bad-source
362
- // -> card` fails to lex, loadQuery throws, the catch swallows it, and the
363
- // misconfigured view is silently skipped -- which would defeat the feature
364
- // for any quote-needing name (the repo already has such a source,
365
- // `gated-source`, in authorize_integration.spec.ts).
366
- fs.writeFileSync(
367
- path.join(tempDir, "hyphen_render.malloy"),
368
- `source: \`bad-source\` is duckdb.sql("select 1 as a, 2 as b") extend {
369
- measure: total is a.sum()
370
- # big_value { sparkline=trend }
371
- view: card is {
372
- aggregate: total
373
- nest:
374
- # line_chart { size=spark }
375
- trend is { group_by: a; aggregate: total }
376
- }
377
- }`,
378
- );
379
-
380
- const warnSpy = spyOn(logger, "warn");
381
- const { malloyConfig, duckdb } = await makeMalloyConfig();
382
- try {
383
- const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
384
- expect(pkg.getModelPaths()).toEqual(["hyphen_render.malloy"]);
385
- const warnings = warnSpy.mock.calls.map((c) => String(c[0]));
386
- expect(
387
- warnings.some(
388
- (m) =>
389
- m.includes("Invalid renderer configuration") &&
390
- m.includes("bad-source -> card"),
391
- ),
392
- ).toBe(true);
393
- } finally {
394
- warnSpy.mockRestore();
395
- await duckdb.close();
396
- }
397
- });
398
-
399
- it("logs a warning for a source name containing an escaped backtick with a bad view render tag", async () => {
400
- writeManifest();
401
- // Source name with a literal backtick (written escaped in Malloy as
402
- // `wei\`rd`). The validation target must re-escape it; a bare wrap would
403
- // produce `run: `wei`rd` -> ...`, where the embedded backtick closes the
404
- // quote early, fails to lex, and the bad view is silently skipped. Mirrors
405
- // malloy's own identifierCode escaping.
406
- fs.writeFileSync(
407
- path.join(tempDir, "backtick_render.malloy"),
408
- `source: \`wei\\\`rd\` is duckdb.sql("select 1 as a, 2 as b") extend {
409
- measure: total is a.sum()
410
- # big_value { sparkline=trend }
411
- view: card is {
412
- aggregate: total
413
- nest:
414
- # line_chart { size=spark }
415
- trend is { group_by: a; aggregate: total }
416
- }
417
- }`,
418
- );
419
-
420
- const warnSpy = spyOn(logger, "warn");
421
- const { malloyConfig, duckdb } = await makeMalloyConfig();
422
- try {
423
- const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
424
- expect(pkg.getModelPaths()).toEqual(["backtick_render.malloy"]);
425
- const warnings = warnSpy.mock.calls.map((c) => String(c[0]));
426
- expect(
427
- warnings.some(
428
- (m) =>
429
- m.includes("Invalid renderer configuration") &&
430
- m.includes("card"),
431
- ),
432
- ).toBe(true);
433
- } finally {
434
- warnSpy.mockRestore();
435
- await duckdb.close();
436
- }
437
- });
438
-
439
- it("logs a warning for a .malloynb notebook whose source view carries an invalid renderer tag", async () => {
440
- writeManifest();
441
- fs.writeFileSync(
442
- path.join(tempDir, "bad_render.malloynb"),
443
- `>>>malloy
444
- source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
445
- measure: total is a.sum()
446
- # big_value { sparkline=trend }
447
- view: card is {
448
- aggregate: total
449
- nest:
450
- # line_chart { size=spark }
451
- trend is { group_by: a; aggregate: total }
452
- }
453
- }`,
454
- );
455
-
456
- const warnSpy = spyOn(logger, "warn");
457
- const { malloyConfig, duckdb } = await makeMalloyConfig();
458
- try {
459
- const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
460
- expect(pkg.getModelPaths()).toEqual(["bad_render.malloynb"]);
461
- const warnings = warnSpy.mock.calls.map((c) => String(c[0]));
462
- expect(
463
- warnings.some((m) => m.includes("Invalid renderer configuration")),
464
- ).toBe(true);
465
- // The notebook finding also rides the package response (not just logs).
466
- const responseWarnings = pkg.getPackageMetadata().warnings ?? [];
467
- expect(
468
- responseWarnings.some(
469
- (w) =>
470
- w.model === "bad_render.malloynb" && w.severity === "error",
471
- ),
472
- ).toBe(true);
473
- } finally {
474
- warnSpy.mockRestore();
475
- await duckdb.close();
476
- }
477
- });
478
-
479
- it("re-validates renderer tags on reload, logging a warning for a model that develops a bad tag", async () => {
480
- writeManifest();
481
- // Start valid so Package.create succeeds.
482
- fs.writeFileSync(
483
- path.join(tempDir, "m.malloy"),
484
- `source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
485
- measure: total is a.sum()
486
- # bar_chart
487
- view: chart is { group_by: a; aggregate: total }
488
- }`,
489
- );
490
-
491
- const { malloyConfig, duckdb } = await makeMalloyConfig();
492
- const warnSpy = spyOn(logger, "warn");
493
- try {
494
- const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
495
- expect(pkg.getModel("m.malloy")).toBeDefined();
496
- warnSpy.mockClear();
497
-
498
- // The model now develops a misconfigured render tag. Reload re-validates
499
- // (not just Package.create) and logs a warning for it, without aborting
500
- // the reload or failing the model.
501
- fs.writeFileSync(
502
- path.join(tempDir, "m.malloy"),
503
- `source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
504
- measure: total is a.sum()
505
- # big_value { sparkline=trend }
506
- view: card is {
507
- aggregate: total
508
- nest:
509
- # line_chart { size=spark }
510
- trend is { group_by: a; aggregate: total }
511
- }
512
- }`,
513
- );
514
- await pkg.reloadAllModels({});
515
-
516
- const reloaded = pkg.getModel("m.malloy");
517
- expect(reloaded).toBeDefined();
518
- await expect(reloaded!.getModel()).resolves.toBeDefined();
519
- const warnings = warnSpy.mock.calls.map((c) => String(c[0]));
520
- expect(
521
- warnings.some((m) => m.includes("Invalid renderer configuration")),
522
- ).toBe(true);
523
- // Reload refreshes the response-level warnings too.
524
- const responseWarnings = pkg.getPackageMetadata().warnings ?? [];
525
- expect(responseWarnings.some((w) => w.model === "m.malloy")).toBe(
526
- true,
527
- );
528
- } finally {
529
- warnSpy.mockRestore();
530
- await duckdb.close();
531
- }
532
- });
533
-
534
- it("clears render-tag warnings on reload when a bad tag is fixed (bad -> clean)", async () => {
535
- writeManifest();
536
- // Start BAD so the package loads with a warning.
537
- fs.writeFileSync(
538
- path.join(tempDir, "m.malloy"),
539
- `source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
540
- measure: total is a.sum()
541
- # big_value { sparkline=trend }
542
- view: card is {
543
- aggregate: total
544
- nest:
545
- # line_chart { size=spark }
546
- trend is { group_by: a; aggregate: total }
547
- }
548
- }`,
549
- );
550
-
551
- const { malloyConfig, duckdb } = await makeMalloyConfig();
552
- try {
553
- const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
554
- expect(
555
- (pkg.getPackageMetadata().warnings ?? []).some(
556
- (w) => w.model === "m.malloy",
557
- ),
558
- ).toBe(true);
559
-
560
- // Fix the tag, then reload: renderTagWarnings is rebuilt from scratch,
561
- // so the stale finding must disappear (field omitted when empty).
562
- fs.writeFileSync(
563
- path.join(tempDir, "m.malloy"),
564
- `source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
565
- measure: total is a.sum()
566
- # bar_chart
567
- view: chart is { group_by: a; aggregate: total }
568
- }`,
569
- );
570
- await pkg.reloadAllModels({});
571
-
572
- expect(pkg.getPackageMetadata().warnings).toBeUndefined();
573
- } finally {
574
- await duckdb.close();
575
- }
576
- });
577
-
578
- it("aggregates render-tag warnings from multiple models, each tagged with its own path", async () => {
579
- writeManifest();
580
- const badModel = (view: string) =>
581
- `source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
582
- measure: total is a.sum()
583
- # big_value { sparkline=trend }
584
- view: ${view} is {
585
- aggregate: total
586
- nest:
587
- # line_chart { size=spark }
588
- trend is { group_by: a; aggregate: total }
589
- }
590
- }`;
591
- fs.writeFileSync(path.join(tempDir, "a.malloy"), badModel("card_a"));
592
- fs.writeFileSync(path.join(tempDir, "b.malloy"), badModel("card_b"));
593
-
594
- const { malloyConfig, duckdb } = await makeMalloyConfig();
595
- try {
596
- const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
597
- const responseWarnings = pkg.getPackageMetadata().warnings ?? [];
598
- const models = new Set(responseWarnings.map((w) => w.model));
599
- expect(models.has("a.malloy")).toBe(true);
600
- expect(models.has("b.malloy")).toBe(true);
601
- expect(models.size).toBe(2);
602
- // The message field is carried through, not just model/target/severity.
603
- expect(
604
- responseWarnings.every((w) => (w.message ?? "").length > 0),
605
- ).toBe(true);
606
- } finally {
607
- await duckdb.close();
608
- }
609
- });
610
-
611
- // NB: kept last in this describe — swapping the singleton for a
612
- // pre-shutdown pool also tears down the shared `pool` (the swap
613
- // implementation shuts down the outgoing singleton). Subsequent
614
- // tests in this describe would see a dead pool. afterAll only
615
- // resets the singleton to null, so this is safe at the tail.
616
- it("rewraps pool-infrastructure failures as ServiceUnavailableError (HTTP 503)", async () => {
617
- writeManifest();
618
- fs.writeFileSync(
619
- path.join(tempDir, "trivial.malloy"),
620
- `source: nums is duckdb.sql("select 1 as a")`,
621
- );
622
-
623
- const deadPool = new PackageLoadPool(1);
624
- await deadPool.shutdown();
625
- await __setPackageLoadPoolForTests(deadPool);
626
-
627
- const { ServiceUnavailableError } = await import("../errors");
628
- const { malloyConfig, duckdb } = await makeMalloyConfig();
629
- try {
630
- await expect(
631
- Package.create("env", "pkg", tempDir, malloyConfig),
632
- ).rejects.toBeInstanceOf(ServiceUnavailableError);
633
- } finally {
634
- await duckdb.close();
635
- // Replace the singleton with a fresh, live pool immediately. The
636
- // outer afterAll also resets to null, but restoring here ensures the
637
- // shut-down pool never outlives this test -- otherwise, under serial
638
- // execution, a later spec file that lazily reuses the singleton via
639
- // getPackageLoadPool() could observe the dead pool before afterAll
640
- // runs and fail with "PackageLoadPool is shutting down".
641
- pool = new PackageLoadPool(1);
642
- await __setPackageLoadPoolForTests(pool);
643
- }
644
- });
645
- });
@@ -1,39 +0,0 @@
1
- import { describe, expect, it } from "bun:test";
2
- import { BadRequestError } from "../errors";
3
- import { deleteDuckLakeConnectionFile } from "./connection";
4
-
5
- const TRAVERSAL_NAMES: ReadonlyArray<readonly [string, string]> = [
6
- ["leading traversal", "../etc"],
7
- ["embedded traversal", "foo/../../bar"],
8
- ["slash in name", "foo/bar"],
9
- ["backslash in name", "foo\\bar"],
10
- ["leading dot", ".staging"],
11
- ["bare dot-dot", ".."],
12
- ["bare dot", "."],
13
- ["empty", ""],
14
- ["NUL byte", "foo\0bar"],
15
- ["oversized", "a".repeat(256)],
16
- ["absolute", "/etc/passwd"],
17
- ] as const;
18
-
19
- describe("deleteDuckLakeConnectionFile path-injection guards", () => {
20
- it.each(TRAVERSAL_NAMES)(
21
- "rejects %s as connectionName (%p)",
22
- async (_label, connectionName) => {
23
- await expect(
24
- deleteDuckLakeConnectionFile(connectionName, "/tmp/env"),
25
- ).rejects.toBeInstanceOf(BadRequestError);
26
- },
27
- );
28
-
29
- it.each([
30
- ["relative", "relative/path"],
31
- ["traversal", "/var/lib/../../etc"],
32
- ["NUL byte", "/var/lib/env\0"],
33
- ["bare dot-dot", ".."],
34
- ])("rejects %s as environmentPath (%p)", async (_label, environmentPath) => {
35
- await expect(
36
- deleteDuckLakeConnectionFile("conn", environmentPath),
37
- ).rejects.toBeInstanceOf(BadRequestError);
38
- });
39
- });