@malloy-publisher/server 0.0.235 → 0.0.236

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (301) hide show
  1. package/dist/app/api-doc.yaml +47 -17
  2. package/dist/package_load_worker.mjs +8 -1
  3. package/dist/server.mjs +471 -54
  4. package/package.json +5 -2
  5. package/.eslintrc.json +0 -44
  6. package/.prettierignore +0 -1
  7. package/.prettierrc +0 -4
  8. package/build.ts +0 -97
  9. package/eslint.config.mjs +0 -76
  10. package/k6-tests/README.md +0 -144
  11. package/k6-tests/bun.lock +0 -797
  12. package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
  13. package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
  14. package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
  15. package/k6-tests/load-test/load-test-crud.ts +0 -142
  16. package/k6-tests/load-test/load-test.ts +0 -784
  17. package/k6-tests/package.json +0 -27
  18. package/k6-tests/scripts/run-with-otel.sh +0 -50
  19. package/k6-tests/smoke-test/smoke-test.ts +0 -158
  20. package/k6-tests/tsconfig.json +0 -29
  21. package/k6-tests/utils/client_factory.ts +0 -128
  22. package/k6-tests/utils/common.ts +0 -489
  23. package/k6-tests/utils/logger.ts +0 -62
  24. package/k6-tests/utils/types.d.ts +0 -35
  25. package/malloy_mcp.dxt +0 -0
  26. package/publisher.config.example.bigquery.json +0 -33
  27. package/publisher.config.example.duckdb.json +0 -23
  28. package/publisher.config.json +0 -23
  29. package/scripts/bake-duckdb-extensions.js +0 -110
  30. package/src/config.spec.ts +0 -1535
  31. package/src/config.theme.spec.ts +0 -178
  32. package/src/config.ts +0 -1142
  33. package/src/constants.ts +0 -105
  34. package/src/controller/compile.controller.ts +0 -40
  35. package/src/controller/connection.controller.spec.ts +0 -849
  36. package/src/controller/connection.controller.ts +0 -901
  37. package/src/controller/database.controller.ts +0 -24
  38. package/src/controller/materialization.controller.spec.ts +0 -293
  39. package/src/controller/materialization.controller.ts +0 -319
  40. package/src/controller/model.controller.ts +0 -164
  41. package/src/controller/package.controller.spec.ts +0 -273
  42. package/src/controller/package.controller.ts +0 -306
  43. package/src/controller/query.controller.ts +0 -152
  44. package/src/controller/theme.controller.ts +0 -83
  45. package/src/controller/watch-mode.controller.ts +0 -243
  46. package/src/data_styles.ts +0 -71
  47. package/src/default-publisher.config.json +0 -23
  48. package/src/ducklake_version.spec.ts +0 -163
  49. package/src/ducklake_version.ts +0 -153
  50. package/src/errors.spec.ts +0 -119
  51. package/src/errors.ts +0 -237
  52. package/src/filter_deprecation.spec.ts +0 -64
  53. package/src/filter_deprecation.ts +0 -42
  54. package/src/health.spec.ts +0 -123
  55. package/src/health.ts +0 -233
  56. package/src/heap_check.spec.ts +0 -144
  57. package/src/heap_check.ts +0 -144
  58. package/src/instrumentation.ts +0 -214
  59. package/src/json_utils.spec.ts +0 -51
  60. package/src/json_utils.ts +0 -33
  61. package/src/logger.spec.ts +0 -210
  62. package/src/logger.ts +0 -251
  63. package/src/malloy_pin_prereqs.spec.ts +0 -25
  64. package/src/materialization_metrics.spec.ts +0 -169
  65. package/src/materialization_metrics.ts +0 -346
  66. package/src/mcp/error_messages.spec.ts +0 -35
  67. package/src/mcp/error_messages.ts +0 -183
  68. package/src/mcp/handler_utils.spec.ts +0 -108
  69. package/src/mcp/handler_utils.ts +0 -251
  70. package/src/mcp/mcp_constants.ts +0 -6
  71. package/src/mcp/query_envelope.spec.ts +0 -229
  72. package/src/mcp/query_envelope.ts +0 -240
  73. package/src/mcp/server.protocol.spec.ts +0 -250
  74. package/src/mcp/server.ts +0 -87
  75. package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
  76. package/src/mcp/skills/build_skills_bundle.ts +0 -176
  77. package/src/mcp/skills/skills_bundle.json +0 -1
  78. package/src/mcp/skills/skills_bundle.spec.ts +0 -171
  79. package/src/mcp/tool_response.spec.ts +0 -108
  80. package/src/mcp/tool_response.ts +0 -138
  81. package/src/mcp/tools/compile_tool.spec.ts +0 -315
  82. package/src/mcp/tools/compile_tool.ts +0 -208
  83. package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
  84. package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
  85. package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
  86. package/src/mcp/tools/docs_search_tool.ts +0 -128
  87. package/src/mcp/tools/embedding_index.spec.ts +0 -1236
  88. package/src/mcp/tools/embedding_index.ts +0 -808
  89. package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
  90. package/src/mcp/tools/execute_query_tool.ts +0 -316
  91. package/src/mcp/tools/get_context_eval.ts +0 -275
  92. package/src/mcp/tools/get_context_tool.spec.ts +0 -592
  93. package/src/mcp/tools/get_context_tool.ts +0 -585
  94. package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
  95. package/src/mcp/tools/reload_package_tool.ts +0 -132
  96. package/src/mcp_config.spec.ts +0 -919
  97. package/src/mcp_config.ts +0 -425
  98. package/src/oom_guards.integration.spec.ts +0 -269
  99. package/src/package_load/package_load_pool.spec.ts +0 -252
  100. package/src/package_load/package_load_pool.ts +0 -931
  101. package/src/package_load/package_load_worker.ts +0 -1034
  102. package/src/package_load/protocol.ts +0 -364
  103. package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
  104. package/src/package_load/rpc_wait_accountant.ts +0 -76
  105. package/src/package_load_metrics.spec.ts +0 -114
  106. package/src/package_load_metrics.ts +0 -127
  107. package/src/path_safety.spec.ts +0 -158
  108. package/src/path_safety.ts +0 -146
  109. package/src/pg_helpers.spec.ts +0 -223
  110. package/src/pg_helpers.ts +0 -52
  111. package/src/query_cap_metrics.spec.ts +0 -89
  112. package/src/query_cap_metrics.ts +0 -116
  113. package/src/query_concurrency.spec.ts +0 -247
  114. package/src/query_concurrency.ts +0 -237
  115. package/src/query_metadata_metrics.ts +0 -49
  116. package/src/query_param_utils.ts +0 -29
  117. package/src/query_timeout.spec.ts +0 -224
  118. package/src/query_timeout.ts +0 -179
  119. package/src/runtime/publisher.js +0 -328
  120. package/src/server-old.ts +0 -988
  121. package/src/server.ts +0 -2059
  122. package/src/service/annotations.spec.ts +0 -118
  123. package/src/service/annotations.ts +0 -91
  124. package/src/service/authorize.spec.ts +0 -154
  125. package/src/service/authorize.ts +0 -498
  126. package/src/service/authorize_integration.spec.ts +0 -2041
  127. package/src/service/build_plan.spec.ts +0 -685
  128. package/src/service/build_plan.ts +0 -797
  129. package/src/service/compile_authorize.spec.ts +0 -85
  130. package/src/service/compile_fragment_techniques.spec.ts +0 -156
  131. package/src/service/connection.spec.ts +0 -2360
  132. package/src/service/connection.ts +0 -2187
  133. package/src/service/connection_config.spec.ts +0 -734
  134. package/src/service/connection_config.ts +0 -899
  135. package/src/service/connection_federation.spec.ts +0 -184
  136. package/src/service/connection_fingerprint.spec.ts +0 -102
  137. package/src/service/connection_service.spec.ts +0 -711
  138. package/src/service/connection_service.ts +0 -291
  139. package/src/service/cron_evaluator.spec.ts +0 -79
  140. package/src/service/cron_evaluator.ts +0 -105
  141. package/src/service/db_utils.spec.ts +0 -1085
  142. package/src/service/db_utils.ts +0 -1502
  143. package/src/service/duckdb_instance_isolation.spec.ts +0 -137
  144. package/src/service/ducklake_lazy_attach.spec.ts +0 -110
  145. package/src/service/embedding_provider.spec.ts +0 -329
  146. package/src/service/embedding_provider.ts +0 -236
  147. package/src/service/environment.ts +0 -2191
  148. package/src/service/environment_admission.spec.ts +0 -344
  149. package/src/service/environment_compile.spec.ts +0 -197
  150. package/src/service/environment_store.spec.ts +0 -2274
  151. package/src/service/environment_store.ts +0 -2434
  152. package/src/service/environment_store_anchoring.spec.ts +0 -107
  153. package/src/service/environment_store_clone.spec.ts +0 -350
  154. package/src/service/explore_visibility.spec.ts +0 -434
  155. package/src/service/exports_probe.spec.ts +0 -107
  156. package/src/service/extension_fetch_policy.spec.ts +0 -256
  157. package/src/service/filter.spec.ts +0 -458
  158. package/src/service/filter.ts +0 -341
  159. package/src/service/filter_bypass.spec.ts +0 -418
  160. package/src/service/filter_integration.spec.ts +0 -1004
  161. package/src/service/freshness.spec.ts +0 -183
  162. package/src/service/freshness.ts +0 -112
  163. package/src/service/gcs_s3_utils.ts +0 -427
  164. package/src/service/given.ts +0 -105
  165. package/src/service/givens_integration.spec.ts +0 -219
  166. package/src/service/manifest_loader.spec.ts +0 -189
  167. package/src/service/manifest_loader.ts +0 -156
  168. package/src/service/materialization_build_session.spec.ts +0 -435
  169. package/src/service/materialization_build_session.ts +0 -681
  170. package/src/service/materialization_config_validation.spec.ts +0 -99
  171. package/src/service/materialization_config_validation.ts +0 -120
  172. package/src/service/materialization_eligibility.spec.ts +0 -158
  173. package/src/service/materialization_eligibility.ts +0 -305
  174. package/src/service/materialization_schedule_surface.spec.ts +0 -256
  175. package/src/service/materialization_scheduler.spec.ts +0 -289
  176. package/src/service/materialization_scheduler.ts +0 -304
  177. package/src/service/materialization_serve_transform.spec.ts +0 -1003
  178. package/src/service/materialization_serve_transform.ts +0 -779
  179. package/src/service/materialization_service.spec.ts +0 -2373
  180. package/src/service/materialization_service.ts +0 -2342
  181. package/src/service/materialization_test_fixtures.ts +0 -249
  182. package/src/service/model.spec.ts +0 -1157
  183. package/src/service/model.ts +0 -3370
  184. package/src/service/model_limits.spec.ts +0 -209
  185. package/src/service/model_limits.ts +0 -131
  186. package/src/service/model_storage_serve.spec.ts +0 -193
  187. package/src/service/model_storage_serve_joins.spec.ts +0 -193
  188. package/src/service/package.spec.ts +0 -612
  189. package/src/service/package.ts +0 -1799
  190. package/src/service/package_manifest.spec.ts +0 -230
  191. package/src/service/package_manifest.ts +0 -241
  192. package/src/service/package_memory_governor.spec.ts +0 -173
  193. package/src/service/package_memory_governor.ts +0 -233
  194. package/src/service/package_quote_bound_tables.spec.ts +0 -171
  195. package/src/service/package_race.spec.ts +0 -208
  196. package/src/service/package_reload_safety.spec.ts +0 -193
  197. package/src/service/package_rollback.spec.ts +0 -190
  198. package/src/service/package_worker_path.spec.ts +0 -645
  199. package/src/service/path_injection.spec.ts +0 -39
  200. package/src/service/persist_annotation_validation.spec.ts +0 -89
  201. package/src/service/persist_annotation_validation.ts +0 -59
  202. package/src/service/persistence_policy.spec.ts +0 -374
  203. package/src/service/proxy.spec.ts +0 -414
  204. package/src/service/proxy.ts +0 -248
  205. package/src/service/query_boundary.spec.ts +0 -470
  206. package/src/service/query_metadata.spec.ts +0 -408
  207. package/src/service/query_metadata.ts +0 -492
  208. package/src/service/query_metadata_identity.spec.ts +0 -149
  209. package/src/service/query_text.spec.ts +0 -79
  210. package/src/service/query_text.ts +0 -43
  211. package/src/service/quoting.spec.ts +0 -135
  212. package/src/service/quoting.ts +0 -78
  213. package/src/service/resolve_environment.ts +0 -15
  214. package/src/service/restricted_mode.spec.ts +0 -299
  215. package/src/service/source_extraction.ts +0 -225
  216. package/src/service/theme_store.ts +0 -199
  217. package/src/storage/DatabaseInterface.ts +0 -198
  218. package/src/storage/StorageManager.ts +0 -113
  219. package/src/storage/duckdb/ConnectionRepository.ts +0 -143
  220. package/src/storage/duckdb/DuckDBConnection.ts +0 -123
  221. package/src/storage/duckdb/DuckDBRepository.ts +0 -208
  222. package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
  223. package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
  224. package/src/storage/duckdb/MaterializationRepository.ts +0 -336
  225. package/src/storage/duckdb/PackageRepository.ts +0 -135
  226. package/src/storage/duckdb/schema.ts +0 -229
  227. package/src/stream_helpers.spec.ts +0 -280
  228. package/src/stream_helpers.ts +0 -162
  229. package/src/telemetry.ts +0 -20
  230. package/src/test_helpers/metrics_harness.ts +0 -166
  231. package/src/theme_key_parity.spec.ts +0 -57
  232. package/src/utils.ts +0 -29
  233. package/tests/fixtures/authorize-compile/model.malloy +0 -9
  234. package/tests/fixtures/authorize-compile/publisher.json +0 -4
  235. package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
  236. package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
  237. package/tests/fixtures/html-pages-test/data.csv +0 -3
  238. package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
  239. package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
  240. package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
  241. package/tests/fixtures/html-pages-test/public/data.json +0 -1
  242. package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
  243. package/tests/fixtures/html-pages-test/public/index.html +0 -9
  244. package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
  245. package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
  246. package/tests/fixtures/html-pages-test/public/slides.html +0 -12
  247. package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
  248. package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
  249. package/tests/fixtures/html-pages-test/publisher.json +0 -5
  250. package/tests/fixtures/html-pages-test/report.malloy +0 -1
  251. package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
  252. package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
  253. package/tests/fixtures/persist-multi-level/publisher.json +0 -5
  254. package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
  255. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
  256. package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
  257. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
  258. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
  259. package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
  260. package/tests/fixtures/persist-test/data/orders.csv +0 -5
  261. package/tests/fixtures/persist-test/persist_test.malloy +0 -11
  262. package/tests/fixtures/persist-test/publisher.json +0 -5
  263. package/tests/fixtures/publisher.config.json +0 -15
  264. package/tests/fixtures/query-givens/data/orders.csv +0 -7
  265. package/tests/fixtures/query-givens/model.malloy +0 -34
  266. package/tests/fixtures/query-givens/publisher.json +0 -5
  267. package/tests/fixtures/xlsx/database.xlsx +0 -0
  268. package/tests/harness/mcp_test_setup.ts +0 -332
  269. package/tests/harness/rest_e2e.ts +0 -68
  270. package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
  271. package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
  272. package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
  273. package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
  274. package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
  275. package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
  276. package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
  277. package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
  278. package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
  279. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
  280. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
  281. package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
  282. package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
  283. package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
  284. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
  285. package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
  286. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
  287. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
  288. package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
  289. package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
  290. package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
  291. package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
  292. package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
  293. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
  294. package/tests/unit/controller/theme.controller.spec.ts +0 -141
  295. package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
  296. package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
  297. package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
  298. package/tests/unit/duckdb/repositories.test.ts +0 -208
  299. package/tests/unit/ducklake/ducklake.test.ts +0 -952
  300. package/tests/unit/service/theme_store.spec.ts +0 -139
  301. package/tsconfig.json +0 -32
@@ -1,243 +0,0 @@
1
- import chokidar, { FSWatcher } from "chokidar";
2
- import { EventEmitter } from "events";
3
- import { RequestHandler } from "express";
4
- import path from "path";
5
- import { components } from "../api";
6
- import { internalErrorToHttpError } from "../errors";
7
- import { logger } from "../logger";
8
- import { assertSafePackageName } from "../path_safety";
9
- import { EnvironmentStore } from "../service/environment_store";
10
-
11
- type StartWatchReq = components["schemas"]["StartWatchRequest"];
12
- type WatchStatusRes = components["schemas"]["WatchStatus"];
13
- type Handler<Req = object, Res = void> = RequestHandler<object, Res, Req>;
14
-
15
- // File extensions that should trigger a live-reload event but NOT a Malloy
16
- // environment reload (these are package assets, not semantic-model sources).
17
- const ASSET_EXTS = new Set([
18
- ".html",
19
- ".htm",
20
- ".css",
21
- ".js",
22
- ".mjs",
23
- ".json",
24
- ".png",
25
- ".jpg",
26
- ".jpeg",
27
- ".gif",
28
- ".svg",
29
- ".webp",
30
- ".ico",
31
- ".woff",
32
- ".woff2",
33
- ]);
34
- const MODEL_EXTS = new Set([".malloy", ".malloynb", ".md"]);
35
-
36
- export class WatchModeController {
37
- watchingPath: string | null;
38
- watchingEnvironmentName: string | null;
39
- watcher: FSWatcher | null = null;
40
- // Serializes ensureWatching so concurrent callers can't both close and
41
- // recreate the watcher and orphan one (each chokidar watcher holds OS
42
- // handles). See ensureWatching.
43
- private setupChain: Promise<void> | null = null;
44
- /**
45
- * Per-package change bus. Event name is `<environmentName>/<packageName>`.
46
- * Used by the SSE live-reload endpoint to push refreshes to embedded HTML
47
- * dashboards. Each emit carries `{ path, kind }` for diagnostics; the SSE
48
- * handler currently ignores the payload and just sends "changed".
49
- */
50
- public events = new EventEmitter();
51
-
52
- constructor(private environmentStore: EnvironmentStore) {
53
- this.watchingPath = null;
54
- this.watchingEnvironmentName = null;
55
- // Live-reload subscribers can be many (one per open browser tab);
56
- // bump the default cap so Node doesn't warn under normal use.
57
- this.events.setMaxListeners(100);
58
- }
59
-
60
- /**
61
- * Idempotent: starts watching `environmentName` if not already.
62
- *
63
- * Throws if the environment can't be resolved — never falls back to a
64
- * `<serverRoot>/<envName>` path that an attacker could control via
65
- * URL-encoded path traversal. Callers (the SSE handler in particular)
66
- * must validate the env name before calling this.
67
- */
68
- public async ensureWatching(environmentName: string): Promise<void> {
69
- if (this.watchingEnvironmentName === environmentName && this.watcher) {
70
- return;
71
- }
72
- // Serialize setup behind any in-flight one, then re-check the guard, so
73
- // concurrent callers don't both close + recreate the watcher (which would
74
- // orphan a chokidar instance and leak its OS file handles).
75
- const run = (this.setupChain ?? Promise.resolve())
76
- .catch(() => {})
77
- .then(async () => {
78
- if (
79
- this.watchingEnvironmentName === environmentName &&
80
- this.watcher
81
- ) {
82
- return;
83
- }
84
- const env = await this.environmentStore.getEnvironment(
85
- environmentName,
86
- false,
87
- );
88
- const watchPath = env.getEnvironmentPath();
89
- if (this.watcher) {
90
- await this.watcher.close();
91
- this.watcher = null;
92
- }
93
- this.startWatcher(environmentName, watchPath);
94
- });
95
- this.setupChain = run;
96
- await run;
97
- }
98
-
99
- /** Whether watch mode is currently running for the given env. */
100
- public isWatching(environmentName: string): boolean {
101
- return !!this.watcher && this.watchingEnvironmentName === environmentName;
102
- }
103
-
104
- private startWatcher(watchName: string, watchPath: string) {
105
- this.watchingEnvironmentName = watchName;
106
- this.watchingPath = watchPath;
107
- this.watcher = chokidar.watch(this.watchingPath, {
108
- ignored: (filePath, stats) => {
109
- if (!stats?.isFile()) return false;
110
- const ext = path.extname(filePath).toLowerCase();
111
- return !MODEL_EXTS.has(ext) && !ASSET_EXTS.has(ext);
112
- },
113
- ignoreInitial: true,
114
- });
115
- // Reload just the package a changed file belongs to: this recompiles
116
- // that package's models from disk (Package.create) and replaces the
117
- // cached entry, so the next query sees the edit. Scoped to watch mode by
118
- // construction — it only runs from this watcher, which is started only
119
- // when watch mode is active.
120
- //
121
- // The previous env-level reload (getEnvironment(reload) + addEnvironment)
122
- // was a no-op for compilation: addEnvironment on an existing env calls
123
- // Environment.update(), which refreshes metadata/connections only and
124
- // never touches this.packages, so edits never took effect.
125
- // Returns true if the package recompiled cleanly. A transient compile
126
- // error (e.g. a half-typed model saved mid-edit) returns false so we can
127
- // avoid bouncing open browser tabs into a compile-error/404 — see onEvent.
128
- const reloadPackage = async (pkgName: string): Promise<boolean> => {
129
- try {
130
- const environment = await this.environmentStore.getEnvironment(
131
- watchName,
132
- false,
133
- );
134
- await environment.getPackage(pkgName, true);
135
- logger.info(
136
- `Watch: recompiled package "${pkgName}" in environment "${watchName}"`,
137
- );
138
- return true;
139
- } catch (error) {
140
- logger.error(
141
- `Watch: failed to recompile package "${pkgName}" in environment "${watchName}"`,
142
- { error },
143
- );
144
- return false;
145
- }
146
- };
147
- const onEvent =
148
- (kind: "add" | "change" | "unlink") => async (filePath: string) => {
149
- logger.info(`Watch ${kind}: ${filePath}; environment=${watchName}`);
150
- // Resolve which package the file belongs to. Packages are
151
- // subdirectories of the environment (env/<pkg>/...), so a file must
152
- // be nested at least one level to belong to a package; files at the
153
- // environment root belong to no package and are ignored.
154
- const rel = path.relative(this.watchingPath ?? "", filePath);
155
- const segments = rel.split(path.sep);
156
- const pkgName =
157
- segments.length > 1 &&
158
- segments[0] &&
159
- !segments[0].startsWith("..")
160
- ? segments[0]
161
- : null;
162
- if (!pkgName) return;
163
-
164
- // Recompile Malloy state only for model files. Asset edits
165
- // (HTML/CSS/JS) skip recompile — they just need the live-reload
166
- // fanout below. For model edits, only signal a reload once the
167
- // recompile succeeds: a transient syntax error shouldn't bounce
168
- // open pages into a compile error or 404.
169
- const ext = path.extname(filePath).toLowerCase();
170
- if (MODEL_EXTS.has(ext)) {
171
- const recompiled = await reloadPackage(pkgName);
172
- if (!recompiled) return;
173
- }
174
- // Fan out to SSE clients embedded in the affected package.
175
- this.events.emit(`${watchName}/${pkgName}`, {
176
- path: filePath,
177
- kind,
178
- });
179
- };
180
- this.watcher.on("add", onEvent("add"));
181
- this.watcher.on("change", onEvent("change"));
182
- this.watcher.on("unlink", onEvent("unlink"));
183
- }
184
-
185
- public getWatchStatus: Handler<void, WatchStatusRes> = async (_req, res) => {
186
- return res.json({
187
- enabled: !!this.watchingPath,
188
- watchingPath: this.watchingPath ?? "",
189
- environmentName: this.watchingEnvironmentName ?? "",
190
- });
191
- };
192
-
193
- public startWatching: Handler<StartWatchReq, { error: string }> = async (
194
- req,
195
- res,
196
- ) => {
197
- const watchName = req.body.environmentName ?? "";
198
- try {
199
- assertSafePackageName(watchName);
200
- } catch (error) {
201
- logger.error(error);
202
- const { status } = internalErrorToHttpError(error as Error);
203
- res.status(status).json({ error: (error as Error).message });
204
- return;
205
- }
206
- const environmentManifest =
207
- await EnvironmentStore.reloadEnvironmentManifest(
208
- this.environmentStore.serverRootPath,
209
- );
210
- const environment = environmentManifest.environments.find(
211
- (e) => e.name === watchName,
212
- );
213
- if (
214
- !environment ||
215
- !environment.packages ||
216
- environment.packages.length === 0
217
- ) {
218
- res.status(404).json({
219
- error: `Environment ${watchName} not found or has no packages`,
220
- });
221
- return;
222
- }
223
- try {
224
- await this.ensureWatching(watchName);
225
- } catch (error) {
226
- logger.error(error);
227
- const { status } = internalErrorToHttpError(error as Error);
228
- res.status(status).json({ error: (error as Error).message });
229
- return;
230
- }
231
- res.json();
232
- };
233
-
234
- public stopWatchMode: Handler = async (_req, res) => {
235
- if (this.watcher) {
236
- await this.watcher.close();
237
- this.watcher = null;
238
- }
239
- this.watchingPath = null;
240
- this.watchingEnvironmentName = null;
241
- res.json();
242
- };
243
- }
@@ -1,71 +0,0 @@
1
- import { URLReader } from "@malloydata/malloy";
2
- import { DataStyles } from "@malloydata/render";
3
- import { logger } from "./logger";
4
-
5
- export function compileDataStyles(styles: string): DataStyles {
6
- try {
7
- return JSON.parse(styles) as DataStyles;
8
- } catch (error) {
9
- throw new Error(`Error compiling data styles: ${error}`);
10
- }
11
- }
12
-
13
- // TODO replace this with actual JSON metadata import functionality, when it exists
14
- export async function dataStylesForFile(
15
- url: string,
16
- text: string,
17
- urlReader: URLReader,
18
- ): Promise<DataStyles> {
19
- const PREFIX = "--! styles ";
20
- let styles: DataStyles = {};
21
- for (const line of text.split("\n")) {
22
- if (line.startsWith(PREFIX)) {
23
- const fileName = line.trimEnd().substring(PREFIX.length);
24
- // TODO instead of failing silently when the file does not exist, perform this after the WebView has been
25
- // created, so that the error can be shown there.
26
- let stylesText;
27
- try {
28
- stylesText = (await urlReader.readURL(
29
- new URL(fileName, url),
30
- )) as string;
31
- } catch (error) {
32
- logger.error(`Error loading data style '${fileName}': ${error}`);
33
- stylesText = "{}";
34
- }
35
- styles = { ...styles, ...compileDataStyles(stylesText) };
36
- }
37
- }
38
-
39
- return styles;
40
- }
41
-
42
- // TODO Come up with a better way to handle data styles. Perhaps this is
43
- // an in-language understanding of model "metadata". For now,
44
- // we abuse the `URLReader` API to keep track of requested URLs
45
- // and accumulate data styles for those files.
46
- export class HackyDataStylesAccumulator implements URLReader {
47
- private urlReader: URLReader;
48
- private dataStyles: DataStyles = {};
49
-
50
- constructor(urlReader: URLReader) {
51
- this.urlReader = urlReader;
52
- }
53
-
54
- async readURL(url: URL): Promise<string> {
55
- const contents = await this.urlReader.readURL(url);
56
- this.dataStyles = {
57
- ...this.dataStyles,
58
- ...(await dataStylesForFile(
59
- url.toString(),
60
- contents as string,
61
- this.urlReader,
62
- )),
63
- };
64
-
65
- return contents as string;
66
- }
67
-
68
- getHackyAccumulatedDataStyles(): DataStyles {
69
- return this.dataStyles;
70
- }
71
- }
@@ -1,23 +0,0 @@
1
- {
2
- "frozenConfig": false,
3
- "environments": [
4
- {
5
- "name": "examples",
6
- "packages": [
7
- {
8
- "name": "storefront",
9
- "location": "https://github.com/malloydata/publisher/tree/main/examples/storefront"
10
- },
11
- {
12
- "name": "governed-analytics",
13
- "location": "https://github.com/malloydata/publisher/tree/main/examples/governed-analytics"
14
- },
15
- {
16
- "name": "html-data-app",
17
- "location": "https://github.com/malloydata/publisher/tree/main/examples/html-data-app"
18
- }
19
- ],
20
- "connections": []
21
- }
22
- ]
23
- }
@@ -1,163 +0,0 @@
1
- import { describe, expect, it } from "bun:test";
2
- import { readFileSync } from "fs";
3
- import { fileURLToPath } from "url";
4
- import {
5
- catalogFormatRangeForEngine,
6
- compareCatalogFormat,
7
- ENGINE_MAX_FORMAT,
8
- isCatalogFormatInRange,
9
- isCatalogFormatSupportedByEngine,
10
- MIN_CATALOG_FORMAT,
11
- parseCatalogFormat,
12
- parseEngineMajorMinor,
13
- } from "./ducklake_version";
14
-
15
- /**
16
- * Resolve the pinned `@duckdb/node-api` engine version from bun.lock, the same
17
- * source `scripts/duckdb-version.js` reads (mirrored here rather than imported
18
- * because the server tsconfig has `allowJs: false`). Prefers the version nested
19
- * under `@malloydata/db-duckdb` — the engine Malloy actually runs on — and
20
- * falls back to the deduped top-level entry.
21
- */
22
- function resolvePinnedDuckDBVersion(): string {
23
- const lock = readFileSync(
24
- fileURLToPath(new URL("../../../bun.lock", import.meta.url)),
25
- "utf8",
26
- );
27
- const nested = lock.match(
28
- /"@malloydata\/db-duckdb\/@duckdb\/node-api":\s*\["@duckdb\/node-api@(\d+\.\d+\.\d+[^"]*)"/,
29
- );
30
- const plain = lock.match(
31
- /"@duckdb\/node-api":\s*\["@duckdb\/node-api@(\d+\.\d+\.\d+[^"]*)"/,
32
- );
33
- const match = nested ?? plain;
34
- if (!match) throw new Error("@duckdb/node-api not found in bun.lock");
35
- return match[1];
36
- }
37
-
38
- describe("ducklake_version", () => {
39
- describe("parseCatalogFormat", () => {
40
- it("parses two-part and pre-release formats", () => {
41
- expect(parseCatalogFormat("1.0")).toEqual({ major: 1, minor: 0 });
42
- expect(parseCatalogFormat("0.3-dev1")).toEqual({
43
- major: 0,
44
- minor: 3,
45
- prerelease: "dev1",
46
- });
47
- });
48
-
49
- it("rejects malformed input without throwing", () => {
50
- // Three-part strings are malformed for a catalog format, not a newer
51
- // version -- they must not parse as a bigger number.
52
- expect(parseCatalogFormat("1.0.0")).toBeNull();
53
- expect(parseCatalogFormat("1.1.0")).toBeNull();
54
- expect(parseCatalogFormat("")).toBeNull();
55
- expect(parseCatalogFormat("not a version")).toBeNull();
56
- expect(parseCatalogFormat("0.3 ")).toBeNull();
57
- });
58
- });
59
-
60
- describe("compareCatalogFormat", () => {
61
- const p = (s: string) => parseCatalogFormat(s)!;
62
- it("orders by major, then minor", () => {
63
- expect(compareCatalogFormat(p("0.3"), p("1.0"))).toBe(-1);
64
- expect(compareCatalogFormat(p("1.1"), p("1.0"))).toBe(1);
65
- expect(compareCatalogFormat(p("1.0"), p("1.0"))).toBe(0);
66
- });
67
- it("sorts a release above its own pre-releases", () => {
68
- expect(compareCatalogFormat(p("1.0"), p("1.0-dev1"))).toBe(1);
69
- expect(compareCatalogFormat(p("0.3-dev1"), p("0.3"))).toBe(-1);
70
- });
71
- });
72
-
73
- describe("parseEngineMajorMinor", () => {
74
- it("extracts the minor line from assorted engine spec forms", () => {
75
- expect(parseEngineMajorMinor("1.5.3")).toEqual({ major: 1, minor: 5 });
76
- expect(parseEngineMajorMinor("1.5.3-r.2")).toEqual({
77
- major: 1,
78
- minor: 5,
79
- });
80
- expect(parseEngineMajorMinor("v1.5.3")).toEqual({
81
- major: 1,
82
- minor: 5,
83
- });
84
- });
85
- it("returns null when no version is present", () => {
86
- expect(parseEngineMajorMinor("nope")).toBeNull();
87
- });
88
- });
89
-
90
- describe("catalogFormatRangeForEngine", () => {
91
- it("derives [1.0, maxFormat] for a covered engine (all patch levels)", () => {
92
- expect(catalogFormatRangeForEngine("1.5.3")).toEqual({
93
- min: "1.0",
94
- max: "1.0",
95
- engineVersion: "1.5",
96
- });
97
- // Any patch of a known minor line resolves to the same range.
98
- expect(catalogFormatRangeForEngine("1.5.9")).toEqual({
99
- min: "1.0",
100
- max: "1.0",
101
- engineVersion: "1.5",
102
- });
103
- });
104
- it("returns null (drift signal) for an engine with no matrix row", () => {
105
- expect(catalogFormatRangeForEngine("1.6.0")).toBeNull();
106
- expect(catalogFormatRangeForEngine("2.0.0")).toBeNull();
107
- expect(catalogFormatRangeForEngine("garbage")).toBeNull();
108
- });
109
- });
110
-
111
- // These reproduce the accept/reject set the prior enumerated-list work
112
- // asserted, now expressed as a RANGE derived from the 1.5.x engine.
113
- describe("isCatalogFormatSupportedByEngine (engine 1.5.x)", () => {
114
- const engine = "1.5.3";
115
- it("accepts the 1.0 catalog format produced by the 1.5.x engine", () => {
116
- expect(isCatalogFormatSupportedByEngine("1.0", engine)).toBe(true);
117
- });
118
- it("rejects the 0.3-line formats (attach only via migration)", () => {
119
- for (const v of ["0.1", "0.2", "0.3-dev1", "0.3"]) {
120
- expect(isCatalogFormatSupportedByEngine(v, engine)).toBe(false);
121
- }
122
- });
123
- it("rejects formats newer than the supported max", () => {
124
- for (const v of ["1.1", "2.0", "0.4", "0.3-dev2"]) {
125
- expect(isCatalogFormatSupportedByEngine(v, engine)).toBe(false);
126
- }
127
- });
128
- it("rejects malformed / three-part input without throwing", () => {
129
- for (const v of ["1.0.0", "1.1.0", "", "not a version", "0.3 "]) {
130
- expect(isCatalogFormatSupportedByEngine(v, engine)).toBe(false);
131
- }
132
- });
133
- it("rejects everything when the engine is unknown (no matrix row)", () => {
134
- expect(isCatalogFormatSupportedByEngine("1.0", "1.6.0")).toBe(false);
135
- });
136
- });
137
-
138
- describe("isCatalogFormatInRange", () => {
139
- const range = catalogFormatRangeForEngine("1.5.3")!;
140
- it("is inclusive at both ends", () => {
141
- expect(isCatalogFormatInRange("1.0", range)).toBe(true);
142
- });
143
- it("rejects below min and above max", () => {
144
- expect(isCatalogFormatInRange("0.9", range)).toBe(false);
145
- expect(isCatalogFormatInRange("1.1", range)).toBe(false);
146
- });
147
- });
148
-
149
- // The version-contract: the ENGINE_MAX_FORMAT matrix MUST cover the DuckDB
150
- // engine this repo is pinned to. This is the same drift guard the CI script
151
- // enforces (scripts/validate-ducklake-catalog-range.ts); asserting it here
152
- // fails `bun test` the moment a Malloy bump moves the engine past the matrix.
153
- describe("pinned-engine drift contract", () => {
154
- it("has a matrix row covering the pinned @duckdb/node-api engine", () => {
155
- const engineVersion = resolvePinnedDuckDBVersion();
156
- const range = catalogFormatRangeForEngine(engineVersion);
157
- expect(range).not.toBeNull();
158
- expect(range!.min).toBe(MIN_CATALOG_FORMAT);
159
- // The matrix must not be empty and its rows must be well-formed.
160
- expect(ENGINE_MAX_FORMAT.length).toBeGreaterThan(0);
161
- });
162
- });
163
- });
@@ -1,153 +0,0 @@
1
- // DuckLake catalog-format version contract.
2
- //
3
- // A DuckLake catalog records the on-disk format version it was written at. The
4
- // DuckLake extension bundled with a given DuckDB engine attaches only a bounded
5
- // RANGE of catalog formats without migration; a catalog outside that range
6
- // fails deep inside DuckDB with an opaque error. This module derives the
7
- // supported range purely from the pinned DuckDB engine version, so it never has
8
- // to enumerate an allow-list of "known good" catalog versions (which drifts
9
- // silently on an engine bump and is easy to forget to update). Both the runtime
10
- // attach preflight (`service/connection.ts`) and the CI version-contract check
11
- // (`scripts/validate-ducklake-catalog-range.ts`) consume this one source.
12
- //
13
- // Lower bound: fixed at "1.0". The 1.x DuckLake line does not attach the older
14
- // 0.x formats without AUTOMATIC_MIGRATION, and the Publisher attach paths never
15
- // pass that flag, so 1.0 is the floor. Upper bound: the maximum format the
16
- // engine's bundled extension writes/attaches, looked up in the migration matrix
17
- // below and keyed by engine (major, minor).
18
-
19
- /** A parsed DuckLake catalog format version, e.g. `1.0` or `0.3-dev1`. */
20
- export interface CatalogFormat {
21
- major: number;
22
- minor: number;
23
- /** e.g. `dev1` in `0.3-dev1`; a pre-release sorts BELOW the same release. */
24
- prerelease?: string;
25
- }
26
-
27
- /** The inclusive `[min, max]` catalog-format range for a given engine. */
28
- export interface CatalogFormatRange {
29
- min: string;
30
- max: string;
31
- /** The engine minor line the range was derived from (for error messages). */
32
- engineVersion: string;
33
- }
34
-
35
- /** Fixed lower bound of the supported range (see file header). */
36
- export const MIN_CATALOG_FORMAT = "1.0";
37
-
38
- /**
39
- * Migration matrix: DuckDB engine minor line -> the maximum DuckLake catalog
40
- * format its bundled DuckLake extension attaches WITHOUT migration. This is NOT
41
- * an allow-list of catalog versions -- it is the single fact, per engine minor
42
- * line, from which the `[MIN_CATALOG_FORMAT, maxFormat]` range is derived.
43
- *
44
- * Verified against the baked extension: DuckDB 1.5.x's DuckLake extension writes
45
- * and attaches the "1.0" catalog format (the older 0.x formats require
46
- * AUTOMATIC_MIGRATION, which Publisher never passes). When a Malloy bump moves
47
- * `@duckdb/node-api` to a new minor line, add its row here after verifying the
48
- * extension's max attach-without-migration format -- the CI version-contract
49
- * check FAILS THE BUILD until a row covers the pinned engine, so this never
50
- * drifts silently.
51
- */
52
- export const ENGINE_MAX_FORMAT: ReadonlyArray<{
53
- engine: { major: number; minor: number };
54
- maxFormat: string;
55
- }> = [{ engine: { major: 1, minor: 5 }, maxFormat: "1.0" }];
56
-
57
- const CATALOG_FORMAT_RE = /^(\d+)\.(\d+)(?:-([0-9A-Za-z.]+))?$/;
58
-
59
- /**
60
- * Parse a catalog format string into its components, or `null` if it is not a
61
- * valid `<major>.<minor>[-<prerelease>]` format. A three-part version like
62
- * `1.0.0` is intentionally rejected: catalog formats are two-part, and a
63
- * three-part string is a malformed input, not a newer format.
64
- */
65
- export function parseCatalogFormat(raw: string): CatalogFormat | null {
66
- if (typeof raw !== "string") return null;
67
- const m = CATALOG_FORMAT_RE.exec(raw);
68
- if (!m) return null;
69
- const format: CatalogFormat = { major: Number(m[1]), minor: Number(m[2]) };
70
- if (m[3] !== undefined) format.prerelease = m[3];
71
- return format;
72
- }
73
-
74
- /**
75
- * Compare two parsed catalog formats: -1 if `a < b`, 0 if equal, 1 if `a > b`.
76
- * A release sorts ABOVE its own pre-releases (`1.0` > `1.0-dev1`), matching
77
- * semver ordering.
78
- */
79
- export function compareCatalogFormat(
80
- a: CatalogFormat,
81
- b: CatalogFormat,
82
- ): number {
83
- if (a.major !== b.major) return a.major < b.major ? -1 : 1;
84
- if (a.minor !== b.minor) return a.minor < b.minor ? -1 : 1;
85
- if (a.prerelease === b.prerelease) return 0;
86
- if (a.prerelease === undefined) return 1; // release > pre-release
87
- if (b.prerelease === undefined) return -1;
88
- return a.prerelease < b.prerelease ? -1 : 1;
89
- }
90
-
91
- /**
92
- * Extract the `{ major, minor }` line from a DuckDB engine version string
93
- * (`1.5.3`, `1.5.3-r.2`, `v1.5.3` all parse to `{1, 5}`), or `null` if none is
94
- * present.
95
- */
96
- export function parseEngineMajorMinor(
97
- raw: string,
98
- ): { major: number; minor: number } | null {
99
- const m = /(\d+)\.(\d+)\.(\d+)/.exec(String(raw));
100
- if (!m) return null;
101
- return { major: Number(m[1]), minor: Number(m[2]) };
102
- }
103
-
104
- /**
105
- * The supported catalog-format range for a DuckDB engine version, or `null`
106
- * when the engine's minor line has no row in {@link ENGINE_MAX_FORMAT}. A
107
- * `null` return is the DRIFT signal: the CI version-contract check fails the
108
- * build on it, and the runtime preflight treats it as "unknown, skip" (the
109
- * preflight is non-load-bearing). The match is by `(major, minor)` so any patch
110
- * release of a known minor line resolves.
111
- */
112
- export function catalogFormatRangeForEngine(
113
- engineVersion: string,
114
- ): CatalogFormatRange | null {
115
- const e = parseEngineMajorMinor(engineVersion);
116
- if (!e) return null;
117
- const entry = ENGINE_MAX_FORMAT.find(
118
- (r) => r.engine.major === e.major && r.engine.minor === e.minor,
119
- );
120
- if (!entry) return null;
121
- return {
122
- min: MIN_CATALOG_FORMAT,
123
- max: entry.maxFormat,
124
- engineVersion: `${e.major}.${e.minor}`,
125
- };
126
- }
127
-
128
- /** Whether `format` falls within the inclusive range (invalid input -> false). */
129
- export function isCatalogFormatInRange(
130
- format: string,
131
- range: CatalogFormatRange,
132
- ): boolean {
133
- const f = parseCatalogFormat(format);
134
- const min = parseCatalogFormat(range.min);
135
- const max = parseCatalogFormat(range.max);
136
- if (!f || !min || !max) return false;
137
- return (
138
- compareCatalogFormat(f, min) >= 0 && compareCatalogFormat(f, max) <= 0
139
- );
140
- }
141
-
142
- /**
143
- * Whether a catalog `format` is supported by the DuckLake extension bundled
144
- * with `engineVersion`. Returns `false` for an unknown engine (no matrix row)
145
- * or a malformed format.
146
- */
147
- export function isCatalogFormatSupportedByEngine(
148
- format: string,
149
- engineVersion: string,
150
- ): boolean {
151
- const range = catalogFormatRangeForEngine(engineVersion);
152
- return range ? isCatalogFormatInRange(format, range) : false;
153
- }