@malloy-publisher/server 0.0.235 → 0.0.236

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (301) hide show
  1. package/dist/app/api-doc.yaml +47 -17
  2. package/dist/package_load_worker.mjs +8 -1
  3. package/dist/server.mjs +471 -54
  4. package/package.json +5 -2
  5. package/.eslintrc.json +0 -44
  6. package/.prettierignore +0 -1
  7. package/.prettierrc +0 -4
  8. package/build.ts +0 -97
  9. package/eslint.config.mjs +0 -76
  10. package/k6-tests/README.md +0 -144
  11. package/k6-tests/bun.lock +0 -797
  12. package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
  13. package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
  14. package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
  15. package/k6-tests/load-test/load-test-crud.ts +0 -142
  16. package/k6-tests/load-test/load-test.ts +0 -784
  17. package/k6-tests/package.json +0 -27
  18. package/k6-tests/scripts/run-with-otel.sh +0 -50
  19. package/k6-tests/smoke-test/smoke-test.ts +0 -158
  20. package/k6-tests/tsconfig.json +0 -29
  21. package/k6-tests/utils/client_factory.ts +0 -128
  22. package/k6-tests/utils/common.ts +0 -489
  23. package/k6-tests/utils/logger.ts +0 -62
  24. package/k6-tests/utils/types.d.ts +0 -35
  25. package/malloy_mcp.dxt +0 -0
  26. package/publisher.config.example.bigquery.json +0 -33
  27. package/publisher.config.example.duckdb.json +0 -23
  28. package/publisher.config.json +0 -23
  29. package/scripts/bake-duckdb-extensions.js +0 -110
  30. package/src/config.spec.ts +0 -1535
  31. package/src/config.theme.spec.ts +0 -178
  32. package/src/config.ts +0 -1142
  33. package/src/constants.ts +0 -105
  34. package/src/controller/compile.controller.ts +0 -40
  35. package/src/controller/connection.controller.spec.ts +0 -849
  36. package/src/controller/connection.controller.ts +0 -901
  37. package/src/controller/database.controller.ts +0 -24
  38. package/src/controller/materialization.controller.spec.ts +0 -293
  39. package/src/controller/materialization.controller.ts +0 -319
  40. package/src/controller/model.controller.ts +0 -164
  41. package/src/controller/package.controller.spec.ts +0 -273
  42. package/src/controller/package.controller.ts +0 -306
  43. package/src/controller/query.controller.ts +0 -152
  44. package/src/controller/theme.controller.ts +0 -83
  45. package/src/controller/watch-mode.controller.ts +0 -243
  46. package/src/data_styles.ts +0 -71
  47. package/src/default-publisher.config.json +0 -23
  48. package/src/ducklake_version.spec.ts +0 -163
  49. package/src/ducklake_version.ts +0 -153
  50. package/src/errors.spec.ts +0 -119
  51. package/src/errors.ts +0 -237
  52. package/src/filter_deprecation.spec.ts +0 -64
  53. package/src/filter_deprecation.ts +0 -42
  54. package/src/health.spec.ts +0 -123
  55. package/src/health.ts +0 -233
  56. package/src/heap_check.spec.ts +0 -144
  57. package/src/heap_check.ts +0 -144
  58. package/src/instrumentation.ts +0 -214
  59. package/src/json_utils.spec.ts +0 -51
  60. package/src/json_utils.ts +0 -33
  61. package/src/logger.spec.ts +0 -210
  62. package/src/logger.ts +0 -251
  63. package/src/malloy_pin_prereqs.spec.ts +0 -25
  64. package/src/materialization_metrics.spec.ts +0 -169
  65. package/src/materialization_metrics.ts +0 -346
  66. package/src/mcp/error_messages.spec.ts +0 -35
  67. package/src/mcp/error_messages.ts +0 -183
  68. package/src/mcp/handler_utils.spec.ts +0 -108
  69. package/src/mcp/handler_utils.ts +0 -251
  70. package/src/mcp/mcp_constants.ts +0 -6
  71. package/src/mcp/query_envelope.spec.ts +0 -229
  72. package/src/mcp/query_envelope.ts +0 -240
  73. package/src/mcp/server.protocol.spec.ts +0 -250
  74. package/src/mcp/server.ts +0 -87
  75. package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
  76. package/src/mcp/skills/build_skills_bundle.ts +0 -176
  77. package/src/mcp/skills/skills_bundle.json +0 -1
  78. package/src/mcp/skills/skills_bundle.spec.ts +0 -171
  79. package/src/mcp/tool_response.spec.ts +0 -108
  80. package/src/mcp/tool_response.ts +0 -138
  81. package/src/mcp/tools/compile_tool.spec.ts +0 -315
  82. package/src/mcp/tools/compile_tool.ts +0 -208
  83. package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
  84. package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
  85. package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
  86. package/src/mcp/tools/docs_search_tool.ts +0 -128
  87. package/src/mcp/tools/embedding_index.spec.ts +0 -1236
  88. package/src/mcp/tools/embedding_index.ts +0 -808
  89. package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
  90. package/src/mcp/tools/execute_query_tool.ts +0 -316
  91. package/src/mcp/tools/get_context_eval.ts +0 -275
  92. package/src/mcp/tools/get_context_tool.spec.ts +0 -592
  93. package/src/mcp/tools/get_context_tool.ts +0 -585
  94. package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
  95. package/src/mcp/tools/reload_package_tool.ts +0 -132
  96. package/src/mcp_config.spec.ts +0 -919
  97. package/src/mcp_config.ts +0 -425
  98. package/src/oom_guards.integration.spec.ts +0 -269
  99. package/src/package_load/package_load_pool.spec.ts +0 -252
  100. package/src/package_load/package_load_pool.ts +0 -931
  101. package/src/package_load/package_load_worker.ts +0 -1034
  102. package/src/package_load/protocol.ts +0 -364
  103. package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
  104. package/src/package_load/rpc_wait_accountant.ts +0 -76
  105. package/src/package_load_metrics.spec.ts +0 -114
  106. package/src/package_load_metrics.ts +0 -127
  107. package/src/path_safety.spec.ts +0 -158
  108. package/src/path_safety.ts +0 -146
  109. package/src/pg_helpers.spec.ts +0 -223
  110. package/src/pg_helpers.ts +0 -52
  111. package/src/query_cap_metrics.spec.ts +0 -89
  112. package/src/query_cap_metrics.ts +0 -116
  113. package/src/query_concurrency.spec.ts +0 -247
  114. package/src/query_concurrency.ts +0 -237
  115. package/src/query_metadata_metrics.ts +0 -49
  116. package/src/query_param_utils.ts +0 -29
  117. package/src/query_timeout.spec.ts +0 -224
  118. package/src/query_timeout.ts +0 -179
  119. package/src/runtime/publisher.js +0 -328
  120. package/src/server-old.ts +0 -988
  121. package/src/server.ts +0 -2059
  122. package/src/service/annotations.spec.ts +0 -118
  123. package/src/service/annotations.ts +0 -91
  124. package/src/service/authorize.spec.ts +0 -154
  125. package/src/service/authorize.ts +0 -498
  126. package/src/service/authorize_integration.spec.ts +0 -2041
  127. package/src/service/build_plan.spec.ts +0 -685
  128. package/src/service/build_plan.ts +0 -797
  129. package/src/service/compile_authorize.spec.ts +0 -85
  130. package/src/service/compile_fragment_techniques.spec.ts +0 -156
  131. package/src/service/connection.spec.ts +0 -2360
  132. package/src/service/connection.ts +0 -2187
  133. package/src/service/connection_config.spec.ts +0 -734
  134. package/src/service/connection_config.ts +0 -899
  135. package/src/service/connection_federation.spec.ts +0 -184
  136. package/src/service/connection_fingerprint.spec.ts +0 -102
  137. package/src/service/connection_service.spec.ts +0 -711
  138. package/src/service/connection_service.ts +0 -291
  139. package/src/service/cron_evaluator.spec.ts +0 -79
  140. package/src/service/cron_evaluator.ts +0 -105
  141. package/src/service/db_utils.spec.ts +0 -1085
  142. package/src/service/db_utils.ts +0 -1502
  143. package/src/service/duckdb_instance_isolation.spec.ts +0 -137
  144. package/src/service/ducklake_lazy_attach.spec.ts +0 -110
  145. package/src/service/embedding_provider.spec.ts +0 -329
  146. package/src/service/embedding_provider.ts +0 -236
  147. package/src/service/environment.ts +0 -2191
  148. package/src/service/environment_admission.spec.ts +0 -344
  149. package/src/service/environment_compile.spec.ts +0 -197
  150. package/src/service/environment_store.spec.ts +0 -2274
  151. package/src/service/environment_store.ts +0 -2434
  152. package/src/service/environment_store_anchoring.spec.ts +0 -107
  153. package/src/service/environment_store_clone.spec.ts +0 -350
  154. package/src/service/explore_visibility.spec.ts +0 -434
  155. package/src/service/exports_probe.spec.ts +0 -107
  156. package/src/service/extension_fetch_policy.spec.ts +0 -256
  157. package/src/service/filter.spec.ts +0 -458
  158. package/src/service/filter.ts +0 -341
  159. package/src/service/filter_bypass.spec.ts +0 -418
  160. package/src/service/filter_integration.spec.ts +0 -1004
  161. package/src/service/freshness.spec.ts +0 -183
  162. package/src/service/freshness.ts +0 -112
  163. package/src/service/gcs_s3_utils.ts +0 -427
  164. package/src/service/given.ts +0 -105
  165. package/src/service/givens_integration.spec.ts +0 -219
  166. package/src/service/manifest_loader.spec.ts +0 -189
  167. package/src/service/manifest_loader.ts +0 -156
  168. package/src/service/materialization_build_session.spec.ts +0 -435
  169. package/src/service/materialization_build_session.ts +0 -681
  170. package/src/service/materialization_config_validation.spec.ts +0 -99
  171. package/src/service/materialization_config_validation.ts +0 -120
  172. package/src/service/materialization_eligibility.spec.ts +0 -158
  173. package/src/service/materialization_eligibility.ts +0 -305
  174. package/src/service/materialization_schedule_surface.spec.ts +0 -256
  175. package/src/service/materialization_scheduler.spec.ts +0 -289
  176. package/src/service/materialization_scheduler.ts +0 -304
  177. package/src/service/materialization_serve_transform.spec.ts +0 -1003
  178. package/src/service/materialization_serve_transform.ts +0 -779
  179. package/src/service/materialization_service.spec.ts +0 -2373
  180. package/src/service/materialization_service.ts +0 -2342
  181. package/src/service/materialization_test_fixtures.ts +0 -249
  182. package/src/service/model.spec.ts +0 -1157
  183. package/src/service/model.ts +0 -3370
  184. package/src/service/model_limits.spec.ts +0 -209
  185. package/src/service/model_limits.ts +0 -131
  186. package/src/service/model_storage_serve.spec.ts +0 -193
  187. package/src/service/model_storage_serve_joins.spec.ts +0 -193
  188. package/src/service/package.spec.ts +0 -612
  189. package/src/service/package.ts +0 -1799
  190. package/src/service/package_manifest.spec.ts +0 -230
  191. package/src/service/package_manifest.ts +0 -241
  192. package/src/service/package_memory_governor.spec.ts +0 -173
  193. package/src/service/package_memory_governor.ts +0 -233
  194. package/src/service/package_quote_bound_tables.spec.ts +0 -171
  195. package/src/service/package_race.spec.ts +0 -208
  196. package/src/service/package_reload_safety.spec.ts +0 -193
  197. package/src/service/package_rollback.spec.ts +0 -190
  198. package/src/service/package_worker_path.spec.ts +0 -645
  199. package/src/service/path_injection.spec.ts +0 -39
  200. package/src/service/persist_annotation_validation.spec.ts +0 -89
  201. package/src/service/persist_annotation_validation.ts +0 -59
  202. package/src/service/persistence_policy.spec.ts +0 -374
  203. package/src/service/proxy.spec.ts +0 -414
  204. package/src/service/proxy.ts +0 -248
  205. package/src/service/query_boundary.spec.ts +0 -470
  206. package/src/service/query_metadata.spec.ts +0 -408
  207. package/src/service/query_metadata.ts +0 -492
  208. package/src/service/query_metadata_identity.spec.ts +0 -149
  209. package/src/service/query_text.spec.ts +0 -79
  210. package/src/service/query_text.ts +0 -43
  211. package/src/service/quoting.spec.ts +0 -135
  212. package/src/service/quoting.ts +0 -78
  213. package/src/service/resolve_environment.ts +0 -15
  214. package/src/service/restricted_mode.spec.ts +0 -299
  215. package/src/service/source_extraction.ts +0 -225
  216. package/src/service/theme_store.ts +0 -199
  217. package/src/storage/DatabaseInterface.ts +0 -198
  218. package/src/storage/StorageManager.ts +0 -113
  219. package/src/storage/duckdb/ConnectionRepository.ts +0 -143
  220. package/src/storage/duckdb/DuckDBConnection.ts +0 -123
  221. package/src/storage/duckdb/DuckDBRepository.ts +0 -208
  222. package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
  223. package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
  224. package/src/storage/duckdb/MaterializationRepository.ts +0 -336
  225. package/src/storage/duckdb/PackageRepository.ts +0 -135
  226. package/src/storage/duckdb/schema.ts +0 -229
  227. package/src/stream_helpers.spec.ts +0 -280
  228. package/src/stream_helpers.ts +0 -162
  229. package/src/telemetry.ts +0 -20
  230. package/src/test_helpers/metrics_harness.ts +0 -166
  231. package/src/theme_key_parity.spec.ts +0 -57
  232. package/src/utils.ts +0 -29
  233. package/tests/fixtures/authorize-compile/model.malloy +0 -9
  234. package/tests/fixtures/authorize-compile/publisher.json +0 -4
  235. package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
  236. package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
  237. package/tests/fixtures/html-pages-test/data.csv +0 -3
  238. package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
  239. package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
  240. package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
  241. package/tests/fixtures/html-pages-test/public/data.json +0 -1
  242. package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
  243. package/tests/fixtures/html-pages-test/public/index.html +0 -9
  244. package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
  245. package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
  246. package/tests/fixtures/html-pages-test/public/slides.html +0 -12
  247. package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
  248. package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
  249. package/tests/fixtures/html-pages-test/publisher.json +0 -5
  250. package/tests/fixtures/html-pages-test/report.malloy +0 -1
  251. package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
  252. package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
  253. package/tests/fixtures/persist-multi-level/publisher.json +0 -5
  254. package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
  255. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
  256. package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
  257. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
  258. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
  259. package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
  260. package/tests/fixtures/persist-test/data/orders.csv +0 -5
  261. package/tests/fixtures/persist-test/persist_test.malloy +0 -11
  262. package/tests/fixtures/persist-test/publisher.json +0 -5
  263. package/tests/fixtures/publisher.config.json +0 -15
  264. package/tests/fixtures/query-givens/data/orders.csv +0 -7
  265. package/tests/fixtures/query-givens/model.malloy +0 -34
  266. package/tests/fixtures/query-givens/publisher.json +0 -5
  267. package/tests/fixtures/xlsx/database.xlsx +0 -0
  268. package/tests/harness/mcp_test_setup.ts +0 -332
  269. package/tests/harness/rest_e2e.ts +0 -68
  270. package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
  271. package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
  272. package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
  273. package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
  274. package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
  275. package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
  276. package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
  277. package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
  278. package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
  279. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
  280. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
  281. package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
  282. package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
  283. package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
  284. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
  285. package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
  286. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
  287. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
  288. package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
  289. package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
  290. package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
  291. package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
  292. package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
  293. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
  294. package/tests/unit/controller/theme.controller.spec.ts +0 -141
  295. package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
  296. package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
  297. package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
  298. package/tests/unit/duckdb/repositories.test.ts +0 -208
  299. package/tests/unit/ducklake/ducklake.test.ts +0 -952
  300. package/tests/unit/service/theme_store.spec.ts +0 -139
  301. package/tsconfig.json +0 -32
@@ -1,259 +0,0 @@
1
- /// <reference types="bun-types" />
2
-
3
- // TODO: Remove this during projects cleanup
4
- /**
5
- * Smoke tests for the legacy `/api/v0/projects/...` REST surface registered
6
- * by `server-old.ts`. These routes exist purely to keep pre-rename SDK
7
- * clients (e.g. `@malloydata/db-publisher`) working after the
8
- * projects→environments rename.
9
- *
10
- * One test per route group: projects CRUD, packages, connections, models,
11
- * notebooks, databases, queries, materializations, manifest. The
12
- * materialization test additionally asserts the response field rename
13
- * (`projectId` not `environmentId`). `/status` is no longer in the legacy
14
- * surface — both old and new clients hit the single `/api/v0/status`
15
- * handler in server.ts, which returns `environments`.
16
- *
17
- * This file is intentionally separate from the regular integration suite so
18
- * it can be deleted in one motion when legacy support is dropped.
19
- */
20
-
21
- import { afterAll, beforeAll, describe, expect, it } from "bun:test";
22
- import path from "path";
23
- import { fileURLToPath } from "url";
24
- import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
25
-
26
- const __filename = fileURLToPath(import.meta.url);
27
- const __dirname = path.dirname(__filename);
28
-
29
- // Use a distinct project name so this suite doesn't collide with the
30
- // materialization integration suite (which also uses "test-project") if
31
- // they ever run in the same DB instance.
32
- const PROJECT_NAME = "legacy-routes-test-project";
33
- const PACKAGE_NAME = "persist-test";
34
-
35
- describe("Legacy /api/v0/projects/* REST routes (E2E)", () => {
36
- let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
37
- let baseUrl: string;
38
-
39
- beforeAll(async () => {
40
- env = await startRestE2E();
41
- baseUrl = env.baseUrl;
42
-
43
- // Create the test environment via the LEGACY route — proves POST
44
- // /projects works end-to-end.
45
- const fixtureDir = path.resolve(__dirname, "../../fixtures/persist-test");
46
- const createRes = await fetch(`${baseUrl}/api/v0/projects`, {
47
- method: "POST",
48
- headers: { "Content-Type": "application/json" },
49
- body: JSON.stringify({
50
- name: PROJECT_NAME,
51
- packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
52
- connections: [],
53
- }),
54
- });
55
- if (!createRes.ok) {
56
- const body = await createRes.text();
57
- throw new Error(
58
- `Failed to create test project via legacy route (${createRes.status}): ${body}`,
59
- );
60
- }
61
-
62
- // Wait for the package to finish loading via the legacy GET path.
63
- const deadline = Date.now() + 30_000;
64
- let pkgReady = false;
65
- while (!pkgReady && Date.now() < deadline) {
66
- try {
67
- const res = await fetch(
68
- `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}`,
69
- );
70
- if (res.ok) {
71
- pkgReady = true;
72
- break;
73
- }
74
- } catch {
75
- // not ready yet
76
- }
77
- await new Promise((r) => setTimeout(r, 500));
78
- }
79
- if (!pkgReady) {
80
- throw new Error("Test package did not become available in time");
81
- }
82
- });
83
-
84
- afterAll(async () => {
85
- if (baseUrl) {
86
- try {
87
- // Clean up via the legacy DELETE route.
88
- await fetch(`${baseUrl}/api/v0/projects/${PROJECT_NAME}`, {
89
- method: "DELETE",
90
- });
91
- } catch {
92
- // best-effort cleanup
93
- }
94
- }
95
- await env?.stop();
96
- env = null;
97
- });
98
-
99
- describe("projects CRUD", () => {
100
- it("GET /projects lists environments under the legacy URL", async () => {
101
- const res = await fetch(`${baseUrl}/api/v0/projects`);
102
- expect(res.status).toBe(200);
103
- const body = (await res.json()) as Array<{ name?: string }>;
104
- expect(Array.isArray(body)).toBe(true);
105
- expect(body.some((e) => e.name === PROJECT_NAME)).toBe(true);
106
- });
107
-
108
- it("GET /projects/:projectName returns the project", async () => {
109
- const res = await fetch(`${baseUrl}/api/v0/projects/${PROJECT_NAME}`);
110
- expect(res.status).toBe(200);
111
- const body = (await res.json()) as { name?: string };
112
- expect(body.name).toBe(PROJECT_NAME);
113
- });
114
- });
115
-
116
- describe("packages", () => {
117
- it("GET /projects/:projectName/packages returns the package list", async () => {
118
- const res = await fetch(
119
- `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages`,
120
- );
121
- expect(res.status).toBe(200);
122
- const body = (await res.json()) as Array<{ name?: string }>;
123
- expect(Array.isArray(body)).toBe(true);
124
- expect(body.some((p) => p.name === PACKAGE_NAME)).toBe(true);
125
- });
126
-
127
- it("GET /projects/:projectName/packages/:packageName returns the package", async () => {
128
- const res = await fetch(
129
- `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}`,
130
- );
131
- expect(res.status).toBe(200);
132
- const body = (await res.json()) as { name?: string };
133
- expect(body.name).toBe(PACKAGE_NAME);
134
- });
135
- });
136
-
137
- describe("connections", () => {
138
- it("GET /projects/:projectName/connections returns 200 (may be empty)", async () => {
139
- const res = await fetch(
140
- `${baseUrl}/api/v0/projects/${PROJECT_NAME}/connections`,
141
- );
142
- expect(res.status).toBe(200);
143
- const body = await res.json();
144
- expect(Array.isArray(body)).toBe(true);
145
- });
146
- });
147
-
148
- describe("models", () => {
149
- it("GET /projects/:projectName/packages/:packageName/models returns the model list", async () => {
150
- const res = await fetch(
151
- `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/models`,
152
- );
153
- expect(res.status).toBe(200);
154
- const body = await res.json();
155
- expect(Array.isArray(body)).toBe(true);
156
- });
157
- });
158
-
159
- describe("notebooks", () => {
160
- it("GET /projects/:projectName/packages/:packageName/notebooks returns 200", async () => {
161
- const res = await fetch(
162
- `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/notebooks`,
163
- );
164
- expect(res.status).toBe(200);
165
- const body = await res.json();
166
- expect(Array.isArray(body)).toBe(true);
167
- });
168
- });
169
-
170
- describe("databases", () => {
171
- it("GET /projects/:projectName/packages/:packageName/databases returns 200", async () => {
172
- const res = await fetch(
173
- `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/databases`,
174
- );
175
- expect(res.status).toBe(200);
176
- const body = await res.json();
177
- expect(Array.isArray(body)).toBe(true);
178
- });
179
- });
180
-
181
- describe("queries", () => {
182
- it("POST /projects/:projectName/packages/:packageName/models/.../query reaches the handler", async () => {
183
- // Hit the route with a bogus model name. We only need to prove the
184
- // legacy URL is wired up to the controller — a structured JSON
185
- // error (not Express's HTML fall-through 404) is sufficient signal.
186
- const res = await fetch(
187
- `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/models/does-not-exist.malloy/query`,
188
- {
189
- method: "POST",
190
- headers: { "Content-Type": "application/json" },
191
- body: JSON.stringify({ query: "run: nothing" }),
192
- },
193
- );
194
- expect(res.status).toBeGreaterThanOrEqual(400);
195
- expect(res.status).toBeLessThan(600);
196
- // Controller errors come back as JSON with a `message` field.
197
- // An unhandled Express 404 returns HTML — that would fail here.
198
- const body = (await res.json()) as Record<string, unknown>;
199
- expect(typeof body.message).toBe("string");
200
- });
201
- });
202
-
203
- describe("materializations", () => {
204
- it("GET list and POST create return 'projectId' (not 'environmentId') under the legacy URL", async () => {
205
- const listRes = await fetch(
206
- `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/materializations`,
207
- );
208
- expect(listRes.status).toBe(200);
209
- const list = (await listRes.json()) as unknown;
210
- expect(Array.isArray(list)).toBe(true);
211
-
212
- // Create one so we can assert the field rename on a populated payload.
213
- const createRes = await fetch(
214
- `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/materializations`,
215
- {
216
- method: "POST",
217
- headers: { "Content-Type": "application/json" },
218
- body: JSON.stringify({ autoLoadManifest: true }),
219
- },
220
- );
221
- expect(createRes.status).toBe(201);
222
- const created = (await createRes.json()) as Record<string, unknown>;
223
-
224
- // Legacy contract: materialization payloads expose `projectId`, not
225
- // `environmentId`. This is the response remapper in server-old.ts.
226
- expect(created).toHaveProperty("projectId");
227
- expect(created).not.toHaveProperty("environmentId");
228
-
229
- const id = created.id as string;
230
- // Best-effort cleanup so we don't leak a PENDING materialization
231
- // into other tests. We don't poll-to-terminal; the suite teardown
232
- // of the project will mop up.
233
- try {
234
- await fetch(
235
- `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/materializations/${id}?action=stop`,
236
- { method: "POST" },
237
- );
238
- await fetch(
239
- `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/materializations/${id}`,
240
- { method: "DELETE" },
241
- );
242
- } catch {
243
- // ignore
244
- }
245
- });
246
- });
247
-
248
- describe("manifest", () => {
249
- it("GET /projects/:projectName/packages/:packageName/manifest returns 200 or a structured 4xx", async () => {
250
- const res = await fetch(
251
- `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/manifest`,
252
- );
253
- // Without a built materialization the manifest may be empty/404 —
254
- // we only assert the legacy URL reaches the handler, not 404 from
255
- // Express's catch-all.
256
- expect([200, 400, 404]).toContain(res.status);
257
- });
258
- });
259
- });
@@ -1,133 +0,0 @@
1
- /// <reference types="bun-types" />
2
-
3
- import { afterAll, beforeAll, describe, expect, it } from "bun:test";
4
- import path from "path";
5
- import { fileURLToPath } from "url";
6
- import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
7
-
8
- const __filename = fileURLToPath(import.meta.url);
9
- const __dirname = path.dirname(__filename);
10
-
11
- const PROJECT = "env-mat-list-project";
12
- const PKG_A = "persist-test";
13
- const PKG_B = "persist-schedule-test";
14
- const ENV_MATS_PATH = `/api/v0/environments/${PROJECT}/packages/materializations`;
15
-
16
- // Verifies the environment-scoped list:
17
- // GET /environments/{env}/packages/materializations returns runs from every
18
- // package in the env, newest-first, each labeled with its packageName.
19
- describe("Environment-scoped materialization list", () => {
20
- let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
21
- let baseUrl: string;
22
-
23
- const fixture = (name: string) =>
24
- path.resolve(__dirname, `../../fixtures/${name}`);
25
-
26
- async function createAndFinish(pkg: string): Promise<string> {
27
- const res = await fetch(
28
- `${baseUrl}/api/v0/environments/${PROJECT}/packages/${pkg}/materializations`,
29
- {
30
- method: "POST",
31
- headers: { "Content-Type": "application/json" },
32
- body: JSON.stringify({ forceRefresh: true }),
33
- },
34
- );
35
- expect(res.status).toBe(201);
36
- const created = (await res.json()) as { id: string };
37
- const deadline = Date.now() + 60_000;
38
- while (Date.now() < deadline) {
39
- const poll = await fetch(
40
- `${baseUrl}/api/v0/environments/${PROJECT}/packages/${pkg}/materializations/${created.id}`,
41
- );
42
- const m = (await poll.json()) as { status: string };
43
- if (
44
- ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"].includes(m.status)
45
- ) {
46
- break;
47
- }
48
- await new Promise((r) => setTimeout(r, 250));
49
- }
50
- return created.id;
51
- }
52
-
53
- beforeAll(async () => {
54
- env = await startRestE2E();
55
- baseUrl = env.baseUrl;
56
-
57
- const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
58
- method: "POST",
59
- headers: { "Content-Type": "application/json" },
60
- body: JSON.stringify({
61
- name: PROJECT,
62
- packages: [
63
- { name: PKG_A, location: fixture(PKG_A) },
64
- { name: PKG_B, location: fixture(PKG_B) },
65
- ],
66
- connections: [],
67
- }),
68
- });
69
- if (!createRes.ok) {
70
- throw new Error(
71
- `Failed to create project (${createRes.status}): ${await createRes.text()}`,
72
- );
73
- }
74
-
75
- // Wait for both packages to load.
76
- for (const pkg of [PKG_A, PKG_B]) {
77
- const deadline = Date.now() + 30_000;
78
- while (Date.now() < deadline) {
79
- const res = await fetch(
80
- `${baseUrl}/api/v0/environments/${PROJECT}/packages/${pkg}`,
81
- );
82
- if (res.ok) break;
83
- await new Promise((r) => setTimeout(r, 500));
84
- }
85
- }
86
- });
87
-
88
- afterAll(async () => {
89
- if (baseUrl) {
90
- try {
91
- await fetch(`${baseUrl}/api/v0/environments/${PROJECT}`, {
92
- method: "DELETE",
93
- });
94
- } catch {
95
- // best-effort
96
- }
97
- }
98
- await env?.stop();
99
- env = null;
100
- });
101
-
102
- it("returns materializations from every package in the environment", async () => {
103
- await createAndFinish(PKG_A);
104
- await createAndFinish(PKG_B);
105
-
106
- const res = await fetch(`${baseUrl}${ENV_MATS_PATH}`);
107
- expect(res.status).toBe(200);
108
- const rows = (await res.json()) as Array<{
109
- packageName: string;
110
- createdAt?: string;
111
- }>;
112
-
113
- const packages = new Set(rows.map((r) => r.packageName));
114
- expect(packages.has(PKG_A)).toBe(true);
115
- expect(packages.has(PKG_B)).toBe(true);
116
-
117
- // Newest-first ordering.
118
- const times = rows
119
- .map((r) => (r.createdAt ? new Date(r.createdAt).getTime() : 0))
120
- .filter((t) => t > 0);
121
- const sorted = [...times].sort((a, b) => b - a);
122
- expect(times).toEqual(sorted);
123
- });
124
-
125
- it("does not shadow the per-package list route", async () => {
126
- const res = await fetch(
127
- `${baseUrl}/api/v0/environments/${PROJECT}/packages/${PKG_A}/materializations`,
128
- );
129
- expect(res.status).toBe(200);
130
- const rows = (await res.json()) as Array<{ packageName: string }>;
131
- expect(rows.every((r) => r.packageName === PKG_A)).toBe(true);
132
- });
133
- });
@@ -1,292 +0,0 @@
1
- /// <reference types="bun-types" />
2
-
3
- import { afterAll, beforeAll, describe, expect, it } from "bun:test";
4
- import * as fsp from "fs/promises";
5
- import os from "os";
6
- import path from "path";
7
- import { fileURLToPath } from "url";
8
- import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
9
-
10
- const __filename = fileURLToPath(import.meta.url);
11
- const __dirname = path.dirname(__filename);
12
-
13
- const PROJECT_NAME = "freshness-gate-project";
14
- const PACKAGE_NAME = "persist-test";
15
- const MODEL_PATH = "persist_test.malloy";
16
- const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
17
-
18
- /**
19
- * Covers the per-query freshness gate (persistence.md §9.3, §14 Phase B): a
20
- * query on a `#@ persist` source uses its materialized table only while the
21
- * table is within its declared freshness window; otherwise it falls back per the
22
- * entry's declared `fallback`.
23
- *
24
- * - fresh (or un-gated) → routes to the materialized table
25
- * - stale + fallback live → serves live SQL (entry dropped)
26
- * - stale + fallback stale_ok→ serves the stale table
27
- * - a fresh entry that crosses its window → serves live on the NEXT query,
28
- * with no rebind (proves per-query re-evaluation, not bind-time filtering)
29
- *
30
- * The manifest is bound once physically (auto-run builds the table), then a
31
- * manifest URI keyed by the source's real sourceEntityId routes served queries.
32
- * `executedSql()` is the routing evidence: it scans `order_summary` when routed
33
- * to the table, and `data/orders.csv` when serving live.
34
- */
35
- describe("Per-query freshness gate (E2E)", () => {
36
- let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
37
- let baseUrl: string;
38
- let tmpDir: string;
39
- let sourceEntityId: string;
40
-
41
- beforeAll(async () => {
42
- env = await startRestE2E();
43
- baseUrl = env.baseUrl;
44
- tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "freshness-gate-"));
45
-
46
- const fixtureDir = path.resolve(__dirname, "../../fixtures/persist-test");
47
- const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
48
- method: "POST",
49
- headers: { "Content-Type": "application/json" },
50
- body: JSON.stringify({
51
- name: PROJECT_NAME,
52
- packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
53
- connections: [],
54
- }),
55
- });
56
- if (!createRes.ok) {
57
- throw new Error(
58
- `Failed to create test project (${createRes.status}): ${await createRes.text()}`,
59
- );
60
- }
61
-
62
- const deadline = Date.now() + 30_000;
63
- while (Date.now() < deadline) {
64
- const res = await fetch(`${baseUrl}${API}`);
65
- if (res.ok) break;
66
- await new Promise((r) => setTimeout(r, 500));
67
- }
68
-
69
- // Physically build the persist source (auto-run self-assigns
70
- // physicalTableName = "order_summary"), then revert to live so the
71
- // manifest-URI bind is the only thing that can route the served query.
72
- await buildTableThenRevertToLive();
73
- sourceEntityId = await orderSummarySourceEntityId();
74
- }, 120_000);
75
-
76
- afterAll(async () => {
77
- if (baseUrl) {
78
- await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
79
- method: "DELETE",
80
- }).catch(() => {});
81
- }
82
- await env?.stop();
83
- await fsp.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
84
- env = null;
85
- });
86
-
87
- function url(p: string): string {
88
- return `${baseUrl}${API}${p}`;
89
- }
90
-
91
- const ROUTING_QUERY = "run: order_summary -> { aggregate: c is count() }";
92
-
93
- /** The SQL the served query actually compiled to (routing evidence). */
94
- async function executedSql(): Promise<string> {
95
- const res = await fetch(`${baseUrl}${API}/models/${MODEL_PATH}/query`, {
96
- method: "POST",
97
- headers: { "Content-Type": "application/json" },
98
- body: JSON.stringify({ query: ROUTING_QUERY }),
99
- });
100
- expect(res.status).toBe(200);
101
- const body = (await res.json()) as { result: string };
102
- return (JSON.parse(body.result) as { sql: string }).sql;
103
- }
104
-
105
- async function orderSummarySourceEntityId(): Promise<string> {
106
- const res = await fetch(url(""));
107
- expect(res.status).toBe(200);
108
- const pkg = (await res.json()) as {
109
- buildPlan?: { sources?: Record<string, { sourceEntityId?: string }> };
110
- };
111
- const sources = pkg.buildPlan?.sources ?? {};
112
- const id = Object.values(sources)[0]?.sourceEntityId;
113
- expect(typeof id).toBe("string");
114
- return id as string;
115
- }
116
-
117
- async function pollUntilTerminal(
118
- id: string,
119
- timeoutMs = 90_000,
120
- ): Promise<Record<string, unknown>> {
121
- const terminal = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"];
122
- const deadline = Date.now() + timeoutMs;
123
- while (Date.now() < deadline) {
124
- const res = await fetch(url(`/materializations/${id}`));
125
- expect(res.status).toBe(200);
126
- const data = (await res.json()) as Record<string, unknown>;
127
- if (terminal.includes(data.status as string)) return data;
128
- await new Promise((r) => setTimeout(r, 250));
129
- }
130
- throw new Error(`Materialization ${id} did not reach a terminal state`);
131
- }
132
-
133
- async function buildTableThenRevertToLive(): Promise<void> {
134
- const createRes = await fetch(url("/materializations"), {
135
- method: "POST",
136
- headers: { "Content-Type": "application/json" },
137
- body: JSON.stringify({}),
138
- });
139
- expect(createRes.status).toBe(201);
140
- const { id } = (await createRes.json()) as { id: string };
141
- const built = await pollUntilTerminal(id);
142
- expect(built.status).toBe("MANIFEST_FILE_READY");
143
- await fetch(url("?reload=true"));
144
- await fetch(url(`/materializations/${id}`), { method: "DELETE" });
145
- }
146
-
147
- /** Write a CP-shaped manifest carrying the freshness fields. */
148
- async function writeFreshnessManifest(opts: {
149
- dataAsOf: string;
150
- freshnessWindowSeconds: number;
151
- freshnessFallback: "live" | "stale_ok" | "fail";
152
- }): Promise<string> {
153
- const file = path.join(
154
- tmpDir,
155
- `freshness-${Date.now()}-${Math.random()}.json`,
156
- );
157
- await fsp.writeFile(
158
- file,
159
- JSON.stringify({
160
- builtAt: new Date().toISOString(),
161
- strict: false,
162
- entries: {
163
- [sourceEntityId]: {
164
- sourceEntityId,
165
- sourceName: "order_summary",
166
- physicalTableName: "order_summary",
167
- connectionName: "duckdb",
168
- dataAsOf: opts.dataAsOf,
169
- freshnessWindowSeconds: opts.freshnessWindowSeconds,
170
- freshnessFallback: opts.freshnessFallback,
171
- },
172
- },
173
- }),
174
- "utf8",
175
- );
176
- return file;
177
- }
178
-
179
- async function bind(manifestFile: string): Promise<void> {
180
- const patchRes = await fetch(url(""), {
181
- method: "PATCH",
182
- headers: { "Content-Type": "application/json" },
183
- body: JSON.stringify({
184
- name: PACKAGE_NAME,
185
- manifestLocation: manifestFile,
186
- }),
187
- });
188
- expect(patchRes.status).toBe(200);
189
- const patched = (await patchRes.json()) as Record<string, unknown>;
190
- // The bind is recorded regardless of per-query freshness — freshness is
191
- // evaluated per query, not at bind.
192
- expect(patched.manifestBindingStatus).toBe("bound");
193
- expect(patched.manifestEntryCount).toBe(1);
194
- }
195
-
196
- async function revertToLive(): Promise<void> {
197
- await fetch(url(""), {
198
- method: "PATCH",
199
- headers: { "Content-Type": "application/json" },
200
- body: JSON.stringify({ name: PACKAGE_NAME, manifestLocation: null }),
201
- });
202
- await fetch(url("?reload=true"));
203
- }
204
-
205
- it(
206
- "routes a fresh entry to the materialized table",
207
- async () => {
208
- const manifestFile = await writeFreshnessManifest({
209
- dataAsOf: new Date().toISOString(),
210
- freshnessWindowSeconds: 86_400, // 1 day — comfortably fresh
211
- freshnessFallback: "live",
212
- });
213
- await bind(manifestFile);
214
-
215
- const routed = await executedSql();
216
- expect(routed).not.toContain("data/orders.csv");
217
- expect(routed).toContain("order_summary");
218
-
219
- await revertToLive();
220
- },
221
- { timeout: 120_000 },
222
- );
223
-
224
- it(
225
- "serves live for a stale entry with fallback live",
226
- async () => {
227
- const manifestFile = await writeFreshnessManifest({
228
- // 2h old against a 1h window ⇒ stale.
229
- dataAsOf: new Date(Date.now() - 7200_000).toISOString(),
230
- freshnessWindowSeconds: 3600,
231
- freshnessFallback: "live",
232
- });
233
- await bind(manifestFile);
234
-
235
- const served = await executedSql();
236
- expect(served).toContain("data/orders.csv");
237
-
238
- await revertToLive();
239
- },
240
- { timeout: 120_000 },
241
- );
242
-
243
- it(
244
- "serves the stale table for a stale entry with fallback stale_ok",
245
- async () => {
246
- const manifestFile = await writeFreshnessManifest({
247
- dataAsOf: new Date(Date.now() - 7200_000).toISOString(),
248
- freshnessWindowSeconds: 3600,
249
- freshnessFallback: "stale_ok",
250
- });
251
- await bind(manifestFile);
252
-
253
- const served = await executedSql();
254
- expect(served).not.toContain("data/orders.csv");
255
- expect(served).toContain("order_summary");
256
-
257
- await revertToLive();
258
- },
259
- { timeout: 120_000 },
260
- );
261
-
262
- it(
263
- "serves live once a fresh entry crosses its window, with no rebind",
264
- async () => {
265
- const windowSeconds = 10;
266
- const anchor = Date.now();
267
- const manifestFile = await writeFreshnessManifest({
268
- dataAsOf: new Date(anchor).toISOString(),
269
- freshnessWindowSeconds: windowSeconds,
270
- freshnessFallback: "live",
271
- });
272
- await bind(manifestFile);
273
-
274
- // While inside the window, the served query routes to the table.
275
- expect(Date.now() - anchor).toBeLessThan(windowSeconds * 1000);
276
- const fresh = await executedSql();
277
- expect(fresh).not.toContain("data/orders.csv");
278
- expect(fresh).toContain("order_summary");
279
-
280
- // Wait for the window to elapse — no rebind, no reload.
281
- const waitMs = windowSeconds * 1000 - (Date.now() - anchor) + 1500;
282
- await new Promise((r) => setTimeout(r, Math.max(waitMs, 0)));
283
-
284
- // The very next query re-evaluates freshness and now serves live.
285
- const afterCrossing = await executedSql();
286
- expect(afterCrossing).toContain("data/orders.csv");
287
-
288
- await revertToLive();
289
- },
290
- { timeout: 120_000 },
291
- );
292
- });