@malloy-publisher/server 0.0.235 → 0.0.236
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +47 -17
- package/dist/package_load_worker.mjs +8 -1
- package/dist/server.mjs +471 -54
- package/package.json +5 -2
- package/.eslintrc.json +0 -44
- package/.prettierignore +0 -1
- package/.prettierrc +0 -4
- package/build.ts +0 -97
- package/eslint.config.mjs +0 -76
- package/k6-tests/README.md +0 -144
- package/k6-tests/bun.lock +0 -797
- package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
- package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
- package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
- package/k6-tests/load-test/load-test-crud.ts +0 -142
- package/k6-tests/load-test/load-test.ts +0 -784
- package/k6-tests/package.json +0 -27
- package/k6-tests/scripts/run-with-otel.sh +0 -50
- package/k6-tests/smoke-test/smoke-test.ts +0 -158
- package/k6-tests/tsconfig.json +0 -29
- package/k6-tests/utils/client_factory.ts +0 -128
- package/k6-tests/utils/common.ts +0 -489
- package/k6-tests/utils/logger.ts +0 -62
- package/k6-tests/utils/types.d.ts +0 -35
- package/malloy_mcp.dxt +0 -0
- package/publisher.config.example.bigquery.json +0 -33
- package/publisher.config.example.duckdb.json +0 -23
- package/publisher.config.json +0 -23
- package/scripts/bake-duckdb-extensions.js +0 -110
- package/src/config.spec.ts +0 -1535
- package/src/config.theme.spec.ts +0 -178
- package/src/config.ts +0 -1142
- package/src/constants.ts +0 -105
- package/src/controller/compile.controller.ts +0 -40
- package/src/controller/connection.controller.spec.ts +0 -849
- package/src/controller/connection.controller.ts +0 -901
- package/src/controller/database.controller.ts +0 -24
- package/src/controller/materialization.controller.spec.ts +0 -293
- package/src/controller/materialization.controller.ts +0 -319
- package/src/controller/model.controller.ts +0 -164
- package/src/controller/package.controller.spec.ts +0 -273
- package/src/controller/package.controller.ts +0 -306
- package/src/controller/query.controller.ts +0 -152
- package/src/controller/theme.controller.ts +0 -83
- package/src/controller/watch-mode.controller.ts +0 -243
- package/src/data_styles.ts +0 -71
- package/src/default-publisher.config.json +0 -23
- package/src/ducklake_version.spec.ts +0 -163
- package/src/ducklake_version.ts +0 -153
- package/src/errors.spec.ts +0 -119
- package/src/errors.ts +0 -237
- package/src/filter_deprecation.spec.ts +0 -64
- package/src/filter_deprecation.ts +0 -42
- package/src/health.spec.ts +0 -123
- package/src/health.ts +0 -233
- package/src/heap_check.spec.ts +0 -144
- package/src/heap_check.ts +0 -144
- package/src/instrumentation.ts +0 -214
- package/src/json_utils.spec.ts +0 -51
- package/src/json_utils.ts +0 -33
- package/src/logger.spec.ts +0 -210
- package/src/logger.ts +0 -251
- package/src/malloy_pin_prereqs.spec.ts +0 -25
- package/src/materialization_metrics.spec.ts +0 -169
- package/src/materialization_metrics.ts +0 -346
- package/src/mcp/error_messages.spec.ts +0 -35
- package/src/mcp/error_messages.ts +0 -183
- package/src/mcp/handler_utils.spec.ts +0 -108
- package/src/mcp/handler_utils.ts +0 -251
- package/src/mcp/mcp_constants.ts +0 -6
- package/src/mcp/query_envelope.spec.ts +0 -229
- package/src/mcp/query_envelope.ts +0 -240
- package/src/mcp/server.protocol.spec.ts +0 -250
- package/src/mcp/server.ts +0 -87
- package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
- package/src/mcp/skills/build_skills_bundle.ts +0 -176
- package/src/mcp/skills/skills_bundle.json +0 -1
- package/src/mcp/skills/skills_bundle.spec.ts +0 -171
- package/src/mcp/tool_response.spec.ts +0 -108
- package/src/mcp/tool_response.ts +0 -138
- package/src/mcp/tools/compile_tool.spec.ts +0 -315
- package/src/mcp/tools/compile_tool.ts +0 -208
- package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
- package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
- package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
- package/src/mcp/tools/docs_search_tool.ts +0 -128
- package/src/mcp/tools/embedding_index.spec.ts +0 -1236
- package/src/mcp/tools/embedding_index.ts +0 -808
- package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
- package/src/mcp/tools/execute_query_tool.ts +0 -316
- package/src/mcp/tools/get_context_eval.ts +0 -275
- package/src/mcp/tools/get_context_tool.spec.ts +0 -592
- package/src/mcp/tools/get_context_tool.ts +0 -585
- package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
- package/src/mcp/tools/reload_package_tool.ts +0 -132
- package/src/mcp_config.spec.ts +0 -919
- package/src/mcp_config.ts +0 -425
- package/src/oom_guards.integration.spec.ts +0 -269
- package/src/package_load/package_load_pool.spec.ts +0 -252
- package/src/package_load/package_load_pool.ts +0 -931
- package/src/package_load/package_load_worker.ts +0 -1034
- package/src/package_load/protocol.ts +0 -364
- package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
- package/src/package_load/rpc_wait_accountant.ts +0 -76
- package/src/package_load_metrics.spec.ts +0 -114
- package/src/package_load_metrics.ts +0 -127
- package/src/path_safety.spec.ts +0 -158
- package/src/path_safety.ts +0 -146
- package/src/pg_helpers.spec.ts +0 -223
- package/src/pg_helpers.ts +0 -52
- package/src/query_cap_metrics.spec.ts +0 -89
- package/src/query_cap_metrics.ts +0 -116
- package/src/query_concurrency.spec.ts +0 -247
- package/src/query_concurrency.ts +0 -237
- package/src/query_metadata_metrics.ts +0 -49
- package/src/query_param_utils.ts +0 -29
- package/src/query_timeout.spec.ts +0 -224
- package/src/query_timeout.ts +0 -179
- package/src/runtime/publisher.js +0 -328
- package/src/server-old.ts +0 -988
- package/src/server.ts +0 -2059
- package/src/service/annotations.spec.ts +0 -118
- package/src/service/annotations.ts +0 -91
- package/src/service/authorize.spec.ts +0 -154
- package/src/service/authorize.ts +0 -498
- package/src/service/authorize_integration.spec.ts +0 -2041
- package/src/service/build_plan.spec.ts +0 -685
- package/src/service/build_plan.ts +0 -797
- package/src/service/compile_authorize.spec.ts +0 -85
- package/src/service/compile_fragment_techniques.spec.ts +0 -156
- package/src/service/connection.spec.ts +0 -2360
- package/src/service/connection.ts +0 -2187
- package/src/service/connection_config.spec.ts +0 -734
- package/src/service/connection_config.ts +0 -899
- package/src/service/connection_federation.spec.ts +0 -184
- package/src/service/connection_fingerprint.spec.ts +0 -102
- package/src/service/connection_service.spec.ts +0 -711
- package/src/service/connection_service.ts +0 -291
- package/src/service/cron_evaluator.spec.ts +0 -79
- package/src/service/cron_evaluator.ts +0 -105
- package/src/service/db_utils.spec.ts +0 -1085
- package/src/service/db_utils.ts +0 -1502
- package/src/service/duckdb_instance_isolation.spec.ts +0 -137
- package/src/service/ducklake_lazy_attach.spec.ts +0 -110
- package/src/service/embedding_provider.spec.ts +0 -329
- package/src/service/embedding_provider.ts +0 -236
- package/src/service/environment.ts +0 -2191
- package/src/service/environment_admission.spec.ts +0 -344
- package/src/service/environment_compile.spec.ts +0 -197
- package/src/service/environment_store.spec.ts +0 -2274
- package/src/service/environment_store.ts +0 -2434
- package/src/service/environment_store_anchoring.spec.ts +0 -107
- package/src/service/environment_store_clone.spec.ts +0 -350
- package/src/service/explore_visibility.spec.ts +0 -434
- package/src/service/exports_probe.spec.ts +0 -107
- package/src/service/extension_fetch_policy.spec.ts +0 -256
- package/src/service/filter.spec.ts +0 -458
- package/src/service/filter.ts +0 -341
- package/src/service/filter_bypass.spec.ts +0 -418
- package/src/service/filter_integration.spec.ts +0 -1004
- package/src/service/freshness.spec.ts +0 -183
- package/src/service/freshness.ts +0 -112
- package/src/service/gcs_s3_utils.ts +0 -427
- package/src/service/given.ts +0 -105
- package/src/service/givens_integration.spec.ts +0 -219
- package/src/service/manifest_loader.spec.ts +0 -189
- package/src/service/manifest_loader.ts +0 -156
- package/src/service/materialization_build_session.spec.ts +0 -435
- package/src/service/materialization_build_session.ts +0 -681
- package/src/service/materialization_config_validation.spec.ts +0 -99
- package/src/service/materialization_config_validation.ts +0 -120
- package/src/service/materialization_eligibility.spec.ts +0 -158
- package/src/service/materialization_eligibility.ts +0 -305
- package/src/service/materialization_schedule_surface.spec.ts +0 -256
- package/src/service/materialization_scheduler.spec.ts +0 -289
- package/src/service/materialization_scheduler.ts +0 -304
- package/src/service/materialization_serve_transform.spec.ts +0 -1003
- package/src/service/materialization_serve_transform.ts +0 -779
- package/src/service/materialization_service.spec.ts +0 -2373
- package/src/service/materialization_service.ts +0 -2342
- package/src/service/materialization_test_fixtures.ts +0 -249
- package/src/service/model.spec.ts +0 -1157
- package/src/service/model.ts +0 -3370
- package/src/service/model_limits.spec.ts +0 -209
- package/src/service/model_limits.ts +0 -131
- package/src/service/model_storage_serve.spec.ts +0 -193
- package/src/service/model_storage_serve_joins.spec.ts +0 -193
- package/src/service/package.spec.ts +0 -612
- package/src/service/package.ts +0 -1799
- package/src/service/package_manifest.spec.ts +0 -230
- package/src/service/package_manifest.ts +0 -241
- package/src/service/package_memory_governor.spec.ts +0 -173
- package/src/service/package_memory_governor.ts +0 -233
- package/src/service/package_quote_bound_tables.spec.ts +0 -171
- package/src/service/package_race.spec.ts +0 -208
- package/src/service/package_reload_safety.spec.ts +0 -193
- package/src/service/package_rollback.spec.ts +0 -190
- package/src/service/package_worker_path.spec.ts +0 -645
- package/src/service/path_injection.spec.ts +0 -39
- package/src/service/persist_annotation_validation.spec.ts +0 -89
- package/src/service/persist_annotation_validation.ts +0 -59
- package/src/service/persistence_policy.spec.ts +0 -374
- package/src/service/proxy.spec.ts +0 -414
- package/src/service/proxy.ts +0 -248
- package/src/service/query_boundary.spec.ts +0 -470
- package/src/service/query_metadata.spec.ts +0 -408
- package/src/service/query_metadata.ts +0 -492
- package/src/service/query_metadata_identity.spec.ts +0 -149
- package/src/service/query_text.spec.ts +0 -79
- package/src/service/query_text.ts +0 -43
- package/src/service/quoting.spec.ts +0 -135
- package/src/service/quoting.ts +0 -78
- package/src/service/resolve_environment.ts +0 -15
- package/src/service/restricted_mode.spec.ts +0 -299
- package/src/service/source_extraction.ts +0 -225
- package/src/service/theme_store.ts +0 -199
- package/src/storage/DatabaseInterface.ts +0 -198
- package/src/storage/StorageManager.ts +0 -113
- package/src/storage/duckdb/ConnectionRepository.ts +0 -143
- package/src/storage/duckdb/DuckDBConnection.ts +0 -123
- package/src/storage/duckdb/DuckDBRepository.ts +0 -208
- package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
- package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
- package/src/storage/duckdb/MaterializationRepository.ts +0 -336
- package/src/storage/duckdb/PackageRepository.ts +0 -135
- package/src/storage/duckdb/schema.ts +0 -229
- package/src/stream_helpers.spec.ts +0 -280
- package/src/stream_helpers.ts +0 -162
- package/src/telemetry.ts +0 -20
- package/src/test_helpers/metrics_harness.ts +0 -166
- package/src/theme_key_parity.spec.ts +0 -57
- package/src/utils.ts +0 -29
- package/tests/fixtures/authorize-compile/model.malloy +0 -9
- package/tests/fixtures/authorize-compile/publisher.json +0 -4
- package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
- package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
- package/tests/fixtures/html-pages-test/data.csv +0 -3
- package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
- package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
- package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
- package/tests/fixtures/html-pages-test/public/data.json +0 -1
- package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
- package/tests/fixtures/html-pages-test/public/index.html +0 -9
- package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
- package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
- package/tests/fixtures/html-pages-test/public/slides.html +0 -12
- package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
- package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
- package/tests/fixtures/html-pages-test/publisher.json +0 -5
- package/tests/fixtures/html-pages-test/report.malloy +0 -1
- package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
- package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
- package/tests/fixtures/persist-multi-level/publisher.json +0 -5
- package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
- package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
- package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
- package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
- package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
- package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
- package/tests/fixtures/persist-test/data/orders.csv +0 -5
- package/tests/fixtures/persist-test/persist_test.malloy +0 -11
- package/tests/fixtures/persist-test/publisher.json +0 -5
- package/tests/fixtures/publisher.config.json +0 -15
- package/tests/fixtures/query-givens/data/orders.csv +0 -7
- package/tests/fixtures/query-givens/model.malloy +0 -34
- package/tests/fixtures/query-givens/publisher.json +0 -5
- package/tests/fixtures/xlsx/database.xlsx +0 -0
- package/tests/harness/mcp_test_setup.ts +0 -332
- package/tests/harness/rest_e2e.ts +0 -68
- package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
- package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
- package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
- package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
- package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
- package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
- package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
- package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
- package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
- package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
- package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
- package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
- package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
- package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
- package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
- package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
- package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
- package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
- package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
- package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
- package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
- package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
- package/tests/unit/controller/theme.controller.spec.ts +0 -141
- package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
- package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
- package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
- package/tests/unit/duckdb/repositories.test.ts +0 -208
- package/tests/unit/ducklake/ducklake.test.ts +0 -952
- package/tests/unit/service/theme_store.spec.ts +0 -139
- package/tsconfig.json +0 -32
|
@@ -1,259 +0,0 @@
|
|
|
1
|
-
/// <reference types="bun-types" />
|
|
2
|
-
|
|
3
|
-
// TODO: Remove this during projects cleanup
|
|
4
|
-
/**
|
|
5
|
-
* Smoke tests for the legacy `/api/v0/projects/...` REST surface registered
|
|
6
|
-
* by `server-old.ts`. These routes exist purely to keep pre-rename SDK
|
|
7
|
-
* clients (e.g. `@malloydata/db-publisher`) working after the
|
|
8
|
-
* projects→environments rename.
|
|
9
|
-
*
|
|
10
|
-
* One test per route group: projects CRUD, packages, connections, models,
|
|
11
|
-
* notebooks, databases, queries, materializations, manifest. The
|
|
12
|
-
* materialization test additionally asserts the response field rename
|
|
13
|
-
* (`projectId` not `environmentId`). `/status` is no longer in the legacy
|
|
14
|
-
* surface — both old and new clients hit the single `/api/v0/status`
|
|
15
|
-
* handler in server.ts, which returns `environments`.
|
|
16
|
-
*
|
|
17
|
-
* This file is intentionally separate from the regular integration suite so
|
|
18
|
-
* it can be deleted in one motion when legacy support is dropped.
|
|
19
|
-
*/
|
|
20
|
-
|
|
21
|
-
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
|
22
|
-
import path from "path";
|
|
23
|
-
import { fileURLToPath } from "url";
|
|
24
|
-
import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
|
|
25
|
-
|
|
26
|
-
const __filename = fileURLToPath(import.meta.url);
|
|
27
|
-
const __dirname = path.dirname(__filename);
|
|
28
|
-
|
|
29
|
-
// Use a distinct project name so this suite doesn't collide with the
|
|
30
|
-
// materialization integration suite (which also uses "test-project") if
|
|
31
|
-
// they ever run in the same DB instance.
|
|
32
|
-
const PROJECT_NAME = "legacy-routes-test-project";
|
|
33
|
-
const PACKAGE_NAME = "persist-test";
|
|
34
|
-
|
|
35
|
-
describe("Legacy /api/v0/projects/* REST routes (E2E)", () => {
|
|
36
|
-
let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
|
|
37
|
-
let baseUrl: string;
|
|
38
|
-
|
|
39
|
-
beforeAll(async () => {
|
|
40
|
-
env = await startRestE2E();
|
|
41
|
-
baseUrl = env.baseUrl;
|
|
42
|
-
|
|
43
|
-
// Create the test environment via the LEGACY route — proves POST
|
|
44
|
-
// /projects works end-to-end.
|
|
45
|
-
const fixtureDir = path.resolve(__dirname, "../../fixtures/persist-test");
|
|
46
|
-
const createRes = await fetch(`${baseUrl}/api/v0/projects`, {
|
|
47
|
-
method: "POST",
|
|
48
|
-
headers: { "Content-Type": "application/json" },
|
|
49
|
-
body: JSON.stringify({
|
|
50
|
-
name: PROJECT_NAME,
|
|
51
|
-
packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
|
|
52
|
-
connections: [],
|
|
53
|
-
}),
|
|
54
|
-
});
|
|
55
|
-
if (!createRes.ok) {
|
|
56
|
-
const body = await createRes.text();
|
|
57
|
-
throw new Error(
|
|
58
|
-
`Failed to create test project via legacy route (${createRes.status}): ${body}`,
|
|
59
|
-
);
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
// Wait for the package to finish loading via the legacy GET path.
|
|
63
|
-
const deadline = Date.now() + 30_000;
|
|
64
|
-
let pkgReady = false;
|
|
65
|
-
while (!pkgReady && Date.now() < deadline) {
|
|
66
|
-
try {
|
|
67
|
-
const res = await fetch(
|
|
68
|
-
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}`,
|
|
69
|
-
);
|
|
70
|
-
if (res.ok) {
|
|
71
|
-
pkgReady = true;
|
|
72
|
-
break;
|
|
73
|
-
}
|
|
74
|
-
} catch {
|
|
75
|
-
// not ready yet
|
|
76
|
-
}
|
|
77
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
78
|
-
}
|
|
79
|
-
if (!pkgReady) {
|
|
80
|
-
throw new Error("Test package did not become available in time");
|
|
81
|
-
}
|
|
82
|
-
});
|
|
83
|
-
|
|
84
|
-
afterAll(async () => {
|
|
85
|
-
if (baseUrl) {
|
|
86
|
-
try {
|
|
87
|
-
// Clean up via the legacy DELETE route.
|
|
88
|
-
await fetch(`${baseUrl}/api/v0/projects/${PROJECT_NAME}`, {
|
|
89
|
-
method: "DELETE",
|
|
90
|
-
});
|
|
91
|
-
} catch {
|
|
92
|
-
// best-effort cleanup
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
await env?.stop();
|
|
96
|
-
env = null;
|
|
97
|
-
});
|
|
98
|
-
|
|
99
|
-
describe("projects CRUD", () => {
|
|
100
|
-
it("GET /projects lists environments under the legacy URL", async () => {
|
|
101
|
-
const res = await fetch(`${baseUrl}/api/v0/projects`);
|
|
102
|
-
expect(res.status).toBe(200);
|
|
103
|
-
const body = (await res.json()) as Array<{ name?: string }>;
|
|
104
|
-
expect(Array.isArray(body)).toBe(true);
|
|
105
|
-
expect(body.some((e) => e.name === PROJECT_NAME)).toBe(true);
|
|
106
|
-
});
|
|
107
|
-
|
|
108
|
-
it("GET /projects/:projectName returns the project", async () => {
|
|
109
|
-
const res = await fetch(`${baseUrl}/api/v0/projects/${PROJECT_NAME}`);
|
|
110
|
-
expect(res.status).toBe(200);
|
|
111
|
-
const body = (await res.json()) as { name?: string };
|
|
112
|
-
expect(body.name).toBe(PROJECT_NAME);
|
|
113
|
-
});
|
|
114
|
-
});
|
|
115
|
-
|
|
116
|
-
describe("packages", () => {
|
|
117
|
-
it("GET /projects/:projectName/packages returns the package list", async () => {
|
|
118
|
-
const res = await fetch(
|
|
119
|
-
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages`,
|
|
120
|
-
);
|
|
121
|
-
expect(res.status).toBe(200);
|
|
122
|
-
const body = (await res.json()) as Array<{ name?: string }>;
|
|
123
|
-
expect(Array.isArray(body)).toBe(true);
|
|
124
|
-
expect(body.some((p) => p.name === PACKAGE_NAME)).toBe(true);
|
|
125
|
-
});
|
|
126
|
-
|
|
127
|
-
it("GET /projects/:projectName/packages/:packageName returns the package", async () => {
|
|
128
|
-
const res = await fetch(
|
|
129
|
-
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}`,
|
|
130
|
-
);
|
|
131
|
-
expect(res.status).toBe(200);
|
|
132
|
-
const body = (await res.json()) as { name?: string };
|
|
133
|
-
expect(body.name).toBe(PACKAGE_NAME);
|
|
134
|
-
});
|
|
135
|
-
});
|
|
136
|
-
|
|
137
|
-
describe("connections", () => {
|
|
138
|
-
it("GET /projects/:projectName/connections returns 200 (may be empty)", async () => {
|
|
139
|
-
const res = await fetch(
|
|
140
|
-
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/connections`,
|
|
141
|
-
);
|
|
142
|
-
expect(res.status).toBe(200);
|
|
143
|
-
const body = await res.json();
|
|
144
|
-
expect(Array.isArray(body)).toBe(true);
|
|
145
|
-
});
|
|
146
|
-
});
|
|
147
|
-
|
|
148
|
-
describe("models", () => {
|
|
149
|
-
it("GET /projects/:projectName/packages/:packageName/models returns the model list", async () => {
|
|
150
|
-
const res = await fetch(
|
|
151
|
-
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/models`,
|
|
152
|
-
);
|
|
153
|
-
expect(res.status).toBe(200);
|
|
154
|
-
const body = await res.json();
|
|
155
|
-
expect(Array.isArray(body)).toBe(true);
|
|
156
|
-
});
|
|
157
|
-
});
|
|
158
|
-
|
|
159
|
-
describe("notebooks", () => {
|
|
160
|
-
it("GET /projects/:projectName/packages/:packageName/notebooks returns 200", async () => {
|
|
161
|
-
const res = await fetch(
|
|
162
|
-
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/notebooks`,
|
|
163
|
-
);
|
|
164
|
-
expect(res.status).toBe(200);
|
|
165
|
-
const body = await res.json();
|
|
166
|
-
expect(Array.isArray(body)).toBe(true);
|
|
167
|
-
});
|
|
168
|
-
});
|
|
169
|
-
|
|
170
|
-
describe("databases", () => {
|
|
171
|
-
it("GET /projects/:projectName/packages/:packageName/databases returns 200", async () => {
|
|
172
|
-
const res = await fetch(
|
|
173
|
-
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/databases`,
|
|
174
|
-
);
|
|
175
|
-
expect(res.status).toBe(200);
|
|
176
|
-
const body = await res.json();
|
|
177
|
-
expect(Array.isArray(body)).toBe(true);
|
|
178
|
-
});
|
|
179
|
-
});
|
|
180
|
-
|
|
181
|
-
describe("queries", () => {
|
|
182
|
-
it("POST /projects/:projectName/packages/:packageName/models/.../query reaches the handler", async () => {
|
|
183
|
-
// Hit the route with a bogus model name. We only need to prove the
|
|
184
|
-
// legacy URL is wired up to the controller — a structured JSON
|
|
185
|
-
// error (not Express's HTML fall-through 404) is sufficient signal.
|
|
186
|
-
const res = await fetch(
|
|
187
|
-
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/models/does-not-exist.malloy/query`,
|
|
188
|
-
{
|
|
189
|
-
method: "POST",
|
|
190
|
-
headers: { "Content-Type": "application/json" },
|
|
191
|
-
body: JSON.stringify({ query: "run: nothing" }),
|
|
192
|
-
},
|
|
193
|
-
);
|
|
194
|
-
expect(res.status).toBeGreaterThanOrEqual(400);
|
|
195
|
-
expect(res.status).toBeLessThan(600);
|
|
196
|
-
// Controller errors come back as JSON with a `message` field.
|
|
197
|
-
// An unhandled Express 404 returns HTML — that would fail here.
|
|
198
|
-
const body = (await res.json()) as Record<string, unknown>;
|
|
199
|
-
expect(typeof body.message).toBe("string");
|
|
200
|
-
});
|
|
201
|
-
});
|
|
202
|
-
|
|
203
|
-
describe("materializations", () => {
|
|
204
|
-
it("GET list and POST create return 'projectId' (not 'environmentId') under the legacy URL", async () => {
|
|
205
|
-
const listRes = await fetch(
|
|
206
|
-
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/materializations`,
|
|
207
|
-
);
|
|
208
|
-
expect(listRes.status).toBe(200);
|
|
209
|
-
const list = (await listRes.json()) as unknown;
|
|
210
|
-
expect(Array.isArray(list)).toBe(true);
|
|
211
|
-
|
|
212
|
-
// Create one so we can assert the field rename on a populated payload.
|
|
213
|
-
const createRes = await fetch(
|
|
214
|
-
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/materializations`,
|
|
215
|
-
{
|
|
216
|
-
method: "POST",
|
|
217
|
-
headers: { "Content-Type": "application/json" },
|
|
218
|
-
body: JSON.stringify({ autoLoadManifest: true }),
|
|
219
|
-
},
|
|
220
|
-
);
|
|
221
|
-
expect(createRes.status).toBe(201);
|
|
222
|
-
const created = (await createRes.json()) as Record<string, unknown>;
|
|
223
|
-
|
|
224
|
-
// Legacy contract: materialization payloads expose `projectId`, not
|
|
225
|
-
// `environmentId`. This is the response remapper in server-old.ts.
|
|
226
|
-
expect(created).toHaveProperty("projectId");
|
|
227
|
-
expect(created).not.toHaveProperty("environmentId");
|
|
228
|
-
|
|
229
|
-
const id = created.id as string;
|
|
230
|
-
// Best-effort cleanup so we don't leak a PENDING materialization
|
|
231
|
-
// into other tests. We don't poll-to-terminal; the suite teardown
|
|
232
|
-
// of the project will mop up.
|
|
233
|
-
try {
|
|
234
|
-
await fetch(
|
|
235
|
-
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/materializations/${id}?action=stop`,
|
|
236
|
-
{ method: "POST" },
|
|
237
|
-
);
|
|
238
|
-
await fetch(
|
|
239
|
-
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/materializations/${id}`,
|
|
240
|
-
{ method: "DELETE" },
|
|
241
|
-
);
|
|
242
|
-
} catch {
|
|
243
|
-
// ignore
|
|
244
|
-
}
|
|
245
|
-
});
|
|
246
|
-
});
|
|
247
|
-
|
|
248
|
-
describe("manifest", () => {
|
|
249
|
-
it("GET /projects/:projectName/packages/:packageName/manifest returns 200 or a structured 4xx", async () => {
|
|
250
|
-
const res = await fetch(
|
|
251
|
-
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/manifest`,
|
|
252
|
-
);
|
|
253
|
-
// Without a built materialization the manifest may be empty/404 —
|
|
254
|
-
// we only assert the legacy URL reaches the handler, not 404 from
|
|
255
|
-
// Express's catch-all.
|
|
256
|
-
expect([200, 400, 404]).toContain(res.status);
|
|
257
|
-
});
|
|
258
|
-
});
|
|
259
|
-
});
|
|
@@ -1,133 +0,0 @@
|
|
|
1
|
-
/// <reference types="bun-types" />
|
|
2
|
-
|
|
3
|
-
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
|
4
|
-
import path from "path";
|
|
5
|
-
import { fileURLToPath } from "url";
|
|
6
|
-
import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
|
|
7
|
-
|
|
8
|
-
const __filename = fileURLToPath(import.meta.url);
|
|
9
|
-
const __dirname = path.dirname(__filename);
|
|
10
|
-
|
|
11
|
-
const PROJECT = "env-mat-list-project";
|
|
12
|
-
const PKG_A = "persist-test";
|
|
13
|
-
const PKG_B = "persist-schedule-test";
|
|
14
|
-
const ENV_MATS_PATH = `/api/v0/environments/${PROJECT}/packages/materializations`;
|
|
15
|
-
|
|
16
|
-
// Verifies the environment-scoped list:
|
|
17
|
-
// GET /environments/{env}/packages/materializations returns runs from every
|
|
18
|
-
// package in the env, newest-first, each labeled with its packageName.
|
|
19
|
-
describe("Environment-scoped materialization list", () => {
|
|
20
|
-
let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
|
|
21
|
-
let baseUrl: string;
|
|
22
|
-
|
|
23
|
-
const fixture = (name: string) =>
|
|
24
|
-
path.resolve(__dirname, `../../fixtures/${name}`);
|
|
25
|
-
|
|
26
|
-
async function createAndFinish(pkg: string): Promise<string> {
|
|
27
|
-
const res = await fetch(
|
|
28
|
-
`${baseUrl}/api/v0/environments/${PROJECT}/packages/${pkg}/materializations`,
|
|
29
|
-
{
|
|
30
|
-
method: "POST",
|
|
31
|
-
headers: { "Content-Type": "application/json" },
|
|
32
|
-
body: JSON.stringify({ forceRefresh: true }),
|
|
33
|
-
},
|
|
34
|
-
);
|
|
35
|
-
expect(res.status).toBe(201);
|
|
36
|
-
const created = (await res.json()) as { id: string };
|
|
37
|
-
const deadline = Date.now() + 60_000;
|
|
38
|
-
while (Date.now() < deadline) {
|
|
39
|
-
const poll = await fetch(
|
|
40
|
-
`${baseUrl}/api/v0/environments/${PROJECT}/packages/${pkg}/materializations/${created.id}`,
|
|
41
|
-
);
|
|
42
|
-
const m = (await poll.json()) as { status: string };
|
|
43
|
-
if (
|
|
44
|
-
["MANIFEST_FILE_READY", "FAILED", "CANCELLED"].includes(m.status)
|
|
45
|
-
) {
|
|
46
|
-
break;
|
|
47
|
-
}
|
|
48
|
-
await new Promise((r) => setTimeout(r, 250));
|
|
49
|
-
}
|
|
50
|
-
return created.id;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
beforeAll(async () => {
|
|
54
|
-
env = await startRestE2E();
|
|
55
|
-
baseUrl = env.baseUrl;
|
|
56
|
-
|
|
57
|
-
const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
|
|
58
|
-
method: "POST",
|
|
59
|
-
headers: { "Content-Type": "application/json" },
|
|
60
|
-
body: JSON.stringify({
|
|
61
|
-
name: PROJECT,
|
|
62
|
-
packages: [
|
|
63
|
-
{ name: PKG_A, location: fixture(PKG_A) },
|
|
64
|
-
{ name: PKG_B, location: fixture(PKG_B) },
|
|
65
|
-
],
|
|
66
|
-
connections: [],
|
|
67
|
-
}),
|
|
68
|
-
});
|
|
69
|
-
if (!createRes.ok) {
|
|
70
|
-
throw new Error(
|
|
71
|
-
`Failed to create project (${createRes.status}): ${await createRes.text()}`,
|
|
72
|
-
);
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
// Wait for both packages to load.
|
|
76
|
-
for (const pkg of [PKG_A, PKG_B]) {
|
|
77
|
-
const deadline = Date.now() + 30_000;
|
|
78
|
-
while (Date.now() < deadline) {
|
|
79
|
-
const res = await fetch(
|
|
80
|
-
`${baseUrl}/api/v0/environments/${PROJECT}/packages/${pkg}`,
|
|
81
|
-
);
|
|
82
|
-
if (res.ok) break;
|
|
83
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
});
|
|
87
|
-
|
|
88
|
-
afterAll(async () => {
|
|
89
|
-
if (baseUrl) {
|
|
90
|
-
try {
|
|
91
|
-
await fetch(`${baseUrl}/api/v0/environments/${PROJECT}`, {
|
|
92
|
-
method: "DELETE",
|
|
93
|
-
});
|
|
94
|
-
} catch {
|
|
95
|
-
// best-effort
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
await env?.stop();
|
|
99
|
-
env = null;
|
|
100
|
-
});
|
|
101
|
-
|
|
102
|
-
it("returns materializations from every package in the environment", async () => {
|
|
103
|
-
await createAndFinish(PKG_A);
|
|
104
|
-
await createAndFinish(PKG_B);
|
|
105
|
-
|
|
106
|
-
const res = await fetch(`${baseUrl}${ENV_MATS_PATH}`);
|
|
107
|
-
expect(res.status).toBe(200);
|
|
108
|
-
const rows = (await res.json()) as Array<{
|
|
109
|
-
packageName: string;
|
|
110
|
-
createdAt?: string;
|
|
111
|
-
}>;
|
|
112
|
-
|
|
113
|
-
const packages = new Set(rows.map((r) => r.packageName));
|
|
114
|
-
expect(packages.has(PKG_A)).toBe(true);
|
|
115
|
-
expect(packages.has(PKG_B)).toBe(true);
|
|
116
|
-
|
|
117
|
-
// Newest-first ordering.
|
|
118
|
-
const times = rows
|
|
119
|
-
.map((r) => (r.createdAt ? new Date(r.createdAt).getTime() : 0))
|
|
120
|
-
.filter((t) => t > 0);
|
|
121
|
-
const sorted = [...times].sort((a, b) => b - a);
|
|
122
|
-
expect(times).toEqual(sorted);
|
|
123
|
-
});
|
|
124
|
-
|
|
125
|
-
it("does not shadow the per-package list route", async () => {
|
|
126
|
-
const res = await fetch(
|
|
127
|
-
`${baseUrl}/api/v0/environments/${PROJECT}/packages/${PKG_A}/materializations`,
|
|
128
|
-
);
|
|
129
|
-
expect(res.status).toBe(200);
|
|
130
|
-
const rows = (await res.json()) as Array<{ packageName: string }>;
|
|
131
|
-
expect(rows.every((r) => r.packageName === PKG_A)).toBe(true);
|
|
132
|
-
});
|
|
133
|
-
});
|
|
@@ -1,292 +0,0 @@
|
|
|
1
|
-
/// <reference types="bun-types" />
|
|
2
|
-
|
|
3
|
-
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
|
4
|
-
import * as fsp from "fs/promises";
|
|
5
|
-
import os from "os";
|
|
6
|
-
import path from "path";
|
|
7
|
-
import { fileURLToPath } from "url";
|
|
8
|
-
import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
|
|
9
|
-
|
|
10
|
-
const __filename = fileURLToPath(import.meta.url);
|
|
11
|
-
const __dirname = path.dirname(__filename);
|
|
12
|
-
|
|
13
|
-
const PROJECT_NAME = "freshness-gate-project";
|
|
14
|
-
const PACKAGE_NAME = "persist-test";
|
|
15
|
-
const MODEL_PATH = "persist_test.malloy";
|
|
16
|
-
const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
|
|
17
|
-
|
|
18
|
-
/**
|
|
19
|
-
* Covers the per-query freshness gate (persistence.md §9.3, §14 Phase B): a
|
|
20
|
-
* query on a `#@ persist` source uses its materialized table only while the
|
|
21
|
-
* table is within its declared freshness window; otherwise it falls back per the
|
|
22
|
-
* entry's declared `fallback`.
|
|
23
|
-
*
|
|
24
|
-
* - fresh (or un-gated) → routes to the materialized table
|
|
25
|
-
* - stale + fallback live → serves live SQL (entry dropped)
|
|
26
|
-
* - stale + fallback stale_ok→ serves the stale table
|
|
27
|
-
* - a fresh entry that crosses its window → serves live on the NEXT query,
|
|
28
|
-
* with no rebind (proves per-query re-evaluation, not bind-time filtering)
|
|
29
|
-
*
|
|
30
|
-
* The manifest is bound once physically (auto-run builds the table), then a
|
|
31
|
-
* manifest URI keyed by the source's real sourceEntityId routes served queries.
|
|
32
|
-
* `executedSql()` is the routing evidence: it scans `order_summary` when routed
|
|
33
|
-
* to the table, and `data/orders.csv` when serving live.
|
|
34
|
-
*/
|
|
35
|
-
describe("Per-query freshness gate (E2E)", () => {
|
|
36
|
-
let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
|
|
37
|
-
let baseUrl: string;
|
|
38
|
-
let tmpDir: string;
|
|
39
|
-
let sourceEntityId: string;
|
|
40
|
-
|
|
41
|
-
beforeAll(async () => {
|
|
42
|
-
env = await startRestE2E();
|
|
43
|
-
baseUrl = env.baseUrl;
|
|
44
|
-
tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "freshness-gate-"));
|
|
45
|
-
|
|
46
|
-
const fixtureDir = path.resolve(__dirname, "../../fixtures/persist-test");
|
|
47
|
-
const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
|
|
48
|
-
method: "POST",
|
|
49
|
-
headers: { "Content-Type": "application/json" },
|
|
50
|
-
body: JSON.stringify({
|
|
51
|
-
name: PROJECT_NAME,
|
|
52
|
-
packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
|
|
53
|
-
connections: [],
|
|
54
|
-
}),
|
|
55
|
-
});
|
|
56
|
-
if (!createRes.ok) {
|
|
57
|
-
throw new Error(
|
|
58
|
-
`Failed to create test project (${createRes.status}): ${await createRes.text()}`,
|
|
59
|
-
);
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
const deadline = Date.now() + 30_000;
|
|
63
|
-
while (Date.now() < deadline) {
|
|
64
|
-
const res = await fetch(`${baseUrl}${API}`);
|
|
65
|
-
if (res.ok) break;
|
|
66
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
// Physically build the persist source (auto-run self-assigns
|
|
70
|
-
// physicalTableName = "order_summary"), then revert to live so the
|
|
71
|
-
// manifest-URI bind is the only thing that can route the served query.
|
|
72
|
-
await buildTableThenRevertToLive();
|
|
73
|
-
sourceEntityId = await orderSummarySourceEntityId();
|
|
74
|
-
}, 120_000);
|
|
75
|
-
|
|
76
|
-
afterAll(async () => {
|
|
77
|
-
if (baseUrl) {
|
|
78
|
-
await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
|
|
79
|
-
method: "DELETE",
|
|
80
|
-
}).catch(() => {});
|
|
81
|
-
}
|
|
82
|
-
await env?.stop();
|
|
83
|
-
await fsp.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
|
|
84
|
-
env = null;
|
|
85
|
-
});
|
|
86
|
-
|
|
87
|
-
function url(p: string): string {
|
|
88
|
-
return `${baseUrl}${API}${p}`;
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
const ROUTING_QUERY = "run: order_summary -> { aggregate: c is count() }";
|
|
92
|
-
|
|
93
|
-
/** The SQL the served query actually compiled to (routing evidence). */
|
|
94
|
-
async function executedSql(): Promise<string> {
|
|
95
|
-
const res = await fetch(`${baseUrl}${API}/models/${MODEL_PATH}/query`, {
|
|
96
|
-
method: "POST",
|
|
97
|
-
headers: { "Content-Type": "application/json" },
|
|
98
|
-
body: JSON.stringify({ query: ROUTING_QUERY }),
|
|
99
|
-
});
|
|
100
|
-
expect(res.status).toBe(200);
|
|
101
|
-
const body = (await res.json()) as { result: string };
|
|
102
|
-
return (JSON.parse(body.result) as { sql: string }).sql;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
async function orderSummarySourceEntityId(): Promise<string> {
|
|
106
|
-
const res = await fetch(url(""));
|
|
107
|
-
expect(res.status).toBe(200);
|
|
108
|
-
const pkg = (await res.json()) as {
|
|
109
|
-
buildPlan?: { sources?: Record<string, { sourceEntityId?: string }> };
|
|
110
|
-
};
|
|
111
|
-
const sources = pkg.buildPlan?.sources ?? {};
|
|
112
|
-
const id = Object.values(sources)[0]?.sourceEntityId;
|
|
113
|
-
expect(typeof id).toBe("string");
|
|
114
|
-
return id as string;
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
async function pollUntilTerminal(
|
|
118
|
-
id: string,
|
|
119
|
-
timeoutMs = 90_000,
|
|
120
|
-
): Promise<Record<string, unknown>> {
|
|
121
|
-
const terminal = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"];
|
|
122
|
-
const deadline = Date.now() + timeoutMs;
|
|
123
|
-
while (Date.now() < deadline) {
|
|
124
|
-
const res = await fetch(url(`/materializations/${id}`));
|
|
125
|
-
expect(res.status).toBe(200);
|
|
126
|
-
const data = (await res.json()) as Record<string, unknown>;
|
|
127
|
-
if (terminal.includes(data.status as string)) return data;
|
|
128
|
-
await new Promise((r) => setTimeout(r, 250));
|
|
129
|
-
}
|
|
130
|
-
throw new Error(`Materialization ${id} did not reach a terminal state`);
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
async function buildTableThenRevertToLive(): Promise<void> {
|
|
134
|
-
const createRes = await fetch(url("/materializations"), {
|
|
135
|
-
method: "POST",
|
|
136
|
-
headers: { "Content-Type": "application/json" },
|
|
137
|
-
body: JSON.stringify({}),
|
|
138
|
-
});
|
|
139
|
-
expect(createRes.status).toBe(201);
|
|
140
|
-
const { id } = (await createRes.json()) as { id: string };
|
|
141
|
-
const built = await pollUntilTerminal(id);
|
|
142
|
-
expect(built.status).toBe("MANIFEST_FILE_READY");
|
|
143
|
-
await fetch(url("?reload=true"));
|
|
144
|
-
await fetch(url(`/materializations/${id}`), { method: "DELETE" });
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
/** Write a CP-shaped manifest carrying the freshness fields. */
|
|
148
|
-
async function writeFreshnessManifest(opts: {
|
|
149
|
-
dataAsOf: string;
|
|
150
|
-
freshnessWindowSeconds: number;
|
|
151
|
-
freshnessFallback: "live" | "stale_ok" | "fail";
|
|
152
|
-
}): Promise<string> {
|
|
153
|
-
const file = path.join(
|
|
154
|
-
tmpDir,
|
|
155
|
-
`freshness-${Date.now()}-${Math.random()}.json`,
|
|
156
|
-
);
|
|
157
|
-
await fsp.writeFile(
|
|
158
|
-
file,
|
|
159
|
-
JSON.stringify({
|
|
160
|
-
builtAt: new Date().toISOString(),
|
|
161
|
-
strict: false,
|
|
162
|
-
entries: {
|
|
163
|
-
[sourceEntityId]: {
|
|
164
|
-
sourceEntityId,
|
|
165
|
-
sourceName: "order_summary",
|
|
166
|
-
physicalTableName: "order_summary",
|
|
167
|
-
connectionName: "duckdb",
|
|
168
|
-
dataAsOf: opts.dataAsOf,
|
|
169
|
-
freshnessWindowSeconds: opts.freshnessWindowSeconds,
|
|
170
|
-
freshnessFallback: opts.freshnessFallback,
|
|
171
|
-
},
|
|
172
|
-
},
|
|
173
|
-
}),
|
|
174
|
-
"utf8",
|
|
175
|
-
);
|
|
176
|
-
return file;
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
async function bind(manifestFile: string): Promise<void> {
|
|
180
|
-
const patchRes = await fetch(url(""), {
|
|
181
|
-
method: "PATCH",
|
|
182
|
-
headers: { "Content-Type": "application/json" },
|
|
183
|
-
body: JSON.stringify({
|
|
184
|
-
name: PACKAGE_NAME,
|
|
185
|
-
manifestLocation: manifestFile,
|
|
186
|
-
}),
|
|
187
|
-
});
|
|
188
|
-
expect(patchRes.status).toBe(200);
|
|
189
|
-
const patched = (await patchRes.json()) as Record<string, unknown>;
|
|
190
|
-
// The bind is recorded regardless of per-query freshness — freshness is
|
|
191
|
-
// evaluated per query, not at bind.
|
|
192
|
-
expect(patched.manifestBindingStatus).toBe("bound");
|
|
193
|
-
expect(patched.manifestEntryCount).toBe(1);
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
async function revertToLive(): Promise<void> {
|
|
197
|
-
await fetch(url(""), {
|
|
198
|
-
method: "PATCH",
|
|
199
|
-
headers: { "Content-Type": "application/json" },
|
|
200
|
-
body: JSON.stringify({ name: PACKAGE_NAME, manifestLocation: null }),
|
|
201
|
-
});
|
|
202
|
-
await fetch(url("?reload=true"));
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
it(
|
|
206
|
-
"routes a fresh entry to the materialized table",
|
|
207
|
-
async () => {
|
|
208
|
-
const manifestFile = await writeFreshnessManifest({
|
|
209
|
-
dataAsOf: new Date().toISOString(),
|
|
210
|
-
freshnessWindowSeconds: 86_400, // 1 day — comfortably fresh
|
|
211
|
-
freshnessFallback: "live",
|
|
212
|
-
});
|
|
213
|
-
await bind(manifestFile);
|
|
214
|
-
|
|
215
|
-
const routed = await executedSql();
|
|
216
|
-
expect(routed).not.toContain("data/orders.csv");
|
|
217
|
-
expect(routed).toContain("order_summary");
|
|
218
|
-
|
|
219
|
-
await revertToLive();
|
|
220
|
-
},
|
|
221
|
-
{ timeout: 120_000 },
|
|
222
|
-
);
|
|
223
|
-
|
|
224
|
-
it(
|
|
225
|
-
"serves live for a stale entry with fallback live",
|
|
226
|
-
async () => {
|
|
227
|
-
const manifestFile = await writeFreshnessManifest({
|
|
228
|
-
// 2h old against a 1h window ⇒ stale.
|
|
229
|
-
dataAsOf: new Date(Date.now() - 7200_000).toISOString(),
|
|
230
|
-
freshnessWindowSeconds: 3600,
|
|
231
|
-
freshnessFallback: "live",
|
|
232
|
-
});
|
|
233
|
-
await bind(manifestFile);
|
|
234
|
-
|
|
235
|
-
const served = await executedSql();
|
|
236
|
-
expect(served).toContain("data/orders.csv");
|
|
237
|
-
|
|
238
|
-
await revertToLive();
|
|
239
|
-
},
|
|
240
|
-
{ timeout: 120_000 },
|
|
241
|
-
);
|
|
242
|
-
|
|
243
|
-
it(
|
|
244
|
-
"serves the stale table for a stale entry with fallback stale_ok",
|
|
245
|
-
async () => {
|
|
246
|
-
const manifestFile = await writeFreshnessManifest({
|
|
247
|
-
dataAsOf: new Date(Date.now() - 7200_000).toISOString(),
|
|
248
|
-
freshnessWindowSeconds: 3600,
|
|
249
|
-
freshnessFallback: "stale_ok",
|
|
250
|
-
});
|
|
251
|
-
await bind(manifestFile);
|
|
252
|
-
|
|
253
|
-
const served = await executedSql();
|
|
254
|
-
expect(served).not.toContain("data/orders.csv");
|
|
255
|
-
expect(served).toContain("order_summary");
|
|
256
|
-
|
|
257
|
-
await revertToLive();
|
|
258
|
-
},
|
|
259
|
-
{ timeout: 120_000 },
|
|
260
|
-
);
|
|
261
|
-
|
|
262
|
-
it(
|
|
263
|
-
"serves live once a fresh entry crosses its window, with no rebind",
|
|
264
|
-
async () => {
|
|
265
|
-
const windowSeconds = 10;
|
|
266
|
-
const anchor = Date.now();
|
|
267
|
-
const manifestFile = await writeFreshnessManifest({
|
|
268
|
-
dataAsOf: new Date(anchor).toISOString(),
|
|
269
|
-
freshnessWindowSeconds: windowSeconds,
|
|
270
|
-
freshnessFallback: "live",
|
|
271
|
-
});
|
|
272
|
-
await bind(manifestFile);
|
|
273
|
-
|
|
274
|
-
// While inside the window, the served query routes to the table.
|
|
275
|
-
expect(Date.now() - anchor).toBeLessThan(windowSeconds * 1000);
|
|
276
|
-
const fresh = await executedSql();
|
|
277
|
-
expect(fresh).not.toContain("data/orders.csv");
|
|
278
|
-
expect(fresh).toContain("order_summary");
|
|
279
|
-
|
|
280
|
-
// Wait for the window to elapse — no rebind, no reload.
|
|
281
|
-
const waitMs = windowSeconds * 1000 - (Date.now() - anchor) + 1500;
|
|
282
|
-
await new Promise((r) => setTimeout(r, Math.max(waitMs, 0)));
|
|
283
|
-
|
|
284
|
-
// The very next query re-evaluates freshness and now serves live.
|
|
285
|
-
const afterCrossing = await executedSql();
|
|
286
|
-
expect(afterCrossing).toContain("data/orders.csv");
|
|
287
|
-
|
|
288
|
-
await revertToLive();
|
|
289
|
-
},
|
|
290
|
-
{ timeout: 120_000 },
|
|
291
|
-
);
|
|
292
|
-
});
|