@malloy-publisher/server 0.0.235 → 0.0.236
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +47 -17
- package/dist/package_load_worker.mjs +8 -1
- package/dist/server.mjs +471 -54
- package/package.json +5 -2
- package/.eslintrc.json +0 -44
- package/.prettierignore +0 -1
- package/.prettierrc +0 -4
- package/build.ts +0 -97
- package/eslint.config.mjs +0 -76
- package/k6-tests/README.md +0 -144
- package/k6-tests/bun.lock +0 -797
- package/k6-tests/load-test/load-test-crud-connections.ts +0 -263
- package/k6-tests/load-test/load-test-crud-packages.ts +0 -461
- package/k6-tests/load-test/load-test-crud-projects.ts +0 -160
- package/k6-tests/load-test/load-test-crud.ts +0 -142
- package/k6-tests/load-test/load-test.ts +0 -784
- package/k6-tests/package.json +0 -27
- package/k6-tests/scripts/run-with-otel.sh +0 -50
- package/k6-tests/smoke-test/smoke-test.ts +0 -158
- package/k6-tests/tsconfig.json +0 -29
- package/k6-tests/utils/client_factory.ts +0 -128
- package/k6-tests/utils/common.ts +0 -489
- package/k6-tests/utils/logger.ts +0 -62
- package/k6-tests/utils/types.d.ts +0 -35
- package/malloy_mcp.dxt +0 -0
- package/publisher.config.example.bigquery.json +0 -33
- package/publisher.config.example.duckdb.json +0 -23
- package/publisher.config.json +0 -23
- package/scripts/bake-duckdb-extensions.js +0 -110
- package/src/config.spec.ts +0 -1535
- package/src/config.theme.spec.ts +0 -178
- package/src/config.ts +0 -1142
- package/src/constants.ts +0 -105
- package/src/controller/compile.controller.ts +0 -40
- package/src/controller/connection.controller.spec.ts +0 -849
- package/src/controller/connection.controller.ts +0 -901
- package/src/controller/database.controller.ts +0 -24
- package/src/controller/materialization.controller.spec.ts +0 -293
- package/src/controller/materialization.controller.ts +0 -319
- package/src/controller/model.controller.ts +0 -164
- package/src/controller/package.controller.spec.ts +0 -273
- package/src/controller/package.controller.ts +0 -306
- package/src/controller/query.controller.ts +0 -152
- package/src/controller/theme.controller.ts +0 -83
- package/src/controller/watch-mode.controller.ts +0 -243
- package/src/data_styles.ts +0 -71
- package/src/default-publisher.config.json +0 -23
- package/src/ducklake_version.spec.ts +0 -163
- package/src/ducklake_version.ts +0 -153
- package/src/errors.spec.ts +0 -119
- package/src/errors.ts +0 -237
- package/src/filter_deprecation.spec.ts +0 -64
- package/src/filter_deprecation.ts +0 -42
- package/src/health.spec.ts +0 -123
- package/src/health.ts +0 -233
- package/src/heap_check.spec.ts +0 -144
- package/src/heap_check.ts +0 -144
- package/src/instrumentation.ts +0 -214
- package/src/json_utils.spec.ts +0 -51
- package/src/json_utils.ts +0 -33
- package/src/logger.spec.ts +0 -210
- package/src/logger.ts +0 -251
- package/src/malloy_pin_prereqs.spec.ts +0 -25
- package/src/materialization_metrics.spec.ts +0 -169
- package/src/materialization_metrics.ts +0 -346
- package/src/mcp/error_messages.spec.ts +0 -35
- package/src/mcp/error_messages.ts +0 -183
- package/src/mcp/handler_utils.spec.ts +0 -108
- package/src/mcp/handler_utils.ts +0 -251
- package/src/mcp/mcp_constants.ts +0 -6
- package/src/mcp/query_envelope.spec.ts +0 -229
- package/src/mcp/query_envelope.ts +0 -240
- package/src/mcp/server.protocol.spec.ts +0 -250
- package/src/mcp/server.ts +0 -87
- package/src/mcp/skills/build_skills_bundle.spec.ts +0 -51
- package/src/mcp/skills/build_skills_bundle.ts +0 -176
- package/src/mcp/skills/skills_bundle.json +0 -1
- package/src/mcp/skills/skills_bundle.spec.ts +0 -171
- package/src/mcp/tool_response.spec.ts +0 -108
- package/src/mcp/tool_response.ts +0 -138
- package/src/mcp/tools/compile_tool.spec.ts +0 -315
- package/src/mcp/tools/compile_tool.ts +0 -208
- package/src/mcp/tools/docs_search/build_docs_index.ts +0 -132
- package/src/mcp/tools/docs_search/malloy_docs_index.json +0 -1
- package/src/mcp/tools/docs_search_tool.spec.ts +0 -32
- package/src/mcp/tools/docs_search_tool.ts +0 -128
- package/src/mcp/tools/embedding_index.spec.ts +0 -1236
- package/src/mcp/tools/embedding_index.ts +0 -808
- package/src/mcp/tools/execute_query_tool.spec.ts +0 -294
- package/src/mcp/tools/execute_query_tool.ts +0 -316
- package/src/mcp/tools/get_context_eval.ts +0 -275
- package/src/mcp/tools/get_context_tool.spec.ts +0 -592
- package/src/mcp/tools/get_context_tool.ts +0 -585
- package/src/mcp/tools/reload_package_tool.spec.ts +0 -232
- package/src/mcp/tools/reload_package_tool.ts +0 -132
- package/src/mcp_config.spec.ts +0 -919
- package/src/mcp_config.ts +0 -425
- package/src/oom_guards.integration.spec.ts +0 -269
- package/src/package_load/package_load_pool.spec.ts +0 -252
- package/src/package_load/package_load_pool.ts +0 -931
- package/src/package_load/package_load_worker.ts +0 -1034
- package/src/package_load/protocol.ts +0 -364
- package/src/package_load/rpc_wait_accountant.spec.ts +0 -109
- package/src/package_load/rpc_wait_accountant.ts +0 -76
- package/src/package_load_metrics.spec.ts +0 -114
- package/src/package_load_metrics.ts +0 -127
- package/src/path_safety.spec.ts +0 -158
- package/src/path_safety.ts +0 -146
- package/src/pg_helpers.spec.ts +0 -223
- package/src/pg_helpers.ts +0 -52
- package/src/query_cap_metrics.spec.ts +0 -89
- package/src/query_cap_metrics.ts +0 -116
- package/src/query_concurrency.spec.ts +0 -247
- package/src/query_concurrency.ts +0 -237
- package/src/query_metadata_metrics.ts +0 -49
- package/src/query_param_utils.ts +0 -29
- package/src/query_timeout.spec.ts +0 -224
- package/src/query_timeout.ts +0 -179
- package/src/runtime/publisher.js +0 -328
- package/src/server-old.ts +0 -988
- package/src/server.ts +0 -2059
- package/src/service/annotations.spec.ts +0 -118
- package/src/service/annotations.ts +0 -91
- package/src/service/authorize.spec.ts +0 -154
- package/src/service/authorize.ts +0 -498
- package/src/service/authorize_integration.spec.ts +0 -2041
- package/src/service/build_plan.spec.ts +0 -685
- package/src/service/build_plan.ts +0 -797
- package/src/service/compile_authorize.spec.ts +0 -85
- package/src/service/compile_fragment_techniques.spec.ts +0 -156
- package/src/service/connection.spec.ts +0 -2360
- package/src/service/connection.ts +0 -2187
- package/src/service/connection_config.spec.ts +0 -734
- package/src/service/connection_config.ts +0 -899
- package/src/service/connection_federation.spec.ts +0 -184
- package/src/service/connection_fingerprint.spec.ts +0 -102
- package/src/service/connection_service.spec.ts +0 -711
- package/src/service/connection_service.ts +0 -291
- package/src/service/cron_evaluator.spec.ts +0 -79
- package/src/service/cron_evaluator.ts +0 -105
- package/src/service/db_utils.spec.ts +0 -1085
- package/src/service/db_utils.ts +0 -1502
- package/src/service/duckdb_instance_isolation.spec.ts +0 -137
- package/src/service/ducklake_lazy_attach.spec.ts +0 -110
- package/src/service/embedding_provider.spec.ts +0 -329
- package/src/service/embedding_provider.ts +0 -236
- package/src/service/environment.ts +0 -2191
- package/src/service/environment_admission.spec.ts +0 -344
- package/src/service/environment_compile.spec.ts +0 -197
- package/src/service/environment_store.spec.ts +0 -2274
- package/src/service/environment_store.ts +0 -2434
- package/src/service/environment_store_anchoring.spec.ts +0 -107
- package/src/service/environment_store_clone.spec.ts +0 -350
- package/src/service/explore_visibility.spec.ts +0 -434
- package/src/service/exports_probe.spec.ts +0 -107
- package/src/service/extension_fetch_policy.spec.ts +0 -256
- package/src/service/filter.spec.ts +0 -458
- package/src/service/filter.ts +0 -341
- package/src/service/filter_bypass.spec.ts +0 -418
- package/src/service/filter_integration.spec.ts +0 -1004
- package/src/service/freshness.spec.ts +0 -183
- package/src/service/freshness.ts +0 -112
- package/src/service/gcs_s3_utils.ts +0 -427
- package/src/service/given.ts +0 -105
- package/src/service/givens_integration.spec.ts +0 -219
- package/src/service/manifest_loader.spec.ts +0 -189
- package/src/service/manifest_loader.ts +0 -156
- package/src/service/materialization_build_session.spec.ts +0 -435
- package/src/service/materialization_build_session.ts +0 -681
- package/src/service/materialization_config_validation.spec.ts +0 -99
- package/src/service/materialization_config_validation.ts +0 -120
- package/src/service/materialization_eligibility.spec.ts +0 -158
- package/src/service/materialization_eligibility.ts +0 -305
- package/src/service/materialization_schedule_surface.spec.ts +0 -256
- package/src/service/materialization_scheduler.spec.ts +0 -289
- package/src/service/materialization_scheduler.ts +0 -304
- package/src/service/materialization_serve_transform.spec.ts +0 -1003
- package/src/service/materialization_serve_transform.ts +0 -779
- package/src/service/materialization_service.spec.ts +0 -2373
- package/src/service/materialization_service.ts +0 -2342
- package/src/service/materialization_test_fixtures.ts +0 -249
- package/src/service/model.spec.ts +0 -1157
- package/src/service/model.ts +0 -3370
- package/src/service/model_limits.spec.ts +0 -209
- package/src/service/model_limits.ts +0 -131
- package/src/service/model_storage_serve.spec.ts +0 -193
- package/src/service/model_storage_serve_joins.spec.ts +0 -193
- package/src/service/package.spec.ts +0 -612
- package/src/service/package.ts +0 -1799
- package/src/service/package_manifest.spec.ts +0 -230
- package/src/service/package_manifest.ts +0 -241
- package/src/service/package_memory_governor.spec.ts +0 -173
- package/src/service/package_memory_governor.ts +0 -233
- package/src/service/package_quote_bound_tables.spec.ts +0 -171
- package/src/service/package_race.spec.ts +0 -208
- package/src/service/package_reload_safety.spec.ts +0 -193
- package/src/service/package_rollback.spec.ts +0 -190
- package/src/service/package_worker_path.spec.ts +0 -645
- package/src/service/path_injection.spec.ts +0 -39
- package/src/service/persist_annotation_validation.spec.ts +0 -89
- package/src/service/persist_annotation_validation.ts +0 -59
- package/src/service/persistence_policy.spec.ts +0 -374
- package/src/service/proxy.spec.ts +0 -414
- package/src/service/proxy.ts +0 -248
- package/src/service/query_boundary.spec.ts +0 -470
- package/src/service/query_metadata.spec.ts +0 -408
- package/src/service/query_metadata.ts +0 -492
- package/src/service/query_metadata_identity.spec.ts +0 -149
- package/src/service/query_text.spec.ts +0 -79
- package/src/service/query_text.ts +0 -43
- package/src/service/quoting.spec.ts +0 -135
- package/src/service/quoting.ts +0 -78
- package/src/service/resolve_environment.ts +0 -15
- package/src/service/restricted_mode.spec.ts +0 -299
- package/src/service/source_extraction.ts +0 -225
- package/src/service/theme_store.ts +0 -199
- package/src/storage/DatabaseInterface.ts +0 -198
- package/src/storage/StorageManager.ts +0 -113
- package/src/storage/duckdb/ConnectionRepository.ts +0 -143
- package/src/storage/duckdb/DuckDBConnection.ts +0 -123
- package/src/storage/duckdb/DuckDBRepository.ts +0 -208
- package/src/storage/duckdb/EnvironmentRepository.ts +0 -148
- package/src/storage/duckdb/MaterializationRepository.spec.ts +0 -191
- package/src/storage/duckdb/MaterializationRepository.ts +0 -336
- package/src/storage/duckdb/PackageRepository.ts +0 -135
- package/src/storage/duckdb/schema.ts +0 -229
- package/src/stream_helpers.spec.ts +0 -280
- package/src/stream_helpers.ts +0 -162
- package/src/telemetry.ts +0 -20
- package/src/test_helpers/metrics_harness.ts +0 -166
- package/src/theme_key_parity.spec.ts +0 -57
- package/src/utils.ts +0 -29
- package/tests/fixtures/authorize-compile/model.malloy +0 -9
- package/tests/fixtures/authorize-compile/publisher.json +0 -4
- package/tests/fixtures/html-pages-nopublic/model.malloy +0 -1
- package/tests/fixtures/html-pages-nopublic/publisher.json +0 -5
- package/tests/fixtures/html-pages-test/data.csv +0 -3
- package/tests/fixtures/html-pages-test/public/assets/app.css +0 -3
- package/tests/fixtures/html-pages-test/public/barehtml.html +0 -4
- package/tests/fixtures/html-pages-test/public/bodymeta.html +0 -8
- package/tests/fixtures/html-pages-test/public/data.json +0 -1
- package/tests/fixtures/html-pages-test/public/fitcomment.html +0 -11
- package/tests/fixtures/html-pages-test/public/index.html +0 -9
- package/tests/fixtures/html-pages-test/public/nohead.html +0 -8
- package/tests/fixtures/html-pages-test/public/notfit.html +0 -13
- package/tests/fixtures/html-pages-test/public/slides.html +0 -12
- package/tests/fixtures/html-pages-test/public/sub/page2.html +0 -9
- package/tests/fixtures/html-pages-test/public/unterminated.html +0 -10
- package/tests/fixtures/html-pages-test/publisher.json +0 -5
- package/tests/fixtures/html-pages-test/report.malloy +0 -1
- package/tests/fixtures/persist-multi-level/data/orders.csv +0 -5
- package/tests/fixtures/persist-multi-level/multi_level.malloy +0 -18
- package/tests/fixtures/persist-multi-level/publisher.json +0 -5
- package/tests/fixtures/persist-schedule-test/data/orders.csv +0 -5
- package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +0 -11
- package/tests/fixtures/persist-schedule-test/publisher.json +0 -7
- package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +0 -5
- package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +0 -11
- package/tests/fixtures/persist-schedule-warn-test/publisher.json +0 -6
- package/tests/fixtures/persist-test/data/orders.csv +0 -5
- package/tests/fixtures/persist-test/persist_test.malloy +0 -11
- package/tests/fixtures/persist-test/publisher.json +0 -5
- package/tests/fixtures/publisher.config.json +0 -15
- package/tests/fixtures/query-givens/data/orders.csv +0 -7
- package/tests/fixtures/query-givens/model.malloy +0 -34
- package/tests/fixtures/query-givens/publisher.json +0 -5
- package/tests/fixtures/xlsx/database.xlsx +0 -0
- package/tests/harness/mcp_test_setup.ts +0 -332
- package/tests/harness/rest_e2e.ts +0 -68
- package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +0 -92
- package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +0 -280
- package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +0 -138
- package/tests/integration/first_boot/readiness_line.integration.spec.ts +0 -177
- package/tests/integration/html_pages/html_pages.integration.spec.ts +0 -440
- package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +0 -259
- package/tests/integration/materialization/environment_list.integration.spec.ts +0 -133
- package/tests/integration/materialization/freshness_gate.integration.spec.ts +0 -292
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +0 -430
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +0 -441
- package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +0 -150
- package/tests/integration/materialization/reference_manifest.integration.spec.ts +0 -251
- package/tests/integration/materialization/schedule_edit.integration.spec.ts +0 -126
- package/tests/integration/materialization/scheduler.integration.spec.ts +0 -200
- package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +0 -172
- package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +0 -256
- package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +0 -143
- package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +0 -438
- package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +0 -235
- package/tests/integration/mcp/mcp_transport.integration.spec.ts +0 -98
- package/tests/integration/query_givens/query_givens.integration.spec.ts +0 -146
- package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +0 -121
- package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +0 -110
- package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -421
- package/tests/unit/controller/theme.controller.spec.ts +0 -141
- package/tests/unit/duckdb/attached_databases.test.ts +0 -1243
- package/tests/unit/duckdb/duckdb_connection.test.ts +0 -181
- package/tests/unit/duckdb/legacy_schema_migration.test.ts +0 -197
- package/tests/unit/duckdb/repositories.test.ts +0 -208
- package/tests/unit/ducklake/ducklake.test.ts +0 -952
- package/tests/unit/service/theme_store.spec.ts +0 -139
- package/tsconfig.json +0 -32
|
@@ -1,99 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from "bun:test";
|
|
2
|
-
import type { components } from "../api";
|
|
3
|
-
import { materializationConfigWarnings } from "./materialization_config_validation";
|
|
4
|
-
|
|
5
|
-
const source = (
|
|
6
|
-
name: string,
|
|
7
|
-
queryMetadata: Record<string, string> | null,
|
|
8
|
-
): components["schemas"]["PersistSourcePlan"] => ({
|
|
9
|
-
name,
|
|
10
|
-
sourceID: `${name}@m`,
|
|
11
|
-
connectionName: "duckdb",
|
|
12
|
-
sourceEntityId: "bid",
|
|
13
|
-
sql: "SELECT 1",
|
|
14
|
-
columns: [],
|
|
15
|
-
queryMetadata,
|
|
16
|
-
});
|
|
17
|
-
|
|
18
|
-
describe("materializationConfigWarnings", () => {
|
|
19
|
-
it("says nothing about a clean config", () => {
|
|
20
|
-
expect(
|
|
21
|
-
materializationConfigWarnings({
|
|
22
|
-
packageMaterialization: {
|
|
23
|
-
schedule: null,
|
|
24
|
-
freshness: null,
|
|
25
|
-
queryMetadata: { team: "finance" },
|
|
26
|
-
},
|
|
27
|
-
sources: [source("orders", { team: "finance" })],
|
|
28
|
-
}),
|
|
29
|
-
).toEqual([]);
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
it("passes manifest deprecations through", () => {
|
|
33
|
-
expect(
|
|
34
|
-
materializationConfigWarnings({
|
|
35
|
-
manifestWarnings: ['"scope" at the manifest root is deprecated'],
|
|
36
|
-
}),
|
|
37
|
-
).toEqual([{ message: '"scope" at the manifest root is deprecated' }]);
|
|
38
|
-
});
|
|
39
|
-
|
|
40
|
-
it("reports a package-level property that violates the contract", () => {
|
|
41
|
-
const warnings = materializationConfigWarnings({
|
|
42
|
-
packageMaterialization: {
|
|
43
|
-
schedule: null,
|
|
44
|
-
freshness: null,
|
|
45
|
-
queryMetadata: { "team.name": "finance" },
|
|
46
|
-
},
|
|
47
|
-
});
|
|
48
|
-
expect(warnings).toHaveLength(1);
|
|
49
|
-
expect(warnings[0].message).toContain("materialization.queryMetadata");
|
|
50
|
-
expect(warnings[0].message).toContain("team.name");
|
|
51
|
-
expect(warnings[0].target).toBeUndefined();
|
|
52
|
-
});
|
|
53
|
-
|
|
54
|
-
it("reports the offending property at the source that resolves it", () => {
|
|
55
|
-
const warnings = materializationConfigWarnings({
|
|
56
|
-
sources: [source("orders", { "team.name": "finance" })],
|
|
57
|
-
});
|
|
58
|
-
expect(warnings).toHaveLength(1);
|
|
59
|
-
expect(warnings[0].target).toBe("orders");
|
|
60
|
-
expect(warnings[0].message).toContain("#@ persist queryMetadata");
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
it("reports a BigQuery-dropped name as a warning, not silence", () => {
|
|
64
|
-
const warnings = materializationConfigWarnings({
|
|
65
|
-
sources: [source("orders", { _team: "finance" })],
|
|
66
|
-
});
|
|
67
|
-
expect(warnings).toHaveLength(1);
|
|
68
|
-
expect(warnings[0].message).toMatch(/BigQuery/);
|
|
69
|
-
});
|
|
70
|
-
|
|
71
|
-
it("reports the problem at every level that resolves it", () => {
|
|
72
|
-
// The package declares it and both sources inherit it. Reporting each
|
|
73
|
-
// level is what lets an author find the one they can edit; identical
|
|
74
|
-
// messages are collapsed, different levels are not.
|
|
75
|
-
const warnings = materializationConfigWarnings({
|
|
76
|
-
packageMaterialization: {
|
|
77
|
-
schedule: null,
|
|
78
|
-
freshness: null,
|
|
79
|
-
queryMetadata: { "team.name": "finance" },
|
|
80
|
-
},
|
|
81
|
-
sources: [
|
|
82
|
-
source("orders", { "team.name": "finance" }),
|
|
83
|
-
source("returns", { "team.name": "finance" }),
|
|
84
|
-
],
|
|
85
|
-
});
|
|
86
|
-
expect(warnings).toHaveLength(3);
|
|
87
|
-
expect(warnings.map((w) => w.target)).toEqual([
|
|
88
|
-
undefined,
|
|
89
|
-
"orders",
|
|
90
|
-
"returns",
|
|
91
|
-
]);
|
|
92
|
-
});
|
|
93
|
-
|
|
94
|
-
it("ignores sources with no metadata", () => {
|
|
95
|
-
expect(
|
|
96
|
-
materializationConfigWarnings({ sources: [source("orders", null)] }),
|
|
97
|
-
).toEqual([]);
|
|
98
|
-
});
|
|
99
|
-
});
|
|
@@ -1,120 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Publish-time validation of a package's materialization config.
|
|
3
|
-
*
|
|
4
|
-
* One module for the rules so they cannot drift between the levels a knob can be
|
|
5
|
-
* declared at. Two principles, both learned from shipped bugs:
|
|
6
|
-
*
|
|
7
|
-
* 1. **Nothing is silently dropped.** A declaration that will not do what it
|
|
8
|
-
* says has to surface somewhere — a publish error, a publish warning, a 400,
|
|
9
|
-
* or a metered runtime drop. A `#@ persist` name that was silently ignored
|
|
10
|
-
* (source published, never materialized, no error anywhere) is why
|
|
11
|
-
* `persist_annotation_validation.ts` exists; the same rule applies here.
|
|
12
|
-
* 2. **Strict at declaration, lenient at execution.** Publish and the API have a
|
|
13
|
-
* human behind them and report problems; the runtime resolve path clamps and
|
|
14
|
-
* meters instead, because metadata must never break a query or a build.
|
|
15
|
-
*
|
|
16
|
-
* Today this owns the `queryMetadata` rules and the manifest-shape deprecations,
|
|
17
|
-
* which are ADVISORY — they surface on the package's operator warnings array.
|
|
18
|
-
* They deliberately do NOT go through `Package.persistencePolicyWarnings`, which
|
|
19
|
-
* is a publish REJECTION gate (and disarms the scheduler): a deprecated manifest
|
|
20
|
-
* shape or a mistyped tag must not stop a package from publishing. The
|
|
21
|
-
* scope/schedule/freshness coherence rules that do reject still live there;
|
|
22
|
-
* moving them here is a mechanical follow-up with no behavior change.
|
|
23
|
-
*/
|
|
24
|
-
|
|
25
|
-
import type { components } from "../api";
|
|
26
|
-
import {
|
|
27
|
-
queryMetadataAdvisoryWarnings,
|
|
28
|
-
queryMetadataBudgetWarning,
|
|
29
|
-
queryMetadataViolations,
|
|
30
|
-
type QueryMetadata,
|
|
31
|
-
} from "./query_metadata";
|
|
32
|
-
|
|
33
|
-
type WirePackageMaterialization =
|
|
34
|
-
components["schemas"]["PackageMaterializationConfig"];
|
|
35
|
-
type WirePersistSourcePlan = components["schemas"]["PersistSourcePlan"];
|
|
36
|
-
|
|
37
|
-
export interface MaterializationConfigInput {
|
|
38
|
-
/** The package manifest's `materialization` block, as parsed. */
|
|
39
|
-
packageMaterialization?: WirePackageMaterialization | null;
|
|
40
|
-
/** The compiled plan's sources, carrying each one's RESOLVED metadata. */
|
|
41
|
-
sources?: WirePersistSourcePlan[];
|
|
42
|
-
/**
|
|
43
|
-
* Deprecations the manifest parse tolerated (e.g. a root-level `scope`), which
|
|
44
|
-
* a load keeps working but a publish should report.
|
|
45
|
-
*/
|
|
46
|
-
manifestWarnings?: string[];
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
/** One finding, in the shape the wire package's operator warnings array uses. */
|
|
50
|
-
export interface MaterializationConfigWarning {
|
|
51
|
-
/** The persist source the finding belongs to, absent for a package-level one. */
|
|
52
|
-
target?: string;
|
|
53
|
-
message: string;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
function metadataWarnings(
|
|
57
|
-
level: string,
|
|
58
|
-
metadata: QueryMetadata | null | undefined,
|
|
59
|
-
target?: string,
|
|
60
|
-
): MaterializationConfigWarning[] {
|
|
61
|
-
if (!metadata) return [];
|
|
62
|
-
const budget = queryMetadataBudgetWarning(Object.keys(metadata).length);
|
|
63
|
-
return [
|
|
64
|
-
...queryMetadataViolations(metadata),
|
|
65
|
-
...queryMetadataAdvisoryWarnings(metadata),
|
|
66
|
-
...(budget ? [budget] : []),
|
|
67
|
-
].map((message) => ({
|
|
68
|
-
// The level is in the message because a reader needs to know WHICH
|
|
69
|
-
// declaration to edit, and an inherited property has more than one.
|
|
70
|
-
message: `${level}: ${message}`,
|
|
71
|
-
...(target ? { target } : {}),
|
|
72
|
-
}));
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
/**
|
|
76
|
-
* Everything wrong or inadvisable about a package's materialization config, as
|
|
77
|
-
* actionable messages. Warnings rather than errors: a package published before a
|
|
78
|
-
* rule existed must keep loading, and a bad metadata property degrades
|
|
79
|
-
* attribution rather than corrupting anything.
|
|
80
|
-
*
|
|
81
|
-
* A source's metadata is checked in its RESOLVED form (the effective bag from
|
|
82
|
-
* package → model-file → `#@ persist`), because that is what will actually be
|
|
83
|
-
* attached — a property inherited from the manifest is just as broken at the
|
|
84
|
-
* source as it is at the package, and reporting it at both is how an author finds
|
|
85
|
-
* the one they can edit.
|
|
86
|
-
*/
|
|
87
|
-
export function materializationConfigWarnings(
|
|
88
|
-
input: MaterializationConfigInput,
|
|
89
|
-
): MaterializationConfigWarning[] {
|
|
90
|
-
const warnings: MaterializationConfigWarning[] = (
|
|
91
|
-
input.manifestWarnings ?? []
|
|
92
|
-
).map((message) => ({ message }));
|
|
93
|
-
|
|
94
|
-
warnings.push(
|
|
95
|
-
...metadataWarnings(
|
|
96
|
-
"materialization.queryMetadata",
|
|
97
|
-
input.packageMaterialization?.queryMetadata,
|
|
98
|
-
),
|
|
99
|
-
);
|
|
100
|
-
|
|
101
|
-
for (const source of input.sources ?? []) {
|
|
102
|
-
warnings.push(
|
|
103
|
-
...metadataWarnings(
|
|
104
|
-
`#@ persist queryMetadata`,
|
|
105
|
-
source.queryMetadata,
|
|
106
|
-
source.name,
|
|
107
|
-
),
|
|
108
|
-
);
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
// Identical findings collapse (two sources inheriting one bad package
|
|
112
|
-
// property produce one message each, not one per level per source).
|
|
113
|
-
const seen = new Set<string>();
|
|
114
|
-
return warnings.filter((warning) => {
|
|
115
|
-
const key = `${warning.target ?? ""}\u0000${warning.message}`;
|
|
116
|
-
if (seen.has(key)) return false;
|
|
117
|
-
seen.add(key);
|
|
118
|
-
return true;
|
|
119
|
-
});
|
|
120
|
-
}
|
|
@@ -1,158 +0,0 @@
|
|
|
1
|
-
// Real-compiler contract for the materialization-eligibility gate. The gate
|
|
2
|
-
// reads specific compiled-IR shapes (a Parameter's `value: null` for unbound
|
|
3
|
-
// params; `refSummary.givenUsage` / `given` IR nodes for given references), so
|
|
4
|
-
// it is pinned against the real compiler rather than hand-built stubs — a
|
|
5
|
-
// Malloy change to either shape must fail here, not leak an ineligible source
|
|
6
|
-
// (a frozen tenant-filtered table) into the tier.
|
|
7
|
-
import { DuckDBConnection } from "@malloydata/db-duckdb";
|
|
8
|
-
import type { PersistSource } from "@malloydata/malloy";
|
|
9
|
-
import {
|
|
10
|
-
FixedConnectionMap,
|
|
11
|
-
InMemoryURLReader,
|
|
12
|
-
Runtime,
|
|
13
|
-
} from "@malloydata/malloy";
|
|
14
|
-
import { beforeAll, describe, expect, it } from "bun:test";
|
|
15
|
-
import { MaterializationEligibilityError } from "../errors";
|
|
16
|
-
import { assertMaterializationEligible } from "./materialization_eligibility";
|
|
17
|
-
|
|
18
|
-
const ROOT = "file:///elig/";
|
|
19
|
-
let connections: FixedConnectionMap;
|
|
20
|
-
|
|
21
|
-
beforeAll(() => {
|
|
22
|
-
const duckdb = new DuckDBConnection("duckdb", ":memory:");
|
|
23
|
-
connections = new FixedConnectionMap(
|
|
24
|
-
new Map([["duckdb", duckdb]]),
|
|
25
|
-
"duckdb",
|
|
26
|
-
);
|
|
27
|
-
});
|
|
28
|
-
|
|
29
|
-
/** Compile a single-file model and return its persist sources by name. */
|
|
30
|
-
async function persistSources(
|
|
31
|
-
model: string,
|
|
32
|
-
): Promise<Record<string, PersistSource>> {
|
|
33
|
-
const urlReader = new InMemoryURLReader(
|
|
34
|
-
new Map([[`${ROOT}m.malloy`, model]]),
|
|
35
|
-
);
|
|
36
|
-
const runtime = new Runtime({ urlReader, connections });
|
|
37
|
-
const compiled = await runtime
|
|
38
|
-
.loadModel(new URL(`${ROOT}m.malloy`), { importBaseURL: new URL(ROOT) })
|
|
39
|
-
.getModel();
|
|
40
|
-
const byName: Record<string, PersistSource> = {};
|
|
41
|
-
for (const source of Object.values(compiled.getBuildPlan().sources)) {
|
|
42
|
-
byName[source.name] = source;
|
|
43
|
-
}
|
|
44
|
-
return byName;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
describe("assertMaterializationEligible", () => {
|
|
48
|
-
it("accepts a plain persist source (no params, no givens)", async () => {
|
|
49
|
-
const sources = await persistSources(`##! experimental.persistence
|
|
50
|
-
source: base is duckdb.sql("SELECT 1 AS amount, 'US' AS region")
|
|
51
|
-
#@ persist name="mz_plain"
|
|
52
|
-
source: mz_plain is base -> { aggregate: c is count() }`);
|
|
53
|
-
expect(sources.mz_plain).toBeDefined();
|
|
54
|
-
expect(() =>
|
|
55
|
-
assertMaterializationEligible(sources.mz_plain),
|
|
56
|
-
).not.toThrow();
|
|
57
|
-
});
|
|
58
|
-
|
|
59
|
-
it("accepts a parameter bound to a constant", async () => {
|
|
60
|
-
const sources = await persistSources(`##! experimental.persistence
|
|
61
|
-
##! experimental.parameters
|
|
62
|
-
source: base is duckdb.sql("SELECT 1 AS amount, 'US' AS region")
|
|
63
|
-
#@ persist name="mz_bound"
|
|
64
|
-
source: mz_bound(threshold::number is 5) is base -> { aggregate: c is count() }`);
|
|
65
|
-
expect(sources.mz_bound).toBeDefined();
|
|
66
|
-
expect(() =>
|
|
67
|
-
assertMaterializationEligible(sources.mz_bound),
|
|
68
|
-
).not.toThrow();
|
|
69
|
-
});
|
|
70
|
-
|
|
71
|
-
it("refuses a source with an unbound (free) parameter", async () => {
|
|
72
|
-
const sources = await persistSources(`##! experimental.persistence
|
|
73
|
-
##! experimental.parameters
|
|
74
|
-
source: base is duckdb.sql("SELECT 1 AS amount, 'US' AS region")
|
|
75
|
-
#@ persist name="mz_free"
|
|
76
|
-
source: mz_free(threshold::number) is base -> { aggregate: c is count() }`);
|
|
77
|
-
expect(sources.mz_free).toBeDefined();
|
|
78
|
-
expect(() => assertMaterializationEligible(sources.mz_free)).toThrow(
|
|
79
|
-
MaterializationEligibilityError,
|
|
80
|
-
);
|
|
81
|
-
expect(() => assertMaterializationEligible(sources.mz_free)).toThrow(
|
|
82
|
-
/unbound parameter/i,
|
|
83
|
-
);
|
|
84
|
-
});
|
|
85
|
-
|
|
86
|
-
it("refuses a source that references a given (RLAC security refusal)", async () => {
|
|
87
|
-
const sources = await persistSources(`##! experimental.persistence
|
|
88
|
-
##! experimental.givens
|
|
89
|
-
given: tenant :: string is 'acme'
|
|
90
|
-
source: base is duckdb.sql("SELECT 1 AS amount, 'acme' AS tenant")
|
|
91
|
-
#@ persist name="mz_given"
|
|
92
|
-
source: mz_given is base -> { where: tenant = $tenant; aggregate: c is count() }`);
|
|
93
|
-
expect(sources.mz_given).toBeDefined();
|
|
94
|
-
expect(() => assertMaterializationEligible(sources.mz_given)).toThrow(
|
|
95
|
-
MaterializationEligibilityError,
|
|
96
|
-
);
|
|
97
|
-
expect(() => assertMaterializationEligible(sources.mz_given)).toThrow(
|
|
98
|
-
/given/i,
|
|
99
|
-
);
|
|
100
|
-
});
|
|
101
|
-
|
|
102
|
-
it("refuses a source protected by its own #(authorize) gate", async () => {
|
|
103
|
-
const sources = await persistSources(`##! experimental.persistence
|
|
104
|
-
##! experimental.givens
|
|
105
|
-
given: role :: string is 'analyst'
|
|
106
|
-
source: base is duckdb.sql("SELECT 1 AS amount, 'US' AS region")
|
|
107
|
-
#(authorize) "$role = 'analyst'"
|
|
108
|
-
#@ persist name="mz_authz"
|
|
109
|
-
source: mz_authz is base -> { aggregate: c is count() }`);
|
|
110
|
-
expect(sources.mz_authz).toBeDefined();
|
|
111
|
-
expect(() => assertMaterializationEligible(sources.mz_authz)).toThrow(
|
|
112
|
-
MaterializationEligibilityError,
|
|
113
|
-
);
|
|
114
|
-
expect(() => assertMaterializationEligible(sources.mz_authz)).toThrow(
|
|
115
|
-
/authorize/i,
|
|
116
|
-
);
|
|
117
|
-
});
|
|
118
|
-
|
|
119
|
-
it("refuses a source that reaches an #(authorize) gate through a JOIN", async () => {
|
|
120
|
-
// The gate is on the joined source, not on mz_authz_joined itself — a join
|
|
121
|
-
// must not launder an authorize-gated source into a frozen table.
|
|
122
|
-
const sources = await persistSources(`##! experimental.persistence
|
|
123
|
-
##! experimental.givens
|
|
124
|
-
given: role :: string is 'analyst'
|
|
125
|
-
#(authorize) "$role = 'analyst'"
|
|
126
|
-
source: gated is duckdb.sql("SELECT 1 AS amount, 'acme' AS tenant")
|
|
127
|
-
source: joiner is duckdb.sql("SELECT 2 AS n, 'acme' AS tenant")
|
|
128
|
-
#@ persist name="mz_authz_joined"
|
|
129
|
-
source: mz_authz_joined is joiner extend {
|
|
130
|
-
join_one: g is gated on tenant = g.tenant
|
|
131
|
-
} -> { aggregate: c is count() }`);
|
|
132
|
-
expect(sources.mz_authz_joined).toBeDefined();
|
|
133
|
-
expect(() =>
|
|
134
|
-
assertMaterializationEligible(sources.mz_authz_joined),
|
|
135
|
-
).toThrow(MaterializationEligibilityError);
|
|
136
|
-
});
|
|
137
|
-
|
|
138
|
-
it("refuses a source that reaches a given through a JOIN (not just its own pipeline)", async () => {
|
|
139
|
-
// The given lives on a joined source, not on mz_joined's own where/fields.
|
|
140
|
-
// The compiled struct embeds the joined SourceDef, so the fail-closed walk
|
|
141
|
-
// must still reach it — a join must not launder a given-filtered source.
|
|
142
|
-
const sources = await persistSources(`##! experimental.persistence
|
|
143
|
-
##! experimental.givens
|
|
144
|
-
given: tenant :: string is 'acme'
|
|
145
|
-
source: gated is duckdb.sql("SELECT 1 AS amount, 'acme' AS tenant") extend {
|
|
146
|
-
where: tenant = $tenant
|
|
147
|
-
}
|
|
148
|
-
source: joiner is duckdb.sql("SELECT 2 AS n, 'acme' AS tenant")
|
|
149
|
-
#@ persist name="mz_joined"
|
|
150
|
-
source: mz_joined is joiner extend {
|
|
151
|
-
join_one: g is gated on tenant = g.tenant
|
|
152
|
-
} -> { aggregate: c is count() }`);
|
|
153
|
-
expect(sources.mz_joined).toBeDefined();
|
|
154
|
-
expect(() => assertMaterializationEligible(sources.mz_joined)).toThrow(
|
|
155
|
-
MaterializationEligibilityError,
|
|
156
|
-
);
|
|
157
|
-
});
|
|
158
|
-
});
|
|
@@ -1,305 +0,0 @@
|
|
|
1
|
-
import type { PersistSource } from "@malloydata/malloy";
|
|
2
|
-
import { MaterializationEligibilityError } from "../errors";
|
|
3
|
-
import { recordEligibilityRefused } from "../materialization_metrics";
|
|
4
|
-
import { parseAuthorizeAnnotation } from "./authorize";
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Compile-time eligibility gate for materializing a persist source into a
|
|
8
|
-
* *storage* destination (a DuckDB/DuckLake tier table), as opposed to the
|
|
9
|
-
* default colocated path (no `storage=`). This is a HARD REFUSE: an
|
|
10
|
-
* ineligible source never builds an artifact.
|
|
11
|
-
*
|
|
12
|
-
* It exists because a `storage=<duckdb>` table is built once and served frozen
|
|
13
|
-
* to every subsequent query. Two properties that hold for a live query — the
|
|
14
|
-
* relation is recomputed per query, and per-tenant access filters bind at query
|
|
15
|
-
* time — are lost the moment the relation is frozen. So a source is only
|
|
16
|
-
* materializable when its relation is fully determined at build time:
|
|
17
|
-
*
|
|
18
|
-
* 1. **No unbound (free) parameters.** A source declaring an unbound parameter
|
|
19
|
-
* is a *template* instantiated per query — there is no single relation to
|
|
20
|
-
* freeze. Parameters bound to a constant are fine (the relation is fixed,
|
|
21
|
-
* and the bound value already distinguishes the content address).
|
|
22
|
-
* 2. **No given references — a security refusal.** Givens bind at the
|
|
23
|
-
* runtime/query layer and are the documented mechanism for row-level access
|
|
24
|
-
* control (RLAC). A source filtered by a given (`where: tenant_id = $TENANT`)
|
|
25
|
-
* materialized once and served frozen would leak one tenant's rows to every
|
|
26
|
-
* tenant. This check fails closed: if the source references any given, it is
|
|
27
|
-
* refused, no exceptions.
|
|
28
|
-
* 3. **No `#(authorize)` gate — a security refusal.** An authorize expression
|
|
29
|
-
* is a per-request *who-can-query* gate evaluated at query time. The served
|
|
30
|
-
* virtual shape of a materialized source carries no gate to evaluate, so a
|
|
31
|
-
* materialized authorize-gated source would be served to everyone,
|
|
32
|
-
* bypassing the gate. Fails closed and reaches a gate on a JOINED source too
|
|
33
|
-
* (a join must not launder an authorize-gated source), mirroring the
|
|
34
|
-
* transitive `#(authorize)` enforcement on the live serve path (#906).
|
|
35
|
-
*
|
|
36
|
-
* One further eligibility property from the design — the served source must
|
|
37
|
-
* compile in DuckDB (portability) — is enforced at *build* time against the
|
|
38
|
-
* captured table schema (see the DuckDB-compile gate in the build path), not
|
|
39
|
-
* here: it needs the post-build authoritative schema this compile-time pass
|
|
40
|
-
* does not have. This pass covers the properties determinable from the compiled
|
|
41
|
-
* source alone (free parameters, given references, authorize gates), so it can
|
|
42
|
-
* fail fast before any warehouse work.
|
|
43
|
-
*
|
|
44
|
-
* The gate reads the *compiled* source (only the compiled `ModelDef` sees
|
|
45
|
-
* effective annotations, parameters, and given usage through `extend`/imports —
|
|
46
|
-
* a raw-source scan would miss inherited declarations), so it must run after
|
|
47
|
-
* compilation and before the build.
|
|
48
|
-
*
|
|
49
|
-
* @throws {MaterializationEligibilityError} (HTTP 422) naming the source and the
|
|
50
|
-
* specific reason, so the author can either fix the source or drop `storage=`.
|
|
51
|
-
*/
|
|
52
|
-
export function assertMaterializationEligible(
|
|
53
|
-
persistSource: PersistSource,
|
|
54
|
-
): void {
|
|
55
|
-
const sourceName = persistSource.name;
|
|
56
|
-
|
|
57
|
-
// Fail closed, like referencesGiven / referencesAuthorize below: an unreadable
|
|
58
|
-
// parameter surface is a refusal, not an assumed absence of free parameters.
|
|
59
|
-
let unbound: string[];
|
|
60
|
-
try {
|
|
61
|
-
unbound = unboundParameterNames(persistSource);
|
|
62
|
-
} catch (err) {
|
|
63
|
-
recordEligibilityRefused("free_parameter");
|
|
64
|
-
throw new MaterializationEligibilityError({
|
|
65
|
-
message:
|
|
66
|
-
`Source '${sourceName}' cannot be materialized into a storage ` +
|
|
67
|
-
`destination: its parameter surface could not be determined ` +
|
|
68
|
-
`(${err instanceof Error ? err.message : String(err)}), so the ` +
|
|
69
|
-
`publisher cannot prove the source has no free parameters. This is ` +
|
|
70
|
-
`refused for safety. Drop 'storage=' to serve it live from the ` +
|
|
71
|
-
`source warehouse.`,
|
|
72
|
-
});
|
|
73
|
-
}
|
|
74
|
-
if (unbound.length > 0) {
|
|
75
|
-
recordEligibilityRefused("free_parameter");
|
|
76
|
-
throw new MaterializationEligibilityError({
|
|
77
|
-
message:
|
|
78
|
-
`Source '${sourceName}' cannot be materialized into a storage ` +
|
|
79
|
-
`destination: it has unbound parameter(s) ` +
|
|
80
|
-
`${unbound.map((p) => `'${p}'`).join(", ")}. A source with a free ` +
|
|
81
|
-
`parameter is a template instantiated per query, so there is no ` +
|
|
82
|
-
`single relation to materialize. Bind the parameter to a constant, ` +
|
|
83
|
-
`or drop 'storage=' to serve it live from the source warehouse.`,
|
|
84
|
-
});
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
if (referencesGiven(persistSource)) {
|
|
88
|
-
recordEligibilityRefused("given");
|
|
89
|
-
throw new MaterializationEligibilityError({
|
|
90
|
-
message:
|
|
91
|
-
`Source '${sourceName}' cannot be materialized into a storage ` +
|
|
92
|
-
`destination: it references a given. Givens bind per query and are ` +
|
|
93
|
-
`used for row-level access control, so a materialized-once table ` +
|
|
94
|
-
`served to everyone would leak filtered rows across tenants. This ` +
|
|
95
|
-
`is refused for safety. Serve this source live (drop 'storage=').`,
|
|
96
|
-
});
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
if (referencesAuthorize(persistSource)) {
|
|
100
|
-
recordEligibilityRefused("authorize");
|
|
101
|
-
throw new MaterializationEligibilityError({
|
|
102
|
-
message:
|
|
103
|
-
`Source '${sourceName}' cannot be materialized into a storage ` +
|
|
104
|
-
`destination: it is protected by an #(authorize) gate (its own or a ` +
|
|
105
|
-
`joined source's). An authorize expression is evaluated per request; ` +
|
|
106
|
-
`a materialized-once table served frozen carries no gate, so it would ` +
|
|
107
|
-
`be served to everyone, bypassing authorization. This is refused for ` +
|
|
108
|
-
`safety. Serve this source live (drop 'storage=').`,
|
|
109
|
-
});
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
/**
|
|
114
|
-
* Names of the source's parameters that are declared but not bound to a value.
|
|
115
|
-
* A Malloy `Parameter` carries `value: ConstantExpr | null`; `null` is an
|
|
116
|
-
* unbound (free) parameter — bound-to-constant parameters have a non-null value.
|
|
117
|
-
*
|
|
118
|
-
* Fail-closed like the given/authorize walks (the caller turns a throw into a
|
|
119
|
-
* refusal): a parameter counts as unbound unless demonstrably bound, so a
|
|
120
|
-
* compiler shift to `value: undefined` can't let a template through. A MISSING
|
|
121
|
-
* `parameters` field is not an error — most sources declare none — so a renamed
|
|
122
|
-
* or relocated field is the one drift this can't catch; the real-compiler
|
|
123
|
-
* eligibility spec is what pins that shape.
|
|
124
|
-
*/
|
|
125
|
-
function unboundParameterNames(persistSource: PersistSource): string[] {
|
|
126
|
-
const def = persistSource._sourceDef as unknown;
|
|
127
|
-
if (def === null || typeof def !== "object") {
|
|
128
|
-
throw new Error("compiled source definition is not readable");
|
|
129
|
-
}
|
|
130
|
-
const parameters = (def as { parameters?: unknown }).parameters;
|
|
131
|
-
if (parameters === undefined || parameters === null) return [];
|
|
132
|
-
if (typeof parameters !== "object" || Array.isArray(parameters)) {
|
|
133
|
-
throw new Error(
|
|
134
|
-
`compiled source 'parameters' has an unexpected shape (${
|
|
135
|
-
Array.isArray(parameters) ? "array" : typeof parameters
|
|
136
|
-
})`,
|
|
137
|
-
);
|
|
138
|
-
}
|
|
139
|
-
const unbound: string[] = [];
|
|
140
|
-
for (const [name, param] of Object.entries(
|
|
141
|
-
parameters as Record<string, unknown>,
|
|
142
|
-
)) {
|
|
143
|
-
const bound =
|
|
144
|
-
param !== null &&
|
|
145
|
-
typeof param === "object" &&
|
|
146
|
-
(param as { value?: unknown }).value !== null &&
|
|
147
|
-
(param as { value?: unknown }).value !== undefined;
|
|
148
|
-
if (!bound) unbound.push(name);
|
|
149
|
-
}
|
|
150
|
-
return unbound;
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
/**
|
|
154
|
-
* Whether the compiled source (transitively) references any given. Fail-closed:
|
|
155
|
-
* walks the compiled source definition for both of Malloy's given signals —
|
|
156
|
-
* a non-empty `refSummary.givenUsage` (populated on filters and expressions by
|
|
157
|
-
* the reference-tracking walker) and any `given` / `givenReference` IR node —
|
|
158
|
-
* anywhere in the source's filter list, field expressions, or nested pipeline.
|
|
159
|
-
*
|
|
160
|
-
* A generic bounded walk (rather than reading a single well-known field) is
|
|
161
|
-
* deliberate: a given can hide in a field expression or a nested view, not just
|
|
162
|
-
* a top-level `where:`, and missing one is a tenant-isolation breach. The walk
|
|
163
|
-
* is depth- and visited-bounded so a cyclic IR graph cannot hang it, and any
|
|
164
|
-
* introspection failure is treated as "references a given" (fail closed).
|
|
165
|
-
*/
|
|
166
|
-
function referencesGiven(persistSource: PersistSource): boolean {
|
|
167
|
-
try {
|
|
168
|
-
return walkForGiven(persistSource._sourceDef, new WeakSet(), 0);
|
|
169
|
-
} catch {
|
|
170
|
-
// Fail closed: if we cannot prove the source is given-free, refuse it.
|
|
171
|
-
return true;
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
/** Max IR depth to walk; deep enough for real sources, a hang backstop. */
|
|
176
|
-
const MAX_GIVEN_WALK_DEPTH = 200;
|
|
177
|
-
|
|
178
|
-
function walkForGiven(
|
|
179
|
-
node: unknown,
|
|
180
|
-
seen: WeakSet<object>,
|
|
181
|
-
depth: number,
|
|
182
|
-
): boolean {
|
|
183
|
-
if (depth > MAX_GIVEN_WALK_DEPTH) {
|
|
184
|
-
// Refuse rather than risk an unbounded structure hiding a given.
|
|
185
|
-
throw new Error("given-usage walk exceeded max depth");
|
|
186
|
-
}
|
|
187
|
-
if (node === null || typeof node !== "object") return false;
|
|
188
|
-
if (seen.has(node as object)) return false;
|
|
189
|
-
seen.add(node as object);
|
|
190
|
-
|
|
191
|
-
if (Array.isArray(node)) {
|
|
192
|
-
for (const item of node) {
|
|
193
|
-
if (walkForGiven(item, seen, depth + 1)) return true;
|
|
194
|
-
}
|
|
195
|
-
return false;
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
const record = node as Record<string, unknown>;
|
|
199
|
-
|
|
200
|
-
// A given declaration or reference IR node.
|
|
201
|
-
const nodeKind = record.node;
|
|
202
|
-
if (nodeKind === "given" || nodeKind === "givenReference") return true;
|
|
203
|
-
if (record.givenRef !== undefined) return true;
|
|
204
|
-
|
|
205
|
-
// Reference-tracking summary: a non-empty givenUsage means this fragment
|
|
206
|
-
// reads a given (the precise, per-fragment signal). It appears either nested
|
|
207
|
-
// under refSummary (per-fragment) OR as a bare `givenUsage` on a query/struct
|
|
208
|
-
// node — treat a non-empty array in EITHER shape as a hit, so the walk stays
|
|
209
|
-
// correct if a future compiler keeps only the summary and prunes the embedded
|
|
210
|
-
// `node:'given'` leaves.
|
|
211
|
-
const refSummary = record.refSummary as
|
|
212
|
-
| { givenUsage?: unknown[] }
|
|
213
|
-
| undefined;
|
|
214
|
-
if (refSummary?.givenUsage && refSummary.givenUsage.length > 0) return true;
|
|
215
|
-
if (Array.isArray(record.givenUsage) && record.givenUsage.length > 0) {
|
|
216
|
-
return true;
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
for (const value of Object.values(record)) {
|
|
220
|
-
if (walkForGiven(value, seen, depth + 1)) return true;
|
|
221
|
-
}
|
|
222
|
-
return false;
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
/**
|
|
226
|
-
* Whether the compiled source (transitively) carries an `#(authorize)` gate —
|
|
227
|
-
* on the source itself or on any source reachable through a join. Fail-closed:
|
|
228
|
-
* walks the compiled source definition for annotation notes (`blockNotes` /
|
|
229
|
-
* `notes`) whose text is an authorize annotation. A join embeds the joined
|
|
230
|
-
* SourceDef (with its own blockNotes), so a gate reached only through a join is
|
|
231
|
-
* still found — a join must not launder an authorize-gated source, matching the
|
|
232
|
-
* transitive enforcement on the live serve path (#906). Any introspection or
|
|
233
|
-
* parse failure is treated as "carries a gate" (fail closed).
|
|
234
|
-
*
|
|
235
|
-
* Deliberately separate from `Model.collectAllReachableGates`, which answers a
|
|
236
|
-
* different question: that walk collects gates to EVALUATE per request, and needs
|
|
237
|
-
* their expressions and the givens they reference. This one only needs to know
|
|
238
|
-
* whether any gate exists at all, at build time, with no request to evaluate
|
|
239
|
-
* against — so it is a generic fail-closed scan rather than a typed collection.
|
|
240
|
-
* They will not inherit each other's fixes; that is the trade accepted for
|
|
241
|
-
* keeping the build-time refusal independent of the serve-time evaluator.
|
|
242
|
-
*/
|
|
243
|
-
function referencesAuthorize(persistSource: PersistSource): boolean {
|
|
244
|
-
try {
|
|
245
|
-
return walkForAuthorize(persistSource._sourceDef, new WeakSet(), 0);
|
|
246
|
-
} catch {
|
|
247
|
-
// Fail closed: if we cannot prove the source is authorize-free, refuse it.
|
|
248
|
-
return true;
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
/** True if an annotation string is an `#(authorize)`/`##(authorize)` gate. */
|
|
253
|
-
function isAuthorizeAnnotation(text: string): boolean {
|
|
254
|
-
try {
|
|
255
|
-
return parseAuthorizeAnnotation(text) !== null;
|
|
256
|
-
} catch {
|
|
257
|
-
// A malformed authorize annotation still means the author intended a gate.
|
|
258
|
-
return true;
|
|
259
|
-
}
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
function walkForAuthorize(
|
|
263
|
-
node: unknown,
|
|
264
|
-
seen: WeakSet<object>,
|
|
265
|
-
depth: number,
|
|
266
|
-
): boolean {
|
|
267
|
-
if (depth > MAX_GIVEN_WALK_DEPTH) {
|
|
268
|
-
throw new Error("authorize-usage walk exceeded max depth");
|
|
269
|
-
}
|
|
270
|
-
if (node === null || typeof node !== "object") return false;
|
|
271
|
-
if (seen.has(node as object)) return false;
|
|
272
|
-
seen.add(node as object);
|
|
273
|
-
|
|
274
|
-
if (Array.isArray(node)) {
|
|
275
|
-
for (const item of node) {
|
|
276
|
-
if (walkForAuthorize(item, seen, depth + 1)) return true;
|
|
277
|
-
}
|
|
278
|
-
return false;
|
|
279
|
-
}
|
|
280
|
-
|
|
281
|
-
const record = node as Record<string, unknown>;
|
|
282
|
-
|
|
283
|
-
// Annotation notes live under `blockNotes` (source/statement) or `notes`
|
|
284
|
-
// (model/file), as either bare strings or `{ text }` objects.
|
|
285
|
-
for (const key of ["blockNotes", "notes"]) {
|
|
286
|
-
const arr = record[key];
|
|
287
|
-
if (!Array.isArray(arr)) continue;
|
|
288
|
-
for (const n of arr) {
|
|
289
|
-
const text =
|
|
290
|
-
typeof n === "string"
|
|
291
|
-
? n
|
|
292
|
-
: n &&
|
|
293
|
-
typeof n === "object" &&
|
|
294
|
-
typeof (n as { text?: unknown }).text === "string"
|
|
295
|
-
? (n as { text: string }).text
|
|
296
|
-
: undefined;
|
|
297
|
-
if (text !== undefined && isAuthorizeAnnotation(text)) return true;
|
|
298
|
-
}
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
for (const value of Object.values(record)) {
|
|
302
|
-
if (walkForAuthorize(value, seen, depth + 1)) return true;
|
|
303
|
-
}
|
|
304
|
-
return false;
|
|
305
|
-
}
|