@malloy-publisher/server 0.0.204 → 0.0.206
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build.ts +10 -1
- package/dist/app/api-doc.yaml +494 -397
- package/dist/app/assets/{EnvironmentPage-CX06cjOF.js → EnvironmentPage-BYwBeC2F.js} +1 -1
- package/dist/app/assets/HomePage-ivu4vdpj.js +1 -0
- package/dist/app/assets/{MainPage-nUJ9YatG.js → MainPage-B2DnHEDU.js} +2 -2
- package/dist/app/assets/MaterializationsPage-BZEuwF9P.js +1 -0
- package/dist/app/assets/ModelPage-Dpu3bfPg.js +1 -0
- package/dist/app/assets/{PackagePage-BaEVdEAG.js → PackagePage-B8PwcRHt.js} +1 -1
- package/dist/app/assets/{RouteError-BShQjZio.js → RouteError-BhbywAeC.js} +1 -1
- package/dist/app/assets/{WorkbookPage-CBn6ZjJW.js → WorkbookPage-C-JXsJG0.js} +1 -1
- package/dist/app/assets/{core-DECXYL4E.es-OaRfXwuQ.js → core-pPlPr7jK.es-CNEOlxKB.js} +1 -1
- package/dist/app/assets/{index-BLfPC1gy.js → index-BHEm8Egc.js} +1 -1
- package/dist/app/assets/{index-Dy3YhAZQ.js → index-BsvDrV14.js} +1 -1
- package/dist/app/assets/{index-DqiJ0bWp.js → index-ChR1fKR2.js} +134 -134
- package/dist/app/assets/{index.umd-DAN9K8yC.js → index.umd-BVLPYNuj.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +18 -8
- package/dist/package_load_worker.mjs +19 -2
- package/dist/runtime/publisher.js +318 -0
- package/dist/server.mjs +1703 -1443
- package/package.json +5 -4
- package/scripts/bake-duckdb-extensions.js +104 -0
- package/src/constants.ts +12 -0
- package/src/controller/materialization.controller.ts +79 -35
- package/src/controller/package.controller.spec.ts +179 -0
- package/src/controller/package.controller.ts +60 -73
- package/src/controller/watch-mode.controller.ts +176 -46
- package/src/dto/package.dto.ts +16 -1
- package/src/errors.spec.ts +33 -0
- package/src/errors.ts +18 -0
- package/src/health.spec.ts +34 -1
- package/src/instrumentation.ts +33 -17
- package/src/materialization_metrics.ts +66 -0
- package/src/mcp/error_messages.spec.ts +35 -0
- package/src/mcp/error_messages.ts +14 -1
- package/src/mcp/handler_utils.ts +12 -0
- package/src/package_load/package_load_pool.ts +7 -1
- package/src/package_load/package_load_worker.ts +44 -4
- package/src/package_load/protocol.ts +7 -1
- package/src/runtime/publisher.js +318 -0
- package/src/server-old.ts +7 -149
- package/src/server.ts +488 -190
- package/src/service/authorize_integration.spec.ts +163 -2
- package/src/service/compile_authorize.spec.ts +85 -0
- package/src/service/environment.ts +270 -12
- package/src/service/environment_store.spec.ts +0 -81
- package/src/service/environment_store.ts +142 -34
- package/src/service/explore_visibility.spec.ts +434 -0
- package/src/service/exports_probe.spec.ts +107 -0
- package/src/service/manifest_loader.spec.ts +99 -0
- package/src/service/manifest_loader.ts +95 -0
- package/src/service/materialization_service.spec.ts +324 -512
- package/src/service/materialization_service.ts +816 -656
- package/src/service/model.ts +444 -13
- package/src/service/package.spec.ts +14 -2
- package/src/service/package.ts +271 -20
- package/src/service/package_rollback.spec.ts +190 -0
- package/src/service/package_worker_path.spec.ts +223 -0
- package/src/service/query_boundary.spec.ts +470 -0
- package/src/storage/DatabaseInterface.ts +35 -57
- package/src/storage/StorageManager.mock.ts +0 -9
- package/src/storage/StorageManager.ts +7 -290
- package/src/storage/duckdb/DuckDBConnection.ts +70 -124
- package/src/storage/duckdb/DuckDBRepository.ts +2 -35
- package/src/storage/duckdb/MaterializationRepository.ts +52 -27
- package/src/storage/duckdb/schema.ts +4 -20
- package/tests/fixtures/authorize-compile/model.malloy +9 -0
- package/tests/fixtures/authorize-compile/publisher.json +4 -0
- package/tests/fixtures/html-pages-nopublic/model.malloy +1 -0
- package/tests/fixtures/html-pages-nopublic/publisher.json +5 -0
- package/tests/fixtures/html-pages-test/data.csv +3 -0
- package/tests/fixtures/html-pages-test/public/assets/app.css +3 -0
- package/tests/fixtures/html-pages-test/public/data.json +1 -0
- package/tests/fixtures/html-pages-test/public/index.html +9 -0
- package/tests/fixtures/html-pages-test/public/sub/page2.html +9 -0
- package/tests/fixtures/html-pages-test/publisher.json +5 -0
- package/tests/fixtures/html-pages-test/report.malloy +1 -0
- package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +92 -0
- package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +138 -0
- package/tests/integration/html_pages/html_pages.integration.spec.ts +378 -0
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +175 -0
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +277 -266
- package/tests/integration/watch-mode/watch_mode.integration.spec.ts +421 -0
- package/tests/unit/duckdb/attached_databases.test.ts +111 -0
- package/tests/unit/duckdb/duckdb_connection.test.ts +181 -0
- package/tests/unit/duckdb/legacy_schema_migration.test.ts +7 -4
- package/tests/unit/duckdb/repositories.test.ts +208 -0
- package/dist/app/assets/HomePage-CNFt_eUU.js +0 -1
- package/dist/app/assets/MaterializationsPage-B5goxVXW.js +0 -1
- package/dist/app/assets/ModelPage-Ba7Xh4lL.js +0 -1
- package/src/controller/manifest.controller.ts +0 -38
- package/src/service/manifest_service.spec.ts +0 -206
- package/src/service/manifest_service.ts +0 -117
- package/src/service/materialized_table_gc.spec.ts +0 -384
- package/src/service/materialized_table_gc.ts +0 -231
- package/src/storage/duckdb/DuckDBManifestStore.ts +0 -70
- package/src/storage/duckdb/ManifestRepository.ts +0 -120
- package/src/storage/duckdb/manifest_store.spec.ts +0 -133
- package/src/storage/ducklake/DuckLakeManifestStore.ts +0 -155
- package/tests/unit/storage/StorageManager.test.ts +0 -166
|
@@ -7,8 +7,8 @@ import path from "path";
|
|
|
7
7
|
import { AccessDeniedError } from "../errors";
|
|
8
8
|
import { Model } from "./model";
|
|
9
9
|
|
|
10
|
-
// Introspection
|
|
11
|
-
//
|
|
10
|
+
// Introspection, compile-time validation, and the runtime gate for
|
|
11
|
+
// #(authorize) / ##(authorize).
|
|
12
12
|
|
|
13
13
|
const TEST_DIR = path.join(os.tmpdir(), "authorize-integration-tests");
|
|
14
14
|
const TEST_DB_DIR = path.join(TEST_DIR, "db");
|
|
@@ -277,6 +277,11 @@ source: gated is duckdb.table('customers')
|
|
|
277
277
|
// Names the source and surfaces the underlying Malloy reason.
|
|
278
278
|
expect(err?.message).toContain("gated");
|
|
279
279
|
expect(err?.message).toMatch(/NOPE|not declared/i);
|
|
280
|
+
// Redaction policy (pinned): the model-load 424 is author-facing, so it
|
|
281
|
+
// KEEPS the full expression text (needed to fix a malformed annotation).
|
|
282
|
+
// Only the runtime 403 redacts to the source name. If this assertion ever
|
|
283
|
+
// flips, the redaction split was changed — make it a conscious decision.
|
|
284
|
+
expect(err?.message).toContain("$NOPE = 'x'");
|
|
280
285
|
});
|
|
281
286
|
|
|
282
287
|
it("fails model load when an expression references a source field", async () => {
|
|
@@ -836,3 +841,159 @@ source: joiner is duckdb.table('customers') extend {
|
|
|
836
841
|
expect(result.data).toBeDefined();
|
|
837
842
|
});
|
|
838
843
|
});
|
|
844
|
+
|
|
845
|
+
// The /compile path gates via Model.assertAuthorizedForText (early,
|
|
846
|
+
// surface-syntax) and Model.assertAuthorizedForRunnable (compiled-source
|
|
847
|
+
// backstop). These are the enforcement primitives environment.compileSource
|
|
848
|
+
// calls; exercise them directly here.
|
|
849
|
+
describe("authorize compile-path gate", () => {
|
|
850
|
+
const CP_GATE = `##! experimental.givens
|
|
851
|
+
|
|
852
|
+
given:
|
|
853
|
+
ROLE :: string
|
|
854
|
+
|
|
855
|
+
#(authorize) "$ROLE = 'analyst'"
|
|
856
|
+
source: gated is duckdb.table('customers') extend { measure: c is count() }
|
|
857
|
+
|
|
858
|
+
source: open_src is duckdb.table('customers') extend { measure: c is count() }
|
|
859
|
+
`;
|
|
860
|
+
const CP_FILE_LEVEL = `##! experimental.givens
|
|
861
|
+
|
|
862
|
+
given:
|
|
863
|
+
ROLE :: string
|
|
864
|
+
|
|
865
|
+
##(authorize) "$ROLE = 'admin'"
|
|
866
|
+
|
|
867
|
+
source: declared is duckdb.table('customers') extend { measure: c is count() }
|
|
868
|
+
`;
|
|
869
|
+
|
|
870
|
+
async function cpModel(file: string, src: string): Promise<Model> {
|
|
871
|
+
await writeModel(file, src);
|
|
872
|
+
return Model.create("test-pkg", TEST_PKG_DIR, file, getConnections());
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
it("assertAuthorizedForText denies/allows a gated named source by its given", async () => {
|
|
876
|
+
const model = await cpModel("cp_gate.malloy", CP_GATE);
|
|
877
|
+
await expect(
|
|
878
|
+
model.assertAuthorizedForText("run: gated -> { aggregate: c }", {}),
|
|
879
|
+
).rejects.toBeInstanceOf(AccessDeniedError);
|
|
880
|
+
await expect(
|
|
881
|
+
model.assertAuthorizedForText("run: gated -> { aggregate: c }", {
|
|
882
|
+
ROLE: "analyst",
|
|
883
|
+
}),
|
|
884
|
+
).resolves.toBeUndefined();
|
|
885
|
+
});
|
|
886
|
+
|
|
887
|
+
it("assertAuthorizedForText leaves an ungated source unrestricted", async () => {
|
|
888
|
+
const model = await cpModel("cp_gate.malloy", CP_GATE);
|
|
889
|
+
await expect(
|
|
890
|
+
model.assertAuthorizedForText("run: open_src -> { aggregate: c }", {}),
|
|
891
|
+
).resolves.toBeUndefined();
|
|
892
|
+
});
|
|
893
|
+
|
|
894
|
+
it("assertAuthorizedForText applies the model-wide file-level gate to inline/unnamed text", async () => {
|
|
895
|
+
const model = await cpModel("cp_file.malloy", CP_FILE_LEVEL);
|
|
896
|
+
// No named source the regex recognizes -> undefined -> file-level gate.
|
|
897
|
+
await expect(
|
|
898
|
+
model.assertAuthorizedForText(
|
|
899
|
+
`run: duckdb.sql("SELECT 1 AS x") -> { aggregate: n is count() }`,
|
|
900
|
+
{},
|
|
901
|
+
),
|
|
902
|
+
).rejects.toBeInstanceOf(AccessDeniedError);
|
|
903
|
+
await expect(
|
|
904
|
+
model.assertAuthorizedForText(
|
|
905
|
+
`run: duckdb.sql("SELECT 1 AS x") -> { aggregate: n is count() }`,
|
|
906
|
+
{ ROLE: "admin" },
|
|
907
|
+
),
|
|
908
|
+
).resolves.toBeUndefined();
|
|
909
|
+
});
|
|
910
|
+
|
|
911
|
+
it("assertAuthorizedForRunnable gates the compiled-source structRef (alias backstop)", async () => {
|
|
912
|
+
const model = await cpModel("cp_gate.malloy", CP_GATE);
|
|
913
|
+
// Stub a runnable whose compiled query reads `gated` (e.g. via an alias
|
|
914
|
+
// the surface-syntax gate would miss).
|
|
915
|
+
const gatedRunnable = {
|
|
916
|
+
getPreparedQuery: async () => ({ _query: { structRef: "gated" } }),
|
|
917
|
+
};
|
|
918
|
+
await expect(
|
|
919
|
+
model.assertAuthorizedForRunnable(gatedRunnable, {}),
|
|
920
|
+
).rejects.toBeInstanceOf(AccessDeniedError);
|
|
921
|
+
await expect(
|
|
922
|
+
model.assertAuthorizedForRunnable(gatedRunnable, { ROLE: "analyst" }),
|
|
923
|
+
).resolves.toBeUndefined();
|
|
924
|
+
// Ungated compiled source -> unrestricted.
|
|
925
|
+
const openRunnable = {
|
|
926
|
+
getPreparedQuery: async () => ({ _query: { structRef: "open_src" } }),
|
|
927
|
+
};
|
|
928
|
+
await expect(
|
|
929
|
+
model.assertAuthorizedForRunnable(openRunnable, {}),
|
|
930
|
+
).resolves.toBeUndefined();
|
|
931
|
+
});
|
|
932
|
+
});
|
|
933
|
+
|
|
934
|
+
// A given can be both a runtime query parameter (substituted into a view's
|
|
935
|
+
// `where`) AND the subject of an `#(authorize)` gate on the same source. These
|
|
936
|
+
// two layers compose: the gate fires first (before any filtering), and once it
|
|
937
|
+
// passes the parameterized filter applies to the result. This is the layered
|
|
938
|
+
// case the end-to-end story rests on.
|
|
939
|
+
describe("authorize composes with given parameters", () => {
|
|
940
|
+
// `ROLE` gates access; `REGION` is a runtime filter parameter on the view.
|
|
941
|
+
const COMPOSED = `##! experimental.givens
|
|
942
|
+
|
|
943
|
+
given:
|
|
944
|
+
ROLE :: string
|
|
945
|
+
REGION :: string
|
|
946
|
+
|
|
947
|
+
#(authorize) "$ROLE = 'analyst'"
|
|
948
|
+
source: regional is duckdb.table('customers') extend {
|
|
949
|
+
measure: c is count()
|
|
950
|
+
view: in_region is {
|
|
951
|
+
where: region = $REGION
|
|
952
|
+
aggregate: c
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
`;
|
|
956
|
+
|
|
957
|
+
async function runComposed(givens: Record<string, GivenValue>) {
|
|
958
|
+
await writeModel("compose.malloy", COMPOSED);
|
|
959
|
+
const model = await Model.create(
|
|
960
|
+
"test-pkg",
|
|
961
|
+
TEST_PKG_DIR,
|
|
962
|
+
"compose.malloy",
|
|
963
|
+
getConnections(),
|
|
964
|
+
);
|
|
965
|
+
return model.getQueryResults(
|
|
966
|
+
undefined,
|
|
967
|
+
undefined,
|
|
968
|
+
"run: regional -> in_region",
|
|
969
|
+
undefined,
|
|
970
|
+
undefined,
|
|
971
|
+
givens,
|
|
972
|
+
);
|
|
973
|
+
}
|
|
974
|
+
|
|
975
|
+
function countOf(compactResult: unknown): number {
|
|
976
|
+
const rows = compactResult as Array<Record<string, unknown>>;
|
|
977
|
+
return Number(rows[0]?.c);
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
it("denies before filtering when the gate fails, even with the filter given supplied", async () => {
|
|
981
|
+
await expect(runComposed({ REGION: "us-west" })).rejects.toBeInstanceOf(
|
|
982
|
+
AccessDeniedError,
|
|
983
|
+
);
|
|
984
|
+
});
|
|
985
|
+
|
|
986
|
+
it("applies the parameterized filter once the gate passes", async () => {
|
|
987
|
+
// Seed: one 'us-west' customer, one 'us-east'.
|
|
988
|
+
const west = await runComposed({ ROLE: "analyst", REGION: "us-west" });
|
|
989
|
+
expect(countOf(west.compactResult)).toBe(1);
|
|
990
|
+
|
|
991
|
+
const east = await runComposed({ ROLE: "analyst", REGION: "us-east" });
|
|
992
|
+
expect(countOf(east.compactResult)).toBe(1);
|
|
993
|
+
|
|
994
|
+
// A region matching no rows proves the given genuinely filters (not a
|
|
995
|
+
// gate-only pass-through).
|
|
996
|
+
const none = await runComposed({ ROLE: "analyst", REGION: "nowhere" });
|
|
997
|
+
expect(countOf(none.compactResult)).toBe(0);
|
|
998
|
+
});
|
|
999
|
+
});
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
|
2
|
+
import fs from "fs/promises";
|
|
3
|
+
import os from "os";
|
|
4
|
+
import path from "path";
|
|
5
|
+
import { AccessDeniedError } from "../errors";
|
|
6
|
+
import { Environment } from "./environment";
|
|
7
|
+
|
|
8
|
+
// End-to-end gate on the /compile path. Exercises environment.compileSource
|
|
9
|
+
// through a real installed package, not just the Model primitives — pins that
|
|
10
|
+
// the early gate AND the compiled-source backstop fire, the latter REGARDLESS of
|
|
11
|
+
// includeSql (a compile-time schema oracle is closed even with no SQL extraction).
|
|
12
|
+
|
|
13
|
+
const PUBLISHER_JSON = JSON.stringify({
|
|
14
|
+
name: "pkg",
|
|
15
|
+
description: "compile-gate",
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
// `gated` is locked to $ROLE='analyst'; `open_src` is unrestricted.
|
|
19
|
+
const MODEL = `##! experimental.givens
|
|
20
|
+
|
|
21
|
+
given:
|
|
22
|
+
ROLE :: string
|
|
23
|
+
|
|
24
|
+
#(authorize) "$ROLE = 'analyst'"
|
|
25
|
+
source: gated is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }
|
|
26
|
+
|
|
27
|
+
source: open_src is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }
|
|
28
|
+
`;
|
|
29
|
+
|
|
30
|
+
describe("compile-path authorize gate (compileSource)", () => {
|
|
31
|
+
let rootDir: string;
|
|
32
|
+
let env: Environment;
|
|
33
|
+
|
|
34
|
+
beforeEach(async () => {
|
|
35
|
+
rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-compile-"));
|
|
36
|
+
const envPath = path.join(rootDir, "env");
|
|
37
|
+
await fs.mkdir(envPath, { recursive: true });
|
|
38
|
+
env = await Environment.create("testEnv", envPath, []);
|
|
39
|
+
await env.installPackage("pkg", async (stagingPath) => {
|
|
40
|
+
await fs.mkdir(stagingPath, { recursive: true });
|
|
41
|
+
await fs.writeFile(
|
|
42
|
+
path.join(stagingPath, "publisher.json"),
|
|
43
|
+
PUBLISHER_JSON,
|
|
44
|
+
);
|
|
45
|
+
await fs.writeFile(path.join(stagingPath, "model.malloy"), MODEL);
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
afterEach(async () => {
|
|
50
|
+
await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
const compile = (source: string, givens?: Record<string, string>) =>
|
|
54
|
+
env.compileSource("pkg", "model.malloy", source, false, givens);
|
|
55
|
+
|
|
56
|
+
it("denies a direct gated source without the satisfying given (early gate)", async () => {
|
|
57
|
+
await expect(
|
|
58
|
+
compile("run: gated -> { aggregate: c }"),
|
|
59
|
+
).rejects.toBeInstanceOf(AccessDeniedError);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it("denies a gated source reached via the LAST run: statement (backstop, includeSql=false)", async () => {
|
|
63
|
+
// Regression guard: the early gate only matches the first `run:` (ungated
|
|
64
|
+
// open_src here), so the gated source in the executed final statement is
|
|
65
|
+
// caught only by the compiled-source backstop — which must run even when
|
|
66
|
+
// no SQL is requested.
|
|
67
|
+
await expect(
|
|
68
|
+
compile(
|
|
69
|
+
"run: open_src -> { aggregate: c }\nrun: gated -> { aggregate: c }",
|
|
70
|
+
),
|
|
71
|
+
).rejects.toBeInstanceOf(AccessDeniedError);
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it("allows the gated source when the given satisfies the gate", async () => {
|
|
75
|
+
const { problems } = await compile("run: gated -> { aggregate: c }", {
|
|
76
|
+
ROLE: "analyst",
|
|
77
|
+
});
|
|
78
|
+
expect(problems).toBeDefined();
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it("leaves an ungated source compilable without any given", async () => {
|
|
82
|
+
const { problems } = await compile("run: open_src -> { aggregate: c }");
|
|
83
|
+
expect(problems).toBeDefined();
|
|
84
|
+
});
|
|
85
|
+
});
|
|
@@ -5,9 +5,16 @@ import { Mutex } from "async-mutex";
|
|
|
5
5
|
import crypto from "crypto";
|
|
6
6
|
import * as fs from "fs";
|
|
7
7
|
import * as path from "path";
|
|
8
|
+
import { pathToFileURL } from "url";
|
|
8
9
|
import { components } from "../api";
|
|
9
|
-
import { API_PREFIX, README_NAME } from "../constants";
|
|
10
10
|
import {
|
|
11
|
+
API_PREFIX,
|
|
12
|
+
normalizeModelPath,
|
|
13
|
+
NOTEBOOK_FILE_SUFFIX,
|
|
14
|
+
README_NAME,
|
|
15
|
+
} from "../constants";
|
|
16
|
+
import {
|
|
17
|
+
BadRequestError,
|
|
11
18
|
ConnectionNotFoundError,
|
|
12
19
|
EnvironmentNotFoundError,
|
|
13
20
|
PackageNotFoundError,
|
|
@@ -28,6 +35,7 @@ import {
|
|
|
28
35
|
EnvironmentMalloyConfig,
|
|
29
36
|
InternalConnection,
|
|
30
37
|
} from "./connection";
|
|
38
|
+
import { fetchManifestEntries } from "./manifest_loader";
|
|
31
39
|
import { ApiConnection } from "./model";
|
|
32
40
|
import { Package } from "./package";
|
|
33
41
|
import type { PackageMemoryGovernor } from "./package_memory_governor";
|
|
@@ -137,6 +145,11 @@ export class Environment {
|
|
|
137
145
|
// governor as the single owner of the back-pressure boolean.
|
|
138
146
|
private memoryGovernor: PackageMemoryGovernor | null = null;
|
|
139
147
|
|
|
148
|
+
/** Absolute path on disk where this environment's package files live. */
|
|
149
|
+
public getEnvironmentPath(): string {
|
|
150
|
+
return this.environmentPath;
|
|
151
|
+
}
|
|
152
|
+
|
|
140
153
|
constructor(
|
|
141
154
|
environmentName: string,
|
|
142
155
|
environmentPath: string,
|
|
@@ -181,10 +194,6 @@ export class Environment {
|
|
|
181
194
|
await this.writeEnvironmentReadme(payload.readme);
|
|
182
195
|
}
|
|
183
196
|
|
|
184
|
-
if (payload.materializationStorage !== undefined) {
|
|
185
|
-
this.metadata.materializationStorage = payload.materializationStorage;
|
|
186
|
-
}
|
|
187
|
-
|
|
188
197
|
// Handle connections update
|
|
189
198
|
// TODO: Update environment connections should have its own API endpoint
|
|
190
199
|
if (payload.connections) {
|
|
@@ -290,6 +299,17 @@ export class Environment {
|
|
|
290
299
|
): Promise<{ problems: LogMessage[]; sql?: string }> {
|
|
291
300
|
assertSafePackageName(packageName);
|
|
292
301
|
assertSafeRelativeModelPath(modelName);
|
|
302
|
+
// /compile appends the submitted source to the TARGET MODEL's content for
|
|
303
|
+
// namespace context. A notebook (.malloynb) is markdown + cells, not a
|
|
304
|
+
// model, so compiling against it only yields a confusing parse error —
|
|
305
|
+
// reject it up front with an actionable message. (Notebooks remain public
|
|
306
|
+
// for discovery/query; this is specific to the compile context.)
|
|
307
|
+
if (modelName.endsWith(NOTEBOOK_FILE_SUFFIX)) {
|
|
308
|
+
throw new BadRequestError(
|
|
309
|
+
`Cannot compile against a notebook ("${modelName}"). ` +
|
|
310
|
+
`/compile takes a .malloy model path for namespace context.`,
|
|
311
|
+
);
|
|
312
|
+
}
|
|
293
313
|
// Hold the per-package mutex for the duration of every disk read —
|
|
294
314
|
// both the explicit `fs.readFile(modelPath)` below and the implicit
|
|
295
315
|
// import resolution that `runtime.loadModel` does through the URL
|
|
@@ -307,10 +327,18 @@ export class Environment {
|
|
|
307
327
|
packageName,
|
|
308
328
|
modelName,
|
|
309
329
|
);
|
|
310
|
-
// Place the virtual file in the model's directory so relative imports
|
|
330
|
+
// Place the virtual file in the model's directory so relative imports
|
|
331
|
+
// resolve correctly. Use `pathToFileURL` rather than hand-prefixing
|
|
332
|
+
// `file://`: on Windows the latter produces a malformed URL
|
|
333
|
+
// (`file://D:\Temp\…`) that round-trips differently than the URL the
|
|
334
|
+
// Malloy runtime synthesizes from the same path, breaking the
|
|
335
|
+
// intercepting reader's string comparison below and falling through
|
|
336
|
+
// to disk for a virtual file that doesn't exist.
|
|
311
337
|
const modelDir = path.dirname(modelPath);
|
|
312
|
-
const
|
|
313
|
-
|
|
338
|
+
const virtualUrl = pathToFileURL(
|
|
339
|
+
path.join(modelDir, "__compile_check.malloy"),
|
|
340
|
+
);
|
|
341
|
+
const virtualUri = virtualUrl.toString();
|
|
314
342
|
|
|
315
343
|
// Read the full model file so the submitted source inherits the model's
|
|
316
344
|
// complete namespace — imports, source definitions, queries, etc.
|
|
@@ -339,6 +367,29 @@ export class Environment {
|
|
|
339
367
|
// Use the locked variant — we already hold the per-package mutex.
|
|
340
368
|
const pkg = await this._loadOrGetPackageLocked(packageName);
|
|
341
369
|
|
|
370
|
+
// Authorize gate: /compile is compile-only, but it can still act
|
|
371
|
+
// as a schema oracle (a denied caller learns a gated source's columns
|
|
372
|
+
// from compile errors) and, with includeSql, leak its SQL. Gate the
|
|
373
|
+
// named source the submitted text targets BEFORE compiling — mirrors
|
|
374
|
+
// the query path's early surface-syntax gate. Unnamed/inline source
|
|
375
|
+
// text resolves to undefined, so only the model-wide file-level gate
|
|
376
|
+
// applies. The gate runs against the package's cached Model (its
|
|
377
|
+
// `given:` block + authorize annotations), independent of the virtual
|
|
378
|
+
// compile below. If the model isn't loaded, there's nothing to enforce
|
|
379
|
+
// and compilation surfaces its own error.
|
|
380
|
+
const gateModel = pkg.getModel(modelName);
|
|
381
|
+
if (gateModel) {
|
|
382
|
+
// Query boundary first (the *what* axis): /compile compiles ad-hoc
|
|
383
|
+
// text against a model, so gate it like an ad-hoc query — a
|
|
384
|
+
// non-`explores` model file, or text whose surface-resolved target
|
|
385
|
+
// is a non-curated model source (under queryableSources:
|
|
386
|
+
// "declared"), is rejected with a generic 404 before compilation
|
|
387
|
+
// can leak schema/SQL. Text the early gate can't pin is settled by
|
|
388
|
+
// the compiled backstop below.
|
|
389
|
+
gateModel.assertQueryBoundaryEarly(undefined, undefined, source);
|
|
390
|
+
await gateModel.assertAuthorizedForText(source, givens ?? {});
|
|
391
|
+
}
|
|
392
|
+
|
|
342
393
|
// Initialize Runtime with the package's active MalloyConfig so compile
|
|
343
394
|
// checks see the same package-scoped duckdb as execution. This runtime
|
|
344
395
|
// borrows the package config; the package/environment lifecycle owns release.
|
|
@@ -352,11 +403,57 @@ export class Environment {
|
|
|
352
403
|
const modelMaterializer = runtime.loadModel(virtualUrl);
|
|
353
404
|
const model = await modelMaterializer.getModel();
|
|
354
405
|
|
|
406
|
+
// Resolve the final query's materializer once (if there is one).
|
|
407
|
+
let queryMaterializer: ReturnType<
|
|
408
|
+
typeof modelMaterializer.loadFinalQuery
|
|
409
|
+
> | null = null;
|
|
410
|
+
try {
|
|
411
|
+
queryMaterializer = modelMaterializer.loadFinalQuery();
|
|
412
|
+
} catch {
|
|
413
|
+
// No runnable query (e.g. only source definitions) — nothing to
|
|
414
|
+
// gate or extract beyond the early text gate already applied.
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
// Compiled-source backstops — run REGARDLESS of includeSql. They
|
|
418
|
+
// gate the source the COMPILED final query actually reads, closing
|
|
419
|
+
// named-query / multi-statement indirection the early surface-syntax
|
|
420
|
+
// gate misses (e.g. `run: ungated\nrun: gated` — the early gate only
|
|
421
|
+
// matches the FIRST `run:`, but the LAST statement is what executes).
|
|
422
|
+
// Compiling a gated source even without SQL is a schema oracle
|
|
423
|
+
// (field-not-found errors leak its columns), so this must not be
|
|
424
|
+
// conditional on SQL extraction. (A `source: x is gated` alias makes
|
|
425
|
+
// a new ungated source — that's the documented "extend doesn't
|
|
426
|
+
// inherit authorize" footgun, the same as the query path.)
|
|
427
|
+
|
|
428
|
+
// Boundary backstop (the *what* axis, 404) before the authorize
|
|
429
|
+
// one (the *who* axis, 403). /compile text is always ad-hoc — the
|
|
430
|
+
// early gate can only positively deny, never fully clear — so the
|
|
431
|
+
// compiled final query's run target is the authority. Self-gates
|
|
432
|
+
// internally (no-ops when the boundary is inert: "all" / no
|
|
433
|
+
// explores), so it is deliberately NOT guarded by hasAuthorize().
|
|
434
|
+
// Text that compiles only source definitions (no final query) has
|
|
435
|
+
// no run target and nothing to gate.
|
|
436
|
+
if (queryMaterializer && gateModel) {
|
|
437
|
+
await gateModel.assertQueryBoundaryForRunnable(
|
|
438
|
+
queryMaterializer,
|
|
439
|
+
source,
|
|
440
|
+
);
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
// Authorize backstop (the *who* axis, 403). Only run when the model
|
|
444
|
+
// declares gates so ungated compiles don't pay for the extra
|
|
445
|
+
// final-query compile.
|
|
446
|
+
if (queryMaterializer && gateModel?.hasAuthorize()) {
|
|
447
|
+
await gateModel.assertAuthorizedForRunnable(
|
|
448
|
+
queryMaterializer,
|
|
449
|
+
givens ?? {},
|
|
450
|
+
);
|
|
451
|
+
}
|
|
452
|
+
|
|
355
453
|
// If includeSql is requested and compilation succeeded, attempt to extract SQL
|
|
356
454
|
let sql: string | undefined;
|
|
357
|
-
if (includeSql) {
|
|
455
|
+
if (includeSql && queryMaterializer) {
|
|
358
456
|
try {
|
|
359
|
-
const queryMaterializer = modelMaterializer.loadFinalQuery();
|
|
360
457
|
sql = await queryMaterializer.getSQL({ givens });
|
|
361
458
|
} catch {
|
|
362
459
|
// Source may not contain a runnable query (e.g. only source definitions),
|
|
@@ -734,6 +831,7 @@ export class Environment {
|
|
|
734
831
|
packagePath,
|
|
735
832
|
() => this.malloyConfig.malloyConfig,
|
|
736
833
|
);
|
|
834
|
+
await this.bindManifestIfConfigured(_package);
|
|
737
835
|
if (existingPackage !== undefined && reload) {
|
|
738
836
|
this.retireConnectionGeneration(`package ${packageName}`, () =>
|
|
739
837
|
existingPackage.getMalloyConfig().shutdown("close"),
|
|
@@ -839,6 +937,7 @@ export class Environment {
|
|
|
839
937
|
public async installPackage(
|
|
840
938
|
packageName: string,
|
|
841
939
|
downloader: (stagingPath: string) => Promise<void>,
|
|
940
|
+
validate?: (pkg: Package) => string | undefined,
|
|
842
941
|
): Promise<Package> {
|
|
843
942
|
assertSafePackageName(packageName);
|
|
844
943
|
const stagingPath = this.allocateStagingPath(packageName);
|
|
@@ -905,6 +1004,15 @@ export class Environment {
|
|
|
905
1004
|
canonicalPath,
|
|
906
1005
|
() => this.malloyConfig.malloyConfig,
|
|
907
1006
|
);
|
|
1007
|
+
// Strict-reject hook (publish/update only — reload passes no
|
|
1008
|
+
// validator and stays fail-safe). Throw INSIDE the try so the
|
|
1009
|
+
// catch below rolls the swap back: the just-installed tree is
|
|
1010
|
+
// wiped and the retired tree (if any) is restored, so a rejected
|
|
1011
|
+
// publish/update never leaves the bad tree served on disk.
|
|
1012
|
+
const validationMsg = validate?.(newPackage);
|
|
1013
|
+
if (validationMsg) {
|
|
1014
|
+
throw new BadRequestError(validationMsg);
|
|
1015
|
+
}
|
|
908
1016
|
logger.debug("install.phase2.committed", {
|
|
909
1017
|
environmentName: this.environmentName,
|
|
910
1018
|
packageName,
|
|
@@ -950,6 +1058,11 @@ export class Environment {
|
|
|
950
1058
|
throw err;
|
|
951
1059
|
}
|
|
952
1060
|
|
|
1061
|
+
// Best-effort manifest bind happens after the swap commits, outside the
|
|
1062
|
+
// rollback window: a manifest that can't be fetched must not undo an
|
|
1063
|
+
// otherwise-successful install (the package serves live instead).
|
|
1064
|
+
await this.bindManifestIfConfigured(newPackage);
|
|
1065
|
+
|
|
953
1066
|
this.packages.set(packageName, newPackage);
|
|
954
1067
|
this.setPackageStatus(packageName, PackageStatus.SERVING);
|
|
955
1068
|
|
|
@@ -1004,6 +1117,47 @@ export class Environment {
|
|
|
1004
1117
|
});
|
|
1005
1118
|
}
|
|
1006
1119
|
|
|
1120
|
+
/**
|
|
1121
|
+
* If the freshly-loaded package declares a `manifestLocation`, fetch the
|
|
1122
|
+
* control-plane-computed build manifest and rebind its models so persist
|
|
1123
|
+
* references resolve to the materialized tables. Best-effort: a fetch/bind
|
|
1124
|
+
* failure logs a warning and leaves the package serving live (the models are
|
|
1125
|
+
* already loaded without a manifest). Callers must hold the package lock —
|
|
1126
|
+
* this rebinds `pkg` in place rather than re-entering {@link withPackageLock}.
|
|
1127
|
+
*/
|
|
1128
|
+
private async bindManifestIfConfigured(pkg: Package): Promise<void> {
|
|
1129
|
+
const manifestLocation = pkg.getPackageMetadata().manifestLocation;
|
|
1130
|
+
if (!manifestLocation) {
|
|
1131
|
+
return;
|
|
1132
|
+
}
|
|
1133
|
+
await this.bindManifest(pkg, manifestLocation);
|
|
1134
|
+
}
|
|
1135
|
+
|
|
1136
|
+
/** Fetch + bind a specific manifest URI onto an already-loaded package. */
|
|
1137
|
+
private async bindManifest(
|
|
1138
|
+
pkg: Package,
|
|
1139
|
+
manifestLocation: string,
|
|
1140
|
+
): Promise<void> {
|
|
1141
|
+
const packageName = pkg.getPackageName();
|
|
1142
|
+
try {
|
|
1143
|
+
const entries = await fetchManifestEntries(manifestLocation);
|
|
1144
|
+
await pkg.reloadAllModels(entries);
|
|
1145
|
+
logger.info("Bound build manifest to package", {
|
|
1146
|
+
environmentName: this.environmentName,
|
|
1147
|
+
packageName,
|
|
1148
|
+
manifestLocation,
|
|
1149
|
+
entryCount: Object.keys(entries).length,
|
|
1150
|
+
});
|
|
1151
|
+
} catch (err) {
|
|
1152
|
+
logger.warn("Failed to bind build manifest; serving live", {
|
|
1153
|
+
environmentName: this.environmentName,
|
|
1154
|
+
packageName,
|
|
1155
|
+
manifestLocation,
|
|
1156
|
+
error: err instanceof Error ? err.message : String(err),
|
|
1157
|
+
});
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1007
1161
|
/**
|
|
1008
1162
|
* Read a model's source text from disk, holding the per-package mutex
|
|
1009
1163
|
* so the read is serialized against {@link installPackage} /
|
|
@@ -1028,7 +1182,13 @@ export class Environment {
|
|
|
1028
1182
|
|
|
1029
1183
|
private async writePackageManifest(
|
|
1030
1184
|
packageName: string,
|
|
1031
|
-
metadata: {
|
|
1185
|
+
metadata: {
|
|
1186
|
+
name: string;
|
|
1187
|
+
description?: string;
|
|
1188
|
+
explores?: string[];
|
|
1189
|
+
queryableSources?: "declared" | "all";
|
|
1190
|
+
manifestLocation?: string | null;
|
|
1191
|
+
},
|
|
1032
1192
|
): Promise<void> {
|
|
1033
1193
|
const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
1034
1194
|
const manifestPath = safeJoinUnderRoot(packagePath, "publisher.json");
|
|
@@ -1043,11 +1203,23 @@ export class Environment {
|
|
|
1043
1203
|
logger.warn(`Could not read manifest for ${packageName}`);
|
|
1044
1204
|
}
|
|
1045
1205
|
|
|
1046
|
-
// Update with new metadata
|
|
1206
|
+
// Update with new metadata. `explores`/`queryableSources` are only
|
|
1207
|
+
// overwritten when the caller explicitly provides them; otherwise the
|
|
1208
|
+
// existing on-disk value is preserved via the spread (an undefined here
|
|
1209
|
+
// must not erase it).
|
|
1047
1210
|
const updatedManifest = {
|
|
1048
1211
|
...existingManifest,
|
|
1049
1212
|
name: metadata.name,
|
|
1050
1213
|
description: metadata.description,
|
|
1214
|
+
...(metadata.explores !== undefined
|
|
1215
|
+
? { explores: metadata.explores }
|
|
1216
|
+
: {}),
|
|
1217
|
+
...(metadata.queryableSources !== undefined
|
|
1218
|
+
? { queryableSources: metadata.queryableSources }
|
|
1219
|
+
: {}),
|
|
1220
|
+
...(metadata.manifestLocation !== undefined
|
|
1221
|
+
? { manifestLocation: metadata.manifestLocation }
|
|
1222
|
+
: {}),
|
|
1051
1223
|
};
|
|
1052
1224
|
|
|
1053
1225
|
// Write back to file
|
|
@@ -1074,18 +1246,72 @@ export class Environment {
|
|
|
1074
1246
|
if (body.name) {
|
|
1075
1247
|
_package.setName(body.name);
|
|
1076
1248
|
}
|
|
1249
|
+
// Preserve `explores` across a metadata PATCH. `setPackageMetadata`
|
|
1250
|
+
// replaces the whole object, so a name/description-only update must
|
|
1251
|
+
// carry the existing discovery surface through — otherwise the
|
|
1252
|
+
// in-memory `explores` is wiped and `listModels()` silently starts
|
|
1253
|
+
// serving every model until the next reload. When the body explicitly
|
|
1254
|
+
// carries `explores`, honor the new set instead.
|
|
1255
|
+
const existing = _package.getPackageMetadata();
|
|
1256
|
+
// Normalize API-body explores through the same helper the worker uses
|
|
1257
|
+
// for on-disk explores, so `["./index.malloy"]` / backslash paths
|
|
1258
|
+
// validate and persist identically regardless of input channel (no
|
|
1259
|
+
// misleading publish-time 400, no publish-vs-reload divergence).
|
|
1260
|
+
const normalizedExplores = body.explores?.map(normalizeModelPath);
|
|
1261
|
+
const explores =
|
|
1262
|
+
normalizedExplores !== undefined
|
|
1263
|
+
? normalizedExplores
|
|
1264
|
+
: existing.explores;
|
|
1265
|
+
const queryableSources =
|
|
1266
|
+
body.queryableSources !== undefined
|
|
1267
|
+
? body.queryableSources
|
|
1268
|
+
: existing.queryableSources;
|
|
1269
|
+
// Preserve the existing manifestLocation unless the body explicitly
|
|
1270
|
+
// sets it (including to null, which clears it and reverts to live).
|
|
1271
|
+
const manifestLocation =
|
|
1272
|
+
body.manifestLocation !== undefined
|
|
1273
|
+
? body.manifestLocation
|
|
1274
|
+
: existing.manifestLocation;
|
|
1077
1275
|
_package.setPackageMetadata({
|
|
1078
1276
|
name: body.name,
|
|
1079
1277
|
description: body.description,
|
|
1080
1278
|
resource: body.resource,
|
|
1081
1279
|
location: body.location,
|
|
1280
|
+
explores,
|
|
1281
|
+
queryableSources,
|
|
1282
|
+
manifestLocation,
|
|
1082
1283
|
});
|
|
1083
1284
|
|
|
1285
|
+
// Strict-reject, symmetric with the publish path
|
|
1286
|
+
// (package.controller.addPackage): validate the resulting explores
|
|
1287
|
+
// against the live model set and restore the prior metadata before
|
|
1288
|
+
// rejecting, so a bad update neither persists nor mutates the served
|
|
1289
|
+
// surface.
|
|
1290
|
+
const invalidMsg = _package.formatInvalidExplores();
|
|
1291
|
+
if (invalidMsg) {
|
|
1292
|
+
_package.setPackageMetadata(existing);
|
|
1293
|
+
throw new BadRequestError(invalidMsg);
|
|
1294
|
+
}
|
|
1295
|
+
|
|
1084
1296
|
await this.writePackageManifest(packageName, {
|
|
1085
1297
|
name: packageName,
|
|
1086
1298
|
description: body.description,
|
|
1299
|
+
explores: normalizedExplores,
|
|
1300
|
+
queryableSources: body.queryableSources,
|
|
1301
|
+
manifestLocation: body.manifestLocation,
|
|
1087
1302
|
});
|
|
1088
1303
|
|
|
1304
|
+
// When the body changes manifestLocation, apply it now so the new
|
|
1305
|
+
// binding takes effect without a separate reload: a URI rebinds models
|
|
1306
|
+
// to the materialized tables; null/empty reverts the package to live.
|
|
1307
|
+
if (body.manifestLocation !== undefined) {
|
|
1308
|
+
if (body.manifestLocation) {
|
|
1309
|
+
await this.bindManifest(_package, body.manifestLocation);
|
|
1310
|
+
} else {
|
|
1311
|
+
await _package.reloadAllModels({});
|
|
1312
|
+
}
|
|
1313
|
+
}
|
|
1314
|
+
|
|
1089
1315
|
return _package.getPackageMetadata();
|
|
1090
1316
|
});
|
|
1091
1317
|
}
|
|
@@ -1187,6 +1413,38 @@ export class Environment {
|
|
|
1187
1413
|
});
|
|
1188
1414
|
}
|
|
1189
1415
|
|
|
1416
|
+
/**
|
|
1417
|
+
* Evict a package from the in-memory caches WITHOUT touching its on-disk
|
|
1418
|
+
* directory — the non-destructive counterpart to {@link deletePackage}.
|
|
1419
|
+
*
|
|
1420
|
+
* Used to roll back a no-location `addPackage` (which registers a
|
|
1421
|
+
* *pre-existing*, user-owned directory) when post-load validation rejects
|
|
1422
|
+
* it: deleting the tree there would destroy content the publisher never
|
|
1423
|
+
* created. This still drains and closes the connections the just-created
|
|
1424
|
+
* `Package` opened, so the duckdb handle isn't leaked.
|
|
1425
|
+
*/
|
|
1426
|
+
public async unloadPackage(packageName: string): Promise<void> {
|
|
1427
|
+
assertSafePackageName(packageName);
|
|
1428
|
+
return this.withPackageLock(packageName, async () => {
|
|
1429
|
+
const _package = this.packages.get(packageName);
|
|
1430
|
+
if (!_package) {
|
|
1431
|
+
return;
|
|
1432
|
+
}
|
|
1433
|
+
if (
|
|
1434
|
+
this.packageStatuses.get(packageName)?.status ===
|
|
1435
|
+
PackageStatus.SERVING
|
|
1436
|
+
) {
|
|
1437
|
+
this.setPackageStatus(packageName, PackageStatus.UNLOADING);
|
|
1438
|
+
}
|
|
1439
|
+
// Same 30s connection drain as deletePackage — just no fs rename/rm.
|
|
1440
|
+
this.retireConnectionGeneration(`package ${packageName}`, () =>
|
|
1441
|
+
_package.getMalloyConfig().shutdown("close"),
|
|
1442
|
+
);
|
|
1443
|
+
this.packages.delete(packageName);
|
|
1444
|
+
this.packageStatuses.delete(packageName);
|
|
1445
|
+
});
|
|
1446
|
+
}
|
|
1447
|
+
|
|
1190
1448
|
public updateConnections(
|
|
1191
1449
|
malloyConfig: EnvironmentMalloyConfig,
|
|
1192
1450
|
_apiConnections?: ApiConnection[],
|