@malloy-publisher/server 0.0.204 → 0.0.206

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/build.ts +10 -1
  2. package/dist/app/api-doc.yaml +494 -397
  3. package/dist/app/assets/{EnvironmentPage-CX06cjOF.js → EnvironmentPage-BYwBeC2F.js} +1 -1
  4. package/dist/app/assets/HomePage-ivu4vdpj.js +1 -0
  5. package/dist/app/assets/{MainPage-nUJ9YatG.js → MainPage-B2DnHEDU.js} +2 -2
  6. package/dist/app/assets/MaterializationsPage-BZEuwF9P.js +1 -0
  7. package/dist/app/assets/ModelPage-Dpu3bfPg.js +1 -0
  8. package/dist/app/assets/{PackagePage-BaEVdEAG.js → PackagePage-B8PwcRHt.js} +1 -1
  9. package/dist/app/assets/{RouteError-BShQjZio.js → RouteError-BhbywAeC.js} +1 -1
  10. package/dist/app/assets/{WorkbookPage-CBn6ZjJW.js → WorkbookPage-C-JXsJG0.js} +1 -1
  11. package/dist/app/assets/{core-DECXYL4E.es-OaRfXwuQ.js → core-pPlPr7jK.es-CNEOlxKB.js} +1 -1
  12. package/dist/app/assets/{index-BLfPC1gy.js → index-BHEm8Egc.js} +1 -1
  13. package/dist/app/assets/{index-Dy3YhAZQ.js → index-BsvDrV14.js} +1 -1
  14. package/dist/app/assets/{index-DqiJ0bWp.js → index-ChR1fKR2.js} +134 -134
  15. package/dist/app/assets/{index.umd-DAN9K8yC.js → index.umd-BVLPYNuj.js} +1 -1
  16. package/dist/app/index.html +1 -1
  17. package/dist/instrumentation.mjs +18 -8
  18. package/dist/package_load_worker.mjs +19 -2
  19. package/dist/runtime/publisher.js +318 -0
  20. package/dist/server.mjs +1703 -1443
  21. package/package.json +5 -4
  22. package/scripts/bake-duckdb-extensions.js +104 -0
  23. package/src/constants.ts +12 -0
  24. package/src/controller/materialization.controller.ts +79 -35
  25. package/src/controller/package.controller.spec.ts +179 -0
  26. package/src/controller/package.controller.ts +60 -73
  27. package/src/controller/watch-mode.controller.ts +176 -46
  28. package/src/dto/package.dto.ts +16 -1
  29. package/src/errors.spec.ts +33 -0
  30. package/src/errors.ts +18 -0
  31. package/src/health.spec.ts +34 -1
  32. package/src/instrumentation.ts +33 -17
  33. package/src/materialization_metrics.ts +66 -0
  34. package/src/mcp/error_messages.spec.ts +35 -0
  35. package/src/mcp/error_messages.ts +14 -1
  36. package/src/mcp/handler_utils.ts +12 -0
  37. package/src/package_load/package_load_pool.ts +7 -1
  38. package/src/package_load/package_load_worker.ts +44 -4
  39. package/src/package_load/protocol.ts +7 -1
  40. package/src/runtime/publisher.js +318 -0
  41. package/src/server-old.ts +7 -149
  42. package/src/server.ts +488 -190
  43. package/src/service/authorize_integration.spec.ts +163 -2
  44. package/src/service/compile_authorize.spec.ts +85 -0
  45. package/src/service/environment.ts +270 -12
  46. package/src/service/environment_store.spec.ts +0 -81
  47. package/src/service/environment_store.ts +142 -34
  48. package/src/service/explore_visibility.spec.ts +434 -0
  49. package/src/service/exports_probe.spec.ts +107 -0
  50. package/src/service/manifest_loader.spec.ts +99 -0
  51. package/src/service/manifest_loader.ts +95 -0
  52. package/src/service/materialization_service.spec.ts +324 -512
  53. package/src/service/materialization_service.ts +816 -656
  54. package/src/service/model.ts +444 -13
  55. package/src/service/package.spec.ts +14 -2
  56. package/src/service/package.ts +271 -20
  57. package/src/service/package_rollback.spec.ts +190 -0
  58. package/src/service/package_worker_path.spec.ts +223 -0
  59. package/src/service/query_boundary.spec.ts +470 -0
  60. package/src/storage/DatabaseInterface.ts +35 -57
  61. package/src/storage/StorageManager.mock.ts +0 -9
  62. package/src/storage/StorageManager.ts +7 -290
  63. package/src/storage/duckdb/DuckDBConnection.ts +70 -124
  64. package/src/storage/duckdb/DuckDBRepository.ts +2 -35
  65. package/src/storage/duckdb/MaterializationRepository.ts +52 -27
  66. package/src/storage/duckdb/schema.ts +4 -20
  67. package/tests/fixtures/authorize-compile/model.malloy +9 -0
  68. package/tests/fixtures/authorize-compile/publisher.json +4 -0
  69. package/tests/fixtures/html-pages-nopublic/model.malloy +1 -0
  70. package/tests/fixtures/html-pages-nopublic/publisher.json +5 -0
  71. package/tests/fixtures/html-pages-test/data.csv +3 -0
  72. package/tests/fixtures/html-pages-test/public/assets/app.css +3 -0
  73. package/tests/fixtures/html-pages-test/public/data.json +1 -0
  74. package/tests/fixtures/html-pages-test/public/index.html +9 -0
  75. package/tests/fixtures/html-pages-test/public/sub/page2.html +9 -0
  76. package/tests/fixtures/html-pages-test/publisher.json +5 -0
  77. package/tests/fixtures/html-pages-test/report.malloy +1 -0
  78. package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +92 -0
  79. package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +138 -0
  80. package/tests/integration/html_pages/html_pages.integration.spec.ts +378 -0
  81. package/tests/integration/materialization/manifest_binding.integration.spec.ts +175 -0
  82. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +277 -266
  83. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +421 -0
  84. package/tests/unit/duckdb/attached_databases.test.ts +111 -0
  85. package/tests/unit/duckdb/duckdb_connection.test.ts +181 -0
  86. package/tests/unit/duckdb/legacy_schema_migration.test.ts +7 -4
  87. package/tests/unit/duckdb/repositories.test.ts +208 -0
  88. package/dist/app/assets/HomePage-CNFt_eUU.js +0 -1
  89. package/dist/app/assets/MaterializationsPage-B5goxVXW.js +0 -1
  90. package/dist/app/assets/ModelPage-Ba7Xh4lL.js +0 -1
  91. package/src/controller/manifest.controller.ts +0 -38
  92. package/src/service/manifest_service.spec.ts +0 -206
  93. package/src/service/manifest_service.ts +0 -117
  94. package/src/service/materialized_table_gc.spec.ts +0 -384
  95. package/src/service/materialized_table_gc.ts +0 -231
  96. package/src/storage/duckdb/DuckDBManifestStore.ts +0 -70
  97. package/src/storage/duckdb/ManifestRepository.ts +0 -120
  98. package/src/storage/duckdb/manifest_store.spec.ts +0 -133
  99. package/src/storage/ducklake/DuckLakeManifestStore.ts +0 -155
  100. package/tests/unit/storage/StorageManager.test.ts +0 -166
@@ -7,8 +7,8 @@ import path from "path";
7
7
  import { AccessDeniedError } from "../errors";
8
8
  import { Model } from "./model";
9
9
 
10
- // Introspection (PR1), compile-time validation (PR2), and the runtime gate
11
- // (PR3) for #(authorize) / ##(authorize).
10
+ // Introspection, compile-time validation, and the runtime gate for
11
+ // #(authorize) / ##(authorize).
12
12
 
13
13
  const TEST_DIR = path.join(os.tmpdir(), "authorize-integration-tests");
14
14
  const TEST_DB_DIR = path.join(TEST_DIR, "db");
@@ -277,6 +277,11 @@ source: gated is duckdb.table('customers')
277
277
  // Names the source and surfaces the underlying Malloy reason.
278
278
  expect(err?.message).toContain("gated");
279
279
  expect(err?.message).toMatch(/NOPE|not declared/i);
280
+ // Redaction policy (pinned): the model-load 424 is author-facing, so it
281
+ // KEEPS the full expression text (needed to fix a malformed annotation).
282
+ // Only the runtime 403 redacts to the source name. If this assertion ever
283
+ // flips, the redaction split was changed — make it a conscious decision.
284
+ expect(err?.message).toContain("$NOPE = 'x'");
280
285
  });
281
286
 
282
287
  it("fails model load when an expression references a source field", async () => {
@@ -836,3 +841,159 @@ source: joiner is duckdb.table('customers') extend {
836
841
  expect(result.data).toBeDefined();
837
842
  });
838
843
  });
844
+
845
+ // The /compile path gates via Model.assertAuthorizedForText (early,
846
+ // surface-syntax) and Model.assertAuthorizedForRunnable (compiled-source
847
+ // backstop). These are the enforcement primitives environment.compileSource
848
+ // calls; exercise them directly here.
849
+ describe("authorize compile-path gate", () => {
850
+ const CP_GATE = `##! experimental.givens
851
+
852
+ given:
853
+ ROLE :: string
854
+
855
+ #(authorize) "$ROLE = 'analyst'"
856
+ source: gated is duckdb.table('customers') extend { measure: c is count() }
857
+
858
+ source: open_src is duckdb.table('customers') extend { measure: c is count() }
859
+ `;
860
+ const CP_FILE_LEVEL = `##! experimental.givens
861
+
862
+ given:
863
+ ROLE :: string
864
+
865
+ ##(authorize) "$ROLE = 'admin'"
866
+
867
+ source: declared is duckdb.table('customers') extend { measure: c is count() }
868
+ `;
869
+
870
+ async function cpModel(file: string, src: string): Promise<Model> {
871
+ await writeModel(file, src);
872
+ return Model.create("test-pkg", TEST_PKG_DIR, file, getConnections());
873
+ }
874
+
875
+ it("assertAuthorizedForText denies/allows a gated named source by its given", async () => {
876
+ const model = await cpModel("cp_gate.malloy", CP_GATE);
877
+ await expect(
878
+ model.assertAuthorizedForText("run: gated -> { aggregate: c }", {}),
879
+ ).rejects.toBeInstanceOf(AccessDeniedError);
880
+ await expect(
881
+ model.assertAuthorizedForText("run: gated -> { aggregate: c }", {
882
+ ROLE: "analyst",
883
+ }),
884
+ ).resolves.toBeUndefined();
885
+ });
886
+
887
+ it("assertAuthorizedForText leaves an ungated source unrestricted", async () => {
888
+ const model = await cpModel("cp_gate.malloy", CP_GATE);
889
+ await expect(
890
+ model.assertAuthorizedForText("run: open_src -> { aggregate: c }", {}),
891
+ ).resolves.toBeUndefined();
892
+ });
893
+
894
+ it("assertAuthorizedForText applies the model-wide file-level gate to inline/unnamed text", async () => {
895
+ const model = await cpModel("cp_file.malloy", CP_FILE_LEVEL);
896
+ // No named source the regex recognizes -> undefined -> file-level gate.
897
+ await expect(
898
+ model.assertAuthorizedForText(
899
+ `run: duckdb.sql("SELECT 1 AS x") -> { aggregate: n is count() }`,
900
+ {},
901
+ ),
902
+ ).rejects.toBeInstanceOf(AccessDeniedError);
903
+ await expect(
904
+ model.assertAuthorizedForText(
905
+ `run: duckdb.sql("SELECT 1 AS x") -> { aggregate: n is count() }`,
906
+ { ROLE: "admin" },
907
+ ),
908
+ ).resolves.toBeUndefined();
909
+ });
910
+
911
+ it("assertAuthorizedForRunnable gates the compiled-source structRef (alias backstop)", async () => {
912
+ const model = await cpModel("cp_gate.malloy", CP_GATE);
913
+ // Stub a runnable whose compiled query reads `gated` (e.g. via an alias
914
+ // the surface-syntax gate would miss).
915
+ const gatedRunnable = {
916
+ getPreparedQuery: async () => ({ _query: { structRef: "gated" } }),
917
+ };
918
+ await expect(
919
+ model.assertAuthorizedForRunnable(gatedRunnable, {}),
920
+ ).rejects.toBeInstanceOf(AccessDeniedError);
921
+ await expect(
922
+ model.assertAuthorizedForRunnable(gatedRunnable, { ROLE: "analyst" }),
923
+ ).resolves.toBeUndefined();
924
+ // Ungated compiled source -> unrestricted.
925
+ const openRunnable = {
926
+ getPreparedQuery: async () => ({ _query: { structRef: "open_src" } }),
927
+ };
928
+ await expect(
929
+ model.assertAuthorizedForRunnable(openRunnable, {}),
930
+ ).resolves.toBeUndefined();
931
+ });
932
+ });
933
+
934
+ // A given can be both a runtime query parameter (substituted into a view's
935
+ // `where`) AND the subject of an `#(authorize)` gate on the same source. These
936
+ // two layers compose: the gate fires first (before any filtering), and once it
937
+ // passes the parameterized filter applies to the result. This is the layered
938
+ // case the end-to-end story rests on.
939
+ describe("authorize composes with given parameters", () => {
940
+ // `ROLE` gates access; `REGION` is a runtime filter parameter on the view.
941
+ const COMPOSED = `##! experimental.givens
942
+
943
+ given:
944
+ ROLE :: string
945
+ REGION :: string
946
+
947
+ #(authorize) "$ROLE = 'analyst'"
948
+ source: regional is duckdb.table('customers') extend {
949
+ measure: c is count()
950
+ view: in_region is {
951
+ where: region = $REGION
952
+ aggregate: c
953
+ }
954
+ }
955
+ `;
956
+
957
+ async function runComposed(givens: Record<string, GivenValue>) {
958
+ await writeModel("compose.malloy", COMPOSED);
959
+ const model = await Model.create(
960
+ "test-pkg",
961
+ TEST_PKG_DIR,
962
+ "compose.malloy",
963
+ getConnections(),
964
+ );
965
+ return model.getQueryResults(
966
+ undefined,
967
+ undefined,
968
+ "run: regional -> in_region",
969
+ undefined,
970
+ undefined,
971
+ givens,
972
+ );
973
+ }
974
+
975
+ function countOf(compactResult: unknown): number {
976
+ const rows = compactResult as Array<Record<string, unknown>>;
977
+ return Number(rows[0]?.c);
978
+ }
979
+
980
+ it("denies before filtering when the gate fails, even with the filter given supplied", async () => {
981
+ await expect(runComposed({ REGION: "us-west" })).rejects.toBeInstanceOf(
982
+ AccessDeniedError,
983
+ );
984
+ });
985
+
986
+ it("applies the parameterized filter once the gate passes", async () => {
987
+ // Seed: one 'us-west' customer, one 'us-east'.
988
+ const west = await runComposed({ ROLE: "analyst", REGION: "us-west" });
989
+ expect(countOf(west.compactResult)).toBe(1);
990
+
991
+ const east = await runComposed({ ROLE: "analyst", REGION: "us-east" });
992
+ expect(countOf(east.compactResult)).toBe(1);
993
+
994
+ // A region matching no rows proves the given genuinely filters (not a
995
+ // gate-only pass-through).
996
+ const none = await runComposed({ ROLE: "analyst", REGION: "nowhere" });
997
+ expect(countOf(none.compactResult)).toBe(0);
998
+ });
999
+ });
@@ -0,0 +1,85 @@
1
+ import { afterEach, beforeEach, describe, expect, it } from "bun:test";
2
+ import fs from "fs/promises";
3
+ import os from "os";
4
+ import path from "path";
5
+ import { AccessDeniedError } from "../errors";
6
+ import { Environment } from "./environment";
7
+
8
+ // End-to-end gate on the /compile path. Exercises environment.compileSource
9
+ // through a real installed package, not just the Model primitives — pins that
10
+ // the early gate AND the compiled-source backstop fire, the latter REGARDLESS of
11
+ // includeSql (a compile-time schema oracle is closed even with no SQL extraction).
12
+
13
+ const PUBLISHER_JSON = JSON.stringify({
14
+ name: "pkg",
15
+ description: "compile-gate",
16
+ });
17
+
18
+ // `gated` is locked to $ROLE='analyst'; `open_src` is unrestricted.
19
+ const MODEL = `##! experimental.givens
20
+
21
+ given:
22
+ ROLE :: string
23
+
24
+ #(authorize) "$ROLE = 'analyst'"
25
+ source: gated is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }
26
+
27
+ source: open_src is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }
28
+ `;
29
+
30
+ describe("compile-path authorize gate (compileSource)", () => {
31
+ let rootDir: string;
32
+ let env: Environment;
33
+
34
+ beforeEach(async () => {
35
+ rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-compile-"));
36
+ const envPath = path.join(rootDir, "env");
37
+ await fs.mkdir(envPath, { recursive: true });
38
+ env = await Environment.create("testEnv", envPath, []);
39
+ await env.installPackage("pkg", async (stagingPath) => {
40
+ await fs.mkdir(stagingPath, { recursive: true });
41
+ await fs.writeFile(
42
+ path.join(stagingPath, "publisher.json"),
43
+ PUBLISHER_JSON,
44
+ );
45
+ await fs.writeFile(path.join(stagingPath, "model.malloy"), MODEL);
46
+ });
47
+ });
48
+
49
+ afterEach(async () => {
50
+ await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
51
+ });
52
+
53
+ const compile = (source: string, givens?: Record<string, string>) =>
54
+ env.compileSource("pkg", "model.malloy", source, false, givens);
55
+
56
+ it("denies a direct gated source without the satisfying given (early gate)", async () => {
57
+ await expect(
58
+ compile("run: gated -> { aggregate: c }"),
59
+ ).rejects.toBeInstanceOf(AccessDeniedError);
60
+ });
61
+
62
+ it("denies a gated source reached via the LAST run: statement (backstop, includeSql=false)", async () => {
63
+ // Regression guard: the early gate only matches the first `run:` (ungated
64
+ // open_src here), so the gated source in the executed final statement is
65
+ // caught only by the compiled-source backstop — which must run even when
66
+ // no SQL is requested.
67
+ await expect(
68
+ compile(
69
+ "run: open_src -> { aggregate: c }\nrun: gated -> { aggregate: c }",
70
+ ),
71
+ ).rejects.toBeInstanceOf(AccessDeniedError);
72
+ });
73
+
74
+ it("allows the gated source when the given satisfies the gate", async () => {
75
+ const { problems } = await compile("run: gated -> { aggregate: c }", {
76
+ ROLE: "analyst",
77
+ });
78
+ expect(problems).toBeDefined();
79
+ });
80
+
81
+ it("leaves an ungated source compilable without any given", async () => {
82
+ const { problems } = await compile("run: open_src -> { aggregate: c }");
83
+ expect(problems).toBeDefined();
84
+ });
85
+ });
@@ -5,9 +5,16 @@ import { Mutex } from "async-mutex";
5
5
  import crypto from "crypto";
6
6
  import * as fs from "fs";
7
7
  import * as path from "path";
8
+ import { pathToFileURL } from "url";
8
9
  import { components } from "../api";
9
- import { API_PREFIX, README_NAME } from "../constants";
10
10
  import {
11
+ API_PREFIX,
12
+ normalizeModelPath,
13
+ NOTEBOOK_FILE_SUFFIX,
14
+ README_NAME,
15
+ } from "../constants";
16
+ import {
17
+ BadRequestError,
11
18
  ConnectionNotFoundError,
12
19
  EnvironmentNotFoundError,
13
20
  PackageNotFoundError,
@@ -28,6 +35,7 @@ import {
28
35
  EnvironmentMalloyConfig,
29
36
  InternalConnection,
30
37
  } from "./connection";
38
+ import { fetchManifestEntries } from "./manifest_loader";
31
39
  import { ApiConnection } from "./model";
32
40
  import { Package } from "./package";
33
41
  import type { PackageMemoryGovernor } from "./package_memory_governor";
@@ -137,6 +145,11 @@ export class Environment {
137
145
  // governor as the single owner of the back-pressure boolean.
138
146
  private memoryGovernor: PackageMemoryGovernor | null = null;
139
147
 
148
+ /** Absolute path on disk where this environment's package files live. */
149
+ public getEnvironmentPath(): string {
150
+ return this.environmentPath;
151
+ }
152
+
140
153
  constructor(
141
154
  environmentName: string,
142
155
  environmentPath: string,
@@ -181,10 +194,6 @@ export class Environment {
181
194
  await this.writeEnvironmentReadme(payload.readme);
182
195
  }
183
196
 
184
- if (payload.materializationStorage !== undefined) {
185
- this.metadata.materializationStorage = payload.materializationStorage;
186
- }
187
-
188
197
  // Handle connections update
189
198
  // TODO: Update environment connections should have its own API endpoint
190
199
  if (payload.connections) {
@@ -290,6 +299,17 @@ export class Environment {
290
299
  ): Promise<{ problems: LogMessage[]; sql?: string }> {
291
300
  assertSafePackageName(packageName);
292
301
  assertSafeRelativeModelPath(modelName);
302
+ // /compile appends the submitted source to the TARGET MODEL's content for
303
+ // namespace context. A notebook (.malloynb) is markdown + cells, not a
304
+ // model, so compiling against it only yields a confusing parse error —
305
+ // reject it up front with an actionable message. (Notebooks remain public
306
+ // for discovery/query; this is specific to the compile context.)
307
+ if (modelName.endsWith(NOTEBOOK_FILE_SUFFIX)) {
308
+ throw new BadRequestError(
309
+ `Cannot compile against a notebook ("${modelName}"). ` +
310
+ `/compile takes a .malloy model path for namespace context.`,
311
+ );
312
+ }
293
313
  // Hold the per-package mutex for the duration of every disk read —
294
314
  // both the explicit `fs.readFile(modelPath)` below and the implicit
295
315
  // import resolution that `runtime.loadModel` does through the URL
@@ -307,10 +327,18 @@ export class Environment {
307
327
  packageName,
308
328
  modelName,
309
329
  );
310
- // Place the virtual file in the model's directory so relative imports resolve correctly.
330
+ // Place the virtual file in the model's directory so relative imports
331
+ // resolve correctly. Use `pathToFileURL` rather than hand-prefixing
332
+ // `file://`: on Windows the latter produces a malformed URL
333
+ // (`file://D:\Temp\…`) that round-trips differently than the URL the
334
+ // Malloy runtime synthesizes from the same path, breaking the
335
+ // intercepting reader's string comparison below and falling through
336
+ // to disk for a virtual file that doesn't exist.
311
337
  const modelDir = path.dirname(modelPath);
312
- const virtualUri = `file://${path.join(modelDir, "__compile_check.malloy")}`;
313
- const virtualUrl = new URL(virtualUri);
338
+ const virtualUrl = pathToFileURL(
339
+ path.join(modelDir, "__compile_check.malloy"),
340
+ );
341
+ const virtualUri = virtualUrl.toString();
314
342
 
315
343
  // Read the full model file so the submitted source inherits the model's
316
344
  // complete namespace — imports, source definitions, queries, etc.
@@ -339,6 +367,29 @@ export class Environment {
339
367
  // Use the locked variant — we already hold the per-package mutex.
340
368
  const pkg = await this._loadOrGetPackageLocked(packageName);
341
369
 
370
+ // Authorize gate: /compile is compile-only, but it can still act
371
+ // as a schema oracle (a denied caller learns a gated source's columns
372
+ // from compile errors) and, with includeSql, leak its SQL. Gate the
373
+ // named source the submitted text targets BEFORE compiling — mirrors
374
+ // the query path's early surface-syntax gate. Unnamed/inline source
375
+ // text resolves to undefined, so only the model-wide file-level gate
376
+ // applies. The gate runs against the package's cached Model (its
377
+ // `given:` block + authorize annotations), independent of the virtual
378
+ // compile below. If the model isn't loaded, there's nothing to enforce
379
+ // and compilation surfaces its own error.
380
+ const gateModel = pkg.getModel(modelName);
381
+ if (gateModel) {
382
+ // Query boundary first (the *what* axis): /compile compiles ad-hoc
383
+ // text against a model, so gate it like an ad-hoc query — a
384
+ // non-`explores` model file, or text whose surface-resolved target
385
+ // is a non-curated model source (under queryableSources:
386
+ // "declared"), is rejected with a generic 404 before compilation
387
+ // can leak schema/SQL. Text the early gate can't pin is settled by
388
+ // the compiled backstop below.
389
+ gateModel.assertQueryBoundaryEarly(undefined, undefined, source);
390
+ await gateModel.assertAuthorizedForText(source, givens ?? {});
391
+ }
392
+
342
393
  // Initialize Runtime with the package's active MalloyConfig so compile
343
394
  // checks see the same package-scoped duckdb as execution. This runtime
344
395
  // borrows the package config; the package/environment lifecycle owns release.
@@ -352,11 +403,57 @@ export class Environment {
352
403
  const modelMaterializer = runtime.loadModel(virtualUrl);
353
404
  const model = await modelMaterializer.getModel();
354
405
 
406
+ // Resolve the final query's materializer once (if there is one).
407
+ let queryMaterializer: ReturnType<
408
+ typeof modelMaterializer.loadFinalQuery
409
+ > | null = null;
410
+ try {
411
+ queryMaterializer = modelMaterializer.loadFinalQuery();
412
+ } catch {
413
+ // No runnable query (e.g. only source definitions) — nothing to
414
+ // gate or extract beyond the early text gate already applied.
415
+ }
416
+
417
+ // Compiled-source backstops — run REGARDLESS of includeSql. They
418
+ // gate the source the COMPILED final query actually reads, closing
419
+ // named-query / multi-statement indirection the early surface-syntax
420
+ // gate misses (e.g. `run: ungated\nrun: gated` — the early gate only
421
+ // matches the FIRST `run:`, but the LAST statement is what executes).
422
+ // Compiling a gated source even without SQL is a schema oracle
423
+ // (field-not-found errors leak its columns), so this must not be
424
+ // conditional on SQL extraction. (A `source: x is gated` alias makes
425
+ // a new ungated source — that's the documented "extend doesn't
426
+ // inherit authorize" footgun, the same as the query path.)
427
+
428
+ // Boundary backstop (the *what* axis, 404) before the authorize
429
+ // one (the *who* axis, 403). /compile text is always ad-hoc — the
430
+ // early gate can only positively deny, never fully clear — so the
431
+ // compiled final query's run target is the authority. Self-gates
432
+ // internally (no-ops when the boundary is inert: "all" / no
433
+ // explores), so it is deliberately NOT guarded by hasAuthorize().
434
+ // Text that compiles only source definitions (no final query) has
435
+ // no run target and nothing to gate.
436
+ if (queryMaterializer && gateModel) {
437
+ await gateModel.assertQueryBoundaryForRunnable(
438
+ queryMaterializer,
439
+ source,
440
+ );
441
+ }
442
+
443
+ // Authorize backstop (the *who* axis, 403). Only run when the model
444
+ // declares gates so ungated compiles don't pay for the extra
445
+ // final-query compile.
446
+ if (queryMaterializer && gateModel?.hasAuthorize()) {
447
+ await gateModel.assertAuthorizedForRunnable(
448
+ queryMaterializer,
449
+ givens ?? {},
450
+ );
451
+ }
452
+
355
453
  // If includeSql is requested and compilation succeeded, attempt to extract SQL
356
454
  let sql: string | undefined;
357
- if (includeSql) {
455
+ if (includeSql && queryMaterializer) {
358
456
  try {
359
- const queryMaterializer = modelMaterializer.loadFinalQuery();
360
457
  sql = await queryMaterializer.getSQL({ givens });
361
458
  } catch {
362
459
  // Source may not contain a runnable query (e.g. only source definitions),
@@ -734,6 +831,7 @@ export class Environment {
734
831
  packagePath,
735
832
  () => this.malloyConfig.malloyConfig,
736
833
  );
834
+ await this.bindManifestIfConfigured(_package);
737
835
  if (existingPackage !== undefined && reload) {
738
836
  this.retireConnectionGeneration(`package ${packageName}`, () =>
739
837
  existingPackage.getMalloyConfig().shutdown("close"),
@@ -839,6 +937,7 @@ export class Environment {
839
937
  public async installPackage(
840
938
  packageName: string,
841
939
  downloader: (stagingPath: string) => Promise<void>,
940
+ validate?: (pkg: Package) => string | undefined,
842
941
  ): Promise<Package> {
843
942
  assertSafePackageName(packageName);
844
943
  const stagingPath = this.allocateStagingPath(packageName);
@@ -905,6 +1004,15 @@ export class Environment {
905
1004
  canonicalPath,
906
1005
  () => this.malloyConfig.malloyConfig,
907
1006
  );
1007
+ // Strict-reject hook (publish/update only — reload passes no
1008
+ // validator and stays fail-safe). Throw INSIDE the try so the
1009
+ // catch below rolls the swap back: the just-installed tree is
1010
+ // wiped and the retired tree (if any) is restored, so a rejected
1011
+ // publish/update never leaves the bad tree served on disk.
1012
+ const validationMsg = validate?.(newPackage);
1013
+ if (validationMsg) {
1014
+ throw new BadRequestError(validationMsg);
1015
+ }
908
1016
  logger.debug("install.phase2.committed", {
909
1017
  environmentName: this.environmentName,
910
1018
  packageName,
@@ -950,6 +1058,11 @@ export class Environment {
950
1058
  throw err;
951
1059
  }
952
1060
 
1061
+ // Best-effort manifest bind happens after the swap commits, outside the
1062
+ // rollback window: a manifest that can't be fetched must not undo an
1063
+ // otherwise-successful install (the package serves live instead).
1064
+ await this.bindManifestIfConfigured(newPackage);
1065
+
953
1066
  this.packages.set(packageName, newPackage);
954
1067
  this.setPackageStatus(packageName, PackageStatus.SERVING);
955
1068
 
@@ -1004,6 +1117,47 @@ export class Environment {
1004
1117
  });
1005
1118
  }
1006
1119
 
1120
+ /**
1121
+ * If the freshly-loaded package declares a `manifestLocation`, fetch the
1122
+ * control-plane-computed build manifest and rebind its models so persist
1123
+ * references resolve to the materialized tables. Best-effort: a fetch/bind
1124
+ * failure logs a warning and leaves the package serving live (the models are
1125
+ * already loaded without a manifest). Callers must hold the package lock —
1126
+ * this rebinds `pkg` in place rather than re-entering {@link withPackageLock}.
1127
+ */
1128
+ private async bindManifestIfConfigured(pkg: Package): Promise<void> {
1129
+ const manifestLocation = pkg.getPackageMetadata().manifestLocation;
1130
+ if (!manifestLocation) {
1131
+ return;
1132
+ }
1133
+ await this.bindManifest(pkg, manifestLocation);
1134
+ }
1135
+
1136
+ /** Fetch + bind a specific manifest URI onto an already-loaded package. */
1137
+ private async bindManifest(
1138
+ pkg: Package,
1139
+ manifestLocation: string,
1140
+ ): Promise<void> {
1141
+ const packageName = pkg.getPackageName();
1142
+ try {
1143
+ const entries = await fetchManifestEntries(manifestLocation);
1144
+ await pkg.reloadAllModels(entries);
1145
+ logger.info("Bound build manifest to package", {
1146
+ environmentName: this.environmentName,
1147
+ packageName,
1148
+ manifestLocation,
1149
+ entryCount: Object.keys(entries).length,
1150
+ });
1151
+ } catch (err) {
1152
+ logger.warn("Failed to bind build manifest; serving live", {
1153
+ environmentName: this.environmentName,
1154
+ packageName,
1155
+ manifestLocation,
1156
+ error: err instanceof Error ? err.message : String(err),
1157
+ });
1158
+ }
1159
+ }
1160
+
1007
1161
  /**
1008
1162
  * Read a model's source text from disk, holding the per-package mutex
1009
1163
  * so the read is serialized against {@link installPackage} /
@@ -1028,7 +1182,13 @@ export class Environment {
1028
1182
 
1029
1183
  private async writePackageManifest(
1030
1184
  packageName: string,
1031
- metadata: { name: string; description?: string },
1185
+ metadata: {
1186
+ name: string;
1187
+ description?: string;
1188
+ explores?: string[];
1189
+ queryableSources?: "declared" | "all";
1190
+ manifestLocation?: string | null;
1191
+ },
1032
1192
  ): Promise<void> {
1033
1193
  const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
1034
1194
  const manifestPath = safeJoinUnderRoot(packagePath, "publisher.json");
@@ -1043,11 +1203,23 @@ export class Environment {
1043
1203
  logger.warn(`Could not read manifest for ${packageName}`);
1044
1204
  }
1045
1205
 
1046
- // Update with new metadata
1206
+ // Update with new metadata. `explores`/`queryableSources` are only
1207
+ // overwritten when the caller explicitly provides them; otherwise the
1208
+ // existing on-disk value is preserved via the spread (an undefined here
1209
+ // must not erase it).
1047
1210
  const updatedManifest = {
1048
1211
  ...existingManifest,
1049
1212
  name: metadata.name,
1050
1213
  description: metadata.description,
1214
+ ...(metadata.explores !== undefined
1215
+ ? { explores: metadata.explores }
1216
+ : {}),
1217
+ ...(metadata.queryableSources !== undefined
1218
+ ? { queryableSources: metadata.queryableSources }
1219
+ : {}),
1220
+ ...(metadata.manifestLocation !== undefined
1221
+ ? { manifestLocation: metadata.manifestLocation }
1222
+ : {}),
1051
1223
  };
1052
1224
 
1053
1225
  // Write back to file
@@ -1074,18 +1246,72 @@ export class Environment {
1074
1246
  if (body.name) {
1075
1247
  _package.setName(body.name);
1076
1248
  }
1249
+ // Preserve `explores` across a metadata PATCH. `setPackageMetadata`
1250
+ // replaces the whole object, so a name/description-only update must
1251
+ // carry the existing discovery surface through — otherwise the
1252
+ // in-memory `explores` is wiped and `listModels()` silently starts
1253
+ // serving every model until the next reload. When the body explicitly
1254
+ // carries `explores`, honor the new set instead.
1255
+ const existing = _package.getPackageMetadata();
1256
+ // Normalize API-body explores through the same helper the worker uses
1257
+ // for on-disk explores, so `["./index.malloy"]` / backslash paths
1258
+ // validate and persist identically regardless of input channel (no
1259
+ // misleading publish-time 400, no publish-vs-reload divergence).
1260
+ const normalizedExplores = body.explores?.map(normalizeModelPath);
1261
+ const explores =
1262
+ normalizedExplores !== undefined
1263
+ ? normalizedExplores
1264
+ : existing.explores;
1265
+ const queryableSources =
1266
+ body.queryableSources !== undefined
1267
+ ? body.queryableSources
1268
+ : existing.queryableSources;
1269
+ // Preserve the existing manifestLocation unless the body explicitly
1270
+ // sets it (including to null, which clears it and reverts to live).
1271
+ const manifestLocation =
1272
+ body.manifestLocation !== undefined
1273
+ ? body.manifestLocation
1274
+ : existing.manifestLocation;
1077
1275
  _package.setPackageMetadata({
1078
1276
  name: body.name,
1079
1277
  description: body.description,
1080
1278
  resource: body.resource,
1081
1279
  location: body.location,
1280
+ explores,
1281
+ queryableSources,
1282
+ manifestLocation,
1082
1283
  });
1083
1284
 
1285
+ // Strict-reject, symmetric with the publish path
1286
+ // (package.controller.addPackage): validate the resulting explores
1287
+ // against the live model set and restore the prior metadata before
1288
+ // rejecting, so a bad update neither persists nor mutates the served
1289
+ // surface.
1290
+ const invalidMsg = _package.formatInvalidExplores();
1291
+ if (invalidMsg) {
1292
+ _package.setPackageMetadata(existing);
1293
+ throw new BadRequestError(invalidMsg);
1294
+ }
1295
+
1084
1296
  await this.writePackageManifest(packageName, {
1085
1297
  name: packageName,
1086
1298
  description: body.description,
1299
+ explores: normalizedExplores,
1300
+ queryableSources: body.queryableSources,
1301
+ manifestLocation: body.manifestLocation,
1087
1302
  });
1088
1303
 
1304
+ // When the body changes manifestLocation, apply it now so the new
1305
+ // binding takes effect without a separate reload: a URI rebinds models
1306
+ // to the materialized tables; null/empty reverts the package to live.
1307
+ if (body.manifestLocation !== undefined) {
1308
+ if (body.manifestLocation) {
1309
+ await this.bindManifest(_package, body.manifestLocation);
1310
+ } else {
1311
+ await _package.reloadAllModels({});
1312
+ }
1313
+ }
1314
+
1089
1315
  return _package.getPackageMetadata();
1090
1316
  });
1091
1317
  }
@@ -1187,6 +1413,38 @@ export class Environment {
1187
1413
  });
1188
1414
  }
1189
1415
 
1416
+ /**
1417
+ * Evict a package from the in-memory caches WITHOUT touching its on-disk
1418
+ * directory — the non-destructive counterpart to {@link deletePackage}.
1419
+ *
1420
+ * Used to roll back a no-location `addPackage` (which registers a
1421
+ * *pre-existing*, user-owned directory) when post-load validation rejects
1422
+ * it: deleting the tree there would destroy content the publisher never
1423
+ * created. This still drains and closes the connections the just-created
1424
+ * `Package` opened, so the duckdb handle isn't leaked.
1425
+ */
1426
+ public async unloadPackage(packageName: string): Promise<void> {
1427
+ assertSafePackageName(packageName);
1428
+ return this.withPackageLock(packageName, async () => {
1429
+ const _package = this.packages.get(packageName);
1430
+ if (!_package) {
1431
+ return;
1432
+ }
1433
+ if (
1434
+ this.packageStatuses.get(packageName)?.status ===
1435
+ PackageStatus.SERVING
1436
+ ) {
1437
+ this.setPackageStatus(packageName, PackageStatus.UNLOADING);
1438
+ }
1439
+ // Same 30s connection drain as deletePackage — just no fs rename/rm.
1440
+ this.retireConnectionGeneration(`package ${packageName}`, () =>
1441
+ _package.getMalloyConfig().shutdown("close"),
1442
+ );
1443
+ this.packages.delete(packageName);
1444
+ this.packageStatuses.delete(packageName);
1445
+ });
1446
+ }
1447
+
1190
1448
  public updateConnections(
1191
1449
  malloyConfig: EnvironmentMalloyConfig,
1192
1450
  _apiConnections?: ApiConnection[],