@malloy-publisher/server 0.0.204 → 0.0.206

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/build.ts +10 -1
  2. package/dist/app/api-doc.yaml +494 -397
  3. package/dist/app/assets/{EnvironmentPage-CX06cjOF.js → EnvironmentPage-BYwBeC2F.js} +1 -1
  4. package/dist/app/assets/HomePage-ivu4vdpj.js +1 -0
  5. package/dist/app/assets/{MainPage-nUJ9YatG.js → MainPage-B2DnHEDU.js} +2 -2
  6. package/dist/app/assets/MaterializationsPage-BZEuwF9P.js +1 -0
  7. package/dist/app/assets/ModelPage-Dpu3bfPg.js +1 -0
  8. package/dist/app/assets/{PackagePage-BaEVdEAG.js → PackagePage-B8PwcRHt.js} +1 -1
  9. package/dist/app/assets/{RouteError-BShQjZio.js → RouteError-BhbywAeC.js} +1 -1
  10. package/dist/app/assets/{WorkbookPage-CBn6ZjJW.js → WorkbookPage-C-JXsJG0.js} +1 -1
  11. package/dist/app/assets/{core-DECXYL4E.es-OaRfXwuQ.js → core-pPlPr7jK.es-CNEOlxKB.js} +1 -1
  12. package/dist/app/assets/{index-BLfPC1gy.js → index-BHEm8Egc.js} +1 -1
  13. package/dist/app/assets/{index-Dy3YhAZQ.js → index-BsvDrV14.js} +1 -1
  14. package/dist/app/assets/{index-DqiJ0bWp.js → index-ChR1fKR2.js} +134 -134
  15. package/dist/app/assets/{index.umd-DAN9K8yC.js → index.umd-BVLPYNuj.js} +1 -1
  16. package/dist/app/index.html +1 -1
  17. package/dist/instrumentation.mjs +18 -8
  18. package/dist/package_load_worker.mjs +19 -2
  19. package/dist/runtime/publisher.js +318 -0
  20. package/dist/server.mjs +1703 -1443
  21. package/package.json +5 -4
  22. package/scripts/bake-duckdb-extensions.js +104 -0
  23. package/src/constants.ts +12 -0
  24. package/src/controller/materialization.controller.ts +79 -35
  25. package/src/controller/package.controller.spec.ts +179 -0
  26. package/src/controller/package.controller.ts +60 -73
  27. package/src/controller/watch-mode.controller.ts +176 -46
  28. package/src/dto/package.dto.ts +16 -1
  29. package/src/errors.spec.ts +33 -0
  30. package/src/errors.ts +18 -0
  31. package/src/health.spec.ts +34 -1
  32. package/src/instrumentation.ts +33 -17
  33. package/src/materialization_metrics.ts +66 -0
  34. package/src/mcp/error_messages.spec.ts +35 -0
  35. package/src/mcp/error_messages.ts +14 -1
  36. package/src/mcp/handler_utils.ts +12 -0
  37. package/src/package_load/package_load_pool.ts +7 -1
  38. package/src/package_load/package_load_worker.ts +44 -4
  39. package/src/package_load/protocol.ts +7 -1
  40. package/src/runtime/publisher.js +318 -0
  41. package/src/server-old.ts +7 -149
  42. package/src/server.ts +488 -190
  43. package/src/service/authorize_integration.spec.ts +163 -2
  44. package/src/service/compile_authorize.spec.ts +85 -0
  45. package/src/service/environment.ts +270 -12
  46. package/src/service/environment_store.spec.ts +0 -81
  47. package/src/service/environment_store.ts +142 -34
  48. package/src/service/explore_visibility.spec.ts +434 -0
  49. package/src/service/exports_probe.spec.ts +107 -0
  50. package/src/service/manifest_loader.spec.ts +99 -0
  51. package/src/service/manifest_loader.ts +95 -0
  52. package/src/service/materialization_service.spec.ts +324 -512
  53. package/src/service/materialization_service.ts +816 -656
  54. package/src/service/model.ts +444 -13
  55. package/src/service/package.spec.ts +14 -2
  56. package/src/service/package.ts +271 -20
  57. package/src/service/package_rollback.spec.ts +190 -0
  58. package/src/service/package_worker_path.spec.ts +223 -0
  59. package/src/service/query_boundary.spec.ts +470 -0
  60. package/src/storage/DatabaseInterface.ts +35 -57
  61. package/src/storage/StorageManager.mock.ts +0 -9
  62. package/src/storage/StorageManager.ts +7 -290
  63. package/src/storage/duckdb/DuckDBConnection.ts +70 -124
  64. package/src/storage/duckdb/DuckDBRepository.ts +2 -35
  65. package/src/storage/duckdb/MaterializationRepository.ts +52 -27
  66. package/src/storage/duckdb/schema.ts +4 -20
  67. package/tests/fixtures/authorize-compile/model.malloy +9 -0
  68. package/tests/fixtures/authorize-compile/publisher.json +4 -0
  69. package/tests/fixtures/html-pages-nopublic/model.malloy +1 -0
  70. package/tests/fixtures/html-pages-nopublic/publisher.json +5 -0
  71. package/tests/fixtures/html-pages-test/data.csv +3 -0
  72. package/tests/fixtures/html-pages-test/public/assets/app.css +3 -0
  73. package/tests/fixtures/html-pages-test/public/data.json +1 -0
  74. package/tests/fixtures/html-pages-test/public/index.html +9 -0
  75. package/tests/fixtures/html-pages-test/public/sub/page2.html +9 -0
  76. package/tests/fixtures/html-pages-test/publisher.json +5 -0
  77. package/tests/fixtures/html-pages-test/report.malloy +1 -0
  78. package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +92 -0
  79. package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +138 -0
  80. package/tests/integration/html_pages/html_pages.integration.spec.ts +378 -0
  81. package/tests/integration/materialization/manifest_binding.integration.spec.ts +175 -0
  82. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +277 -266
  83. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +421 -0
  84. package/tests/unit/duckdb/attached_databases.test.ts +111 -0
  85. package/tests/unit/duckdb/duckdb_connection.test.ts +181 -0
  86. package/tests/unit/duckdb/legacy_schema_migration.test.ts +7 -4
  87. package/tests/unit/duckdb/repositories.test.ts +208 -0
  88. package/dist/app/assets/HomePage-CNFt_eUU.js +0 -1
  89. package/dist/app/assets/MaterializationsPage-B5goxVXW.js +0 -1
  90. package/dist/app/assets/ModelPage-Ba7Xh4lL.js +0 -1
  91. package/src/controller/manifest.controller.ts +0 -38
  92. package/src/service/manifest_service.spec.ts +0 -206
  93. package/src/service/manifest_service.ts +0 -117
  94. package/src/service/materialized_table_gc.spec.ts +0 -384
  95. package/src/service/materialized_table_gc.ts +0 -231
  96. package/src/storage/duckdb/DuckDBManifestStore.ts +0 -70
  97. package/src/storage/duckdb/ManifestRepository.ts +0 -120
  98. package/src/storage/duckdb/manifest_store.spec.ts +0 -133
  99. package/src/storage/ducklake/DuckLakeManifestStore.ts +0 -155
  100. package/tests/unit/storage/StorageManager.test.ts +0 -166
@@ -42,6 +42,7 @@ import {
42
42
  BadRequestError,
43
43
  ModelCompilationError,
44
44
  ModelNotFoundError,
45
+ NotQueryableError,
45
46
  PayloadTooLargeError,
46
47
  } from "../errors";
47
48
  import { logger } from "../logger";
@@ -101,6 +102,18 @@ interface RunnableNotebookCell {
101
102
  queryInfo?: Malloy.QueryInfo;
102
103
  }
103
104
 
105
+ /**
106
+ * Backtick-quote a Malloy identifier for safe interpolation into a `run:`
107
+ * query string. Escapes backslashes and backticks (in that order) so a name
108
+ * that needs Malloy quoting (hyphen, space, reserved word, leading digit) or
109
+ * contains an embedded backtick cannot break out of the quotes. Mirrors
110
+ * malloy's internal `identifierCode` / `escapeIdentifier` (to_stable.ts), which
111
+ * is not exported.
112
+ */
113
+ function quoteMalloyIdentifier(name: string | undefined): string {
114
+ return "`" + (name ?? "").replace(/\\/g, "\\\\").replace(/`/g, "\\`") + "`";
115
+ }
116
+
104
117
  export class Model {
105
118
  private packageName: string;
106
119
  private modelPath: string;
@@ -123,6 +136,21 @@ export class Model {
123
136
  /** Model-wide `##(authorize)` expressions; apply to every query in the
124
137
  * model, including ad-hoc inline sources not declared in the model. */
125
138
  private fileLevelAuthorize: string[] = [];
139
+ /** Whether discovery accessors curate to the `export {}` closure. Pushed
140
+ * down by the owning Package (see Package.applyDiscoveryPolicyToModels):
141
+ * true only when the package declares `explores` in publisher.json.
142
+ * Defaults to false (legacy listings) so a Model created outside a
143
+ * Package matches pre-opt-in behavior. */
144
+ private discoveryCurationEnabled = false;
145
+ /** Per-package query-boundary policy, pushed down by the owning Package
146
+ * (see `Package.applyQueryBoundaryToModels`). Defaults are inert (mode
147
+ * "all" / not declared) so a Model created outside a Package — or before
148
+ * the policy is applied — never spuriously denies. */
149
+ private queryBoundary: {
150
+ mode: "declared" | "all";
151
+ exploresDeclared: boolean;
152
+ isQueryEntryPoint: boolean;
153
+ } = { mode: "all", exploresDeclared: false, isQueryEntryPoint: true };
126
154
  private meter = metrics.getMeter("publisher");
127
155
  private queryExecutionHistogram = this.meter.createHistogram(
128
156
  "malloy_model_query_duration",
@@ -232,6 +260,18 @@ export class Model {
232
260
  );
233
261
  }
234
262
 
263
+ /**
264
+ * Whether the model declares any `#(authorize)` / `##(authorize)` gate at all
265
+ * (file-level or on any source). Lets callers cheaply skip authorize work for
266
+ * ungated models without compiling a probe.
267
+ */
268
+ public hasAuthorize(): boolean {
269
+ return (
270
+ this.fileLevelAuthorize.length > 0 ||
271
+ (this.sources?.some((s) => (s.authorize?.length ?? 0) > 0) ?? false)
272
+ );
273
+ }
274
+
235
275
  /**
236
276
  * Effective authorize expressions for whatever a query runs against:
237
277
  * - a declared model source → its own list (file-level ++ source-level);
@@ -288,6 +328,38 @@ export class Model {
288
328
  if (!passed) deny();
289
329
  }
290
330
 
331
+ /**
332
+ * Gate ad-hoc compile/query text by the named source it targets. Resolves the
333
+ * source from surface syntax (`extractSourceName`) and applies the gate. An
334
+ * unnamed/inline source resolves to `undefined`, so only the model-wide
335
+ * file-level gate applies — the same top-level-only boundary as the query
336
+ * path's early gate. Used by the `/compile` path, which has no runnable to
337
+ * resolve before it decides whether to compile at all.
338
+ */
339
+ public async assertAuthorizedForText(
340
+ text: string,
341
+ givens: Record<string, GivenValue>,
342
+ ): Promise<void> {
343
+ await this.assertAuthorized(this.extractSourceName(text), givens);
344
+ }
345
+
346
+ /**
347
+ * Gate a compiled query by the source it actually reads, resolved from the
348
+ * prepared query's `structRef` (authoritative — survives named-query and
349
+ * multi-statement indirection that surface syntax misses, e.g. the executed
350
+ * `run:` statement isn't the first one). Used as the `/compile` backstop once
351
+ * a runnable exists.
352
+ */
353
+ public async assertAuthorizedForRunnable(
354
+ runnable: { getPreparedQuery(): Promise<unknown> },
355
+ givens: Record<string, GivenValue>,
356
+ ): Promise<void> {
357
+ await this.assertAuthorized(
358
+ await this.resolveAuthorizeSourceFromRunnable(runnable),
359
+ givens,
360
+ );
361
+ }
362
+
291
363
  /**
292
364
  * Resolve the source a compiled query reads, from its prepared query's
293
365
  * `structRef`. This is authoritative — it survives named-query indirection
@@ -345,14 +417,7 @@ export class Model {
345
417
  const target = this.extractSourceName(query);
346
418
  if (!target || !query) return undefined;
347
419
 
348
- // Map each ad-hoc source name to the base it derives from
349
- // (`source: NAME is BASE ...` → NAME → BASE).
350
- const aliasOf = new Map<string, string>();
351
- const declRe = /source\s*:\s*(\w+)\s+is\s+(\w+)/g;
352
- let match: RegExpExecArray | null;
353
- while ((match = declRe.exec(query)) !== null) {
354
- aliasOf.set(match[1], match[2]);
355
- }
420
+ const aliasOf = Model.buildAliasMap(query);
356
421
 
357
422
  // Walk the derivation chain until we hit a protected source or run out.
358
423
  let current: string | undefined = target;
@@ -644,16 +709,353 @@ export class Model {
644
709
  return this.modelType;
645
710
  }
646
711
 
712
+ /**
713
+ * Restrict a list of named objects to the model's re-export closure
714
+ * (`modelDef.exports`) for discovery. This mirrors what Malloy's stable
715
+ * `modelDefToModelInfo` already does for `modelInfo`/`sourceInfos` (and what
716
+ * the app renders), so the publisher-extracted `sources`/`queries` stay
717
+ * consistent with `modelInfo` within a single response. A model with no
718
+ * `export { … }` has `exports` = all top-level names, so this is a no-op
719
+ * there. Only active when the package declares `explores` (see
720
+ * `discoveryCurationEnabled`). `this.sources`/`this.queries` stay complete so
721
+ * #(authorize)/filter enforcement and join/extend resolution are unaffected.
722
+ * Whether a non-exported source is also non-*queryable* depends on the
723
+ * package's `queryableSources` policy — see {@link assertQueryBoundaryEarly}.
724
+ */
725
+ private curateForDiscovery<T extends { name?: string }>(
726
+ items: T[] | undefined,
727
+ ): T[] | undefined {
728
+ if (!items) return items;
729
+ if (!this.discoveryCurationEnabled) return items;
730
+ const exports = this.modelDef?.exports;
731
+ if (!Array.isArray(exports)) return items;
732
+ const exported = new Set(exports);
733
+ return items.filter(
734
+ (item) => item.name !== undefined && exported.has(item.name),
735
+ );
736
+ }
737
+
738
+ /** Set by the owning Package; see {@link curateForDiscovery}. */
739
+ public setDiscoveryCuration(enabled: boolean): void {
740
+ this.discoveryCurationEnabled = enabled;
741
+ }
742
+
647
743
  public getSources(): ApiSource[] | undefined {
648
- return this.sources;
744
+ return this.curateForDiscovery(this.sources);
649
745
  }
650
746
 
651
747
  public getSourceInfos(): Malloy.SourceInfo[] | undefined {
652
- return this.sourceInfos;
748
+ return this.curateForDiscovery(this.sourceInfos);
653
749
  }
654
750
 
655
751
  public getQueries(): ApiQuery[] | undefined {
656
- return this.queries;
752
+ return this.curateForDiscovery(this.queries);
753
+ }
754
+
755
+ /**
756
+ * True when this is an import-only model: it imports other files but
757
+ * declares and re-exports nothing of its own, so `modelDef.exports` is
758
+ * empty and its discovery surface lists no sources or queries. Legitimate
759
+ * as plumbing, but confusing when the model is *listed* — the page renders
760
+ * blank. Used by the load-time warning (Package.emptyDiscoveryWarnings);
761
+ * the fix is to re-export what should be visible (`export { name }`).
762
+ */
763
+ public hasEmptyDiscoverySurface(): boolean {
764
+ // No curation (no `explores`) ⇒ legacy listings include imported sources.
765
+ if (!this.discoveryCurationEnabled) return false;
766
+ if (this.modelType !== "model" || !this.modelDef) return false;
767
+ const exports = this.modelDef.exports;
768
+ if (!Array.isArray(exports) || exports.length > 0) return false;
769
+ return (this.modelDef.imports?.length ?? 0) > 0;
770
+ }
771
+
772
+ /** Set by the owning Package; see {@link assertQueryBoundaryEarly}. */
773
+ public setQueryBoundary(policy: {
774
+ mode: "declared" | "all";
775
+ exploresDeclared: boolean;
776
+ isQueryEntryPoint: boolean;
777
+ }): void {
778
+ this.queryBoundary = policy;
779
+ }
780
+
781
+ /**
782
+ * Query boundary, step 1 of 2 — the PRE-compilation gate. Enforces the rule
783
+ * "queryable == discoverable": a source is a valid top-level query target
784
+ * only if it is in the package's discovery surface (`explores` files +
785
+ * their `export {}` closure). This is the *what* axis (identity-free);
786
+ * `#(authorize)` is the orthogonal *who* axis, and both must pass. Denials
787
+ * are a generic 404 ({@link NotQueryableError}) so a hidden target is
788
+ * indistinguishable from a non-existent one. Inert unless `explores` is
789
+ * declared AND mode is "declared" (the default). Notebooks are exempt
790
+ * (always public) and never call this.
791
+ *
792
+ * This step runs before compilation so a denied target can't be probed via
793
+ * compile errors (schema oracle), and it only POSITIVELY denies — explicit
794
+ * names it can check, and ad-hoc text whose surface-resolved target is a
795
+ * model-declared non-curated source. Anything it can't pin returns
796
+ * "deferred" for {@link assertQueryBoundaryCompiled} to settle against the
797
+ * compiled run target. "cleared" admissions (explicit curated names) skip
798
+ * the backstop: an exported named query may read hidden sources internally —
799
+ * that is the author's deliberate exposure, and re-deriving its source from
800
+ * the compiled query must not re-deny it.
801
+ */
802
+ public assertQueryBoundaryEarly(
803
+ sourceName?: string,
804
+ queryName?: string,
805
+ query?: string,
806
+ ): "cleared" | "deferred" {
807
+ // Notebooks are always public — they can't be explores and are never
808
+ // gated by the boundary, even when reached via the /compile path.
809
+ if (this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX)) return "cleared";
810
+ const { mode, exploresDeclared, isQueryEntryPoint } = this.queryBoundary;
811
+ // No opt-in surface (no explores) or explicitly decoupled ("all") ⇒ the
812
+ // boundary is discovery-only; everything compiled stays queryable.
813
+ if (mode === "all" || !exploresDeclared) return "cleared";
814
+
815
+ // File-level: a non-`explores` model file is not a query entry point.
816
+ // This is the robust line — the file is named by the request URL, so
817
+ // there is nothing to resolve and nothing to evade.
818
+ if (!isQueryEntryPoint) {
819
+ throw new NotQueryableError(`No queryable model "${this.modelPath}".`);
820
+ }
821
+
822
+ const curatedSources = this.curatedSourceNames();
823
+ const curatedQueries = new Set(
824
+ (this.getQueries() ?? []).map((q) => q.name).filter(Boolean),
825
+ );
826
+
827
+ // A named query/view is an author-exported entry point (the author chose
828
+ // to expose it, even if it reads hidden sources internally) — admit it on
829
+ // its own name, or on the explicitly-named curated source it runs against.
830
+ if (queryName && !query) {
831
+ // The exported-query fast-path admits a *pure* named query (`run: q`)
832
+ // on its own name. It must NOT fire when a source is also named: a
833
+ // request is `run: <sourceName>-><queryName>`, so a hidden source
834
+ // could otherwise be reached via a view whose name happens to collide
835
+ // with an exported top-level query (and clearing skips the compiled
836
+ // backstop). With a source prefix, only the named source's curation
837
+ // gates the request.
838
+ if (!sourceName && curatedQueries.has(queryName)) return "cleared";
839
+ if (sourceName && curatedSources.has(sourceName)) return "cleared";
840
+ throw new NotQueryableError(`No queryable query "${queryName}".`);
841
+ }
842
+
843
+ // An explicitly-named source: admit iff curated.
844
+ if (sourceName) {
845
+ if (curatedSources.has(sourceName)) return "cleared";
846
+ throw new NotQueryableError(`No queryable source "${sourceName}".`);
847
+ }
848
+
849
+ // Ad-hoc text: positively deny only a surface-resolved target that is a
850
+ // model-declared source outside the curated surface (and doesn't derive
851
+ // from a curated one) — pre-compile, so its compile errors can't leak
852
+ // schema. Everything else (inline derivations, multi-statement, forms
853
+ // the regex can't read) defers to the compiled backstop.
854
+ if (query) {
855
+ const target = this.extractSourceName(query);
856
+ if (
857
+ target &&
858
+ !curatedSources.has(target) &&
859
+ !this.derivesFromCurated(target, query) &&
860
+ this.sources?.some((s) => s.name === target)
861
+ ) {
862
+ throw new NotQueryableError(`No queryable source "${target}".`);
863
+ }
864
+ }
865
+ return "deferred";
866
+ }
867
+
868
+ /**
869
+ * Query boundary, step 2 of 2 — the POST-compilation backstop for
870
+ * "deferred" admissions. `compiledSource` is the run target read off the
871
+ * compiled query's `structRef` (see
872
+ * {@link resolveAuthorizeSourceFromRunnable}) — the source Malloy actually
873
+ * executes, surviving named-query indirection, multi-statement forms (the
874
+ * LAST `run:` wins), and derivation. This inspects compiler *output* only;
875
+ * compilation itself is never altered or restricted.
876
+ *
877
+ * Admit iff the compiled target is curated, or is an ad-hoc alias that
878
+ * derives from a curated source (`source: x is customers extend { … }` →
879
+ * `run: x` — composing over a queryable source is itself queryable). FAIL
880
+ * CLOSED otherwise, including when the target can't be resolved at all.
881
+ * (Raw inline `conn.sql(...)` never reaches here on the query path —
882
+ * restricted-mode compilation rejects it first.)
883
+ */
884
+ public assertQueryBoundaryCompiled(
885
+ compiledSource: string | undefined,
886
+ query?: string,
887
+ ): void {
888
+ // Notebooks are always public (mirrors assertQueryBoundaryEarly).
889
+ if (this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX)) return;
890
+ const { mode, exploresDeclared, isQueryEntryPoint } = this.queryBoundary;
891
+ if (mode === "all" || !exploresDeclared) return;
892
+ if (!isQueryEntryPoint) {
893
+ throw new NotQueryableError(`No queryable model "${this.modelPath}".`);
894
+ }
895
+ if (compiledSource) {
896
+ if (this.curatedSourceNames().has(compiledSource)) return;
897
+ if (query && this.derivesFromCurated(compiledSource, query)) return;
898
+ }
899
+ throw new NotQueryableError("Query target is not queryable.");
900
+ }
901
+
902
+ /**
903
+ * The /compile-path wrapper: resolve the compiled run target from the
904
+ * materializer and apply the boundary backstop. No-ops when the boundary is
905
+ * inert, so the resolution work is skipped for unaffected packages.
906
+ */
907
+ public async assertQueryBoundaryForRunnable(
908
+ runnable: { getPreparedQuery(): Promise<unknown> },
909
+ query?: string,
910
+ ): Promise<void> {
911
+ const { mode, exploresDeclared } = this.queryBoundary;
912
+ if (mode === "all" || !exploresDeclared) return;
913
+ this.assertQueryBoundaryCompiled(
914
+ await this.resolveAuthorizeSourceFromRunnable(runnable),
915
+ query,
916
+ );
917
+ }
918
+
919
+ /** Source names in the export-curated discovery surface (= the directly
920
+ * queryable set under the "declared" boundary). */
921
+ private curatedSourceNames(): Set<string> {
922
+ return new Set(
923
+ (this.getSources() ?? [])
924
+ .map((s) => s.name)
925
+ .filter((n): n is string => n !== undefined),
926
+ );
927
+ }
928
+
929
+ /** True if `name` reaches a curated source by walking the ad-hoc text's
930
+ * `source: NAME is BASE` derivation declarations — composition over a
931
+ * queryable source is itself queryable. */
932
+ private derivesFromCurated(name: string, query: string): boolean {
933
+ const curated = this.curatedSourceNames();
934
+ const aliasOf = Model.buildAliasMap(query);
935
+ let current: string | undefined = name;
936
+ const seen = new Set<string>();
937
+ while (current && !seen.has(current)) {
938
+ if (curated.has(current)) return true;
939
+ seen.add(current);
940
+ current = aliasOf.get(current);
941
+ }
942
+ return false;
943
+ }
944
+
945
+ /** Map each ad-hoc source alias to the base it derives from
946
+ * (`source: NAME is BASE …` → NAME → BASE). Shared by filter inheritance
947
+ * ({@link resolveFilterSource}) and the query boundary, which both walk
948
+ * derivation chains in caller-authored query text. */
949
+ private static buildAliasMap(query: string): Map<string, string> {
950
+ const aliasOf = new Map<string, string>();
951
+ // Match a bare `\w+` or a backtick-quoted Malloy identifier on BOTH sides
952
+ // (a hyphenated source like `customer-orders` needs quoting), same pattern
953
+ // as `extractSourceName`. The quoted capture returns the inner name (no
954
+ // backticks), keeping aliases keyed the way `curatedSourceNames()` and the
955
+ // compiled run target are — otherwise a derivation over a quoted source
956
+ // wouldn't walk back to the curated set and would be falsely denied.
957
+ const declRe =
958
+ /source\s*:\s*(?:`([^`]+)`|(\w+))\s+is\s+(?:`([^`]+)`|(\w+))/g;
959
+ let match: RegExpExecArray | null;
960
+ while ((match = declRe.exec(query)) !== null) {
961
+ aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
962
+ }
963
+ return aliasOf;
964
+ }
965
+
966
+ /**
967
+ * Compile-time renderer-tag validation, run on the main thread.
968
+ *
969
+ * The renderer (`@malloydata/render`) is a large solid-js bundle that mutates
970
+ * DOM globals at import; loading it inside the package-load worker isolate
971
+ * destabilizes that thread (it is deliberately kept pure-CPU). So validation
972
+ * runs here, after the worker has hydrated this Model, where the renderer is
973
+ * already used at query time (see query.controller.ts / execute_query_tool.ts).
974
+ *
975
+ * Prepares each annotated top-level named query (`run: <name>`) and each
976
+ * annotated source view (`run: <source> -> <view>`) compile-only -- no
977
+ * execution -- to get a stable result schema, then runs the renderer's
978
+ * headless `validateRenderTags`. Targets with no annotations carry no render
979
+ * tags, so they are skipped without compiling. Any
980
+ * error-severity finding throws a `ModelCompilationError` (HTTP 424) so a
981
+ * misconfigured tag (e.g. a child-only `# big_value { sparkline=... }` placed
982
+ * on a view with no activating big_value) fails the package load with a clear
983
+ * message instead of rendering as "[object Object]" at query time. Warnings
984
+ * (e.g. unread tags) are left for the query-time `renderLogs` surface so a
985
+ * benign render lint never blocks a load.
986
+ */
987
+ public async validateRenderTags(): Promise<void> {
988
+ const mm = this.modelMaterializer;
989
+ if (!mm) {
990
+ return;
991
+ }
992
+ // Dynamic import (like execute_query_tool.ts): the renderer is heavy and
993
+ // mutates DOM globals on load, so only pull it in when there's a model to
994
+ // validate.
995
+ const { validateRenderTags } = await import(
996
+ "@malloydata/render-validator"
997
+ );
998
+
999
+ // Renderable targets: top-level named queries and every view declared on a
1000
+ // source. Source views are where render tags like `# big_value` usually
1001
+ // live, and they are NOT in `this.queries`.
1002
+ const targets: { label: string; queryString: string }[] = [];
1003
+ for (const query of this.queries ?? []) {
1004
+ // Only an annotated, named query can carry a render tag to validate;
1005
+ // skip the rest rather than compiling every query in the package.
1006
+ if (!query.name || !query.annotations?.length) {
1007
+ continue;
1008
+ }
1009
+ // Quote the identifier (see quoteMalloyIdentifier) so a name needing
1010
+ // Malloy quoting still lexes and cannot break out of the quotes.
1011
+ targets.push({
1012
+ label: query.name,
1013
+ queryString: `run: ${quoteMalloyIdentifier(query.name)}`,
1014
+ });
1015
+ }
1016
+ for (const source of this.sources ?? []) {
1017
+ for (const view of source.views ?? []) {
1018
+ // Render tags live on the view's own or inherited annotations, not
1019
+ // via source-to-view inheritance, so an unannotated view has
1020
+ // nothing to validate and need not be compiled. (A model-level
1021
+ // `##` tag is the one case this gate doesn't reach, but those are
1022
+ // theme/config, not the child-only chart tags this guards against.)
1023
+ if (!view.annotations?.length) {
1024
+ continue;
1025
+ }
1026
+ // Quote both identifiers (see quoteMalloyIdentifier): an unquoted
1027
+ // name like `gated-source` fails to lex, and the catch below would
1028
+ // then silently skip the very view this is meant to validate.
1029
+ targets.push({
1030
+ label: `${source.name} -> ${view.name}`,
1031
+ queryString: `run: ${quoteMalloyIdentifier(source.name)} -> ${quoteMalloyIdentifier(view.name)}`,
1032
+ });
1033
+ }
1034
+ }
1035
+
1036
+ for (const target of targets) {
1037
+ let result: Malloy.Result;
1038
+ try {
1039
+ const prepared = await mm
1040
+ .loadQuery(target.queryString)
1041
+ .getPreparedResult();
1042
+ result = prepared.toStableResult();
1043
+ } catch {
1044
+ // A view/query that fails to prepare is reported by the normal
1045
+ // compile path; don't mask that with a render-tag error.
1046
+ continue;
1047
+ }
1048
+ const errors = validateRenderTags(result).filter(
1049
+ (log) => log.severity === "error",
1050
+ );
1051
+ if (errors.length > 0) {
1052
+ throw new ModelCompilationError({
1053
+ message: `Invalid renderer configuration on '${target.label}': ${errors
1054
+ .map((e) => e.message)
1055
+ .join("; ")}`,
1056
+ });
1057
+ }
1058
+ }
657
1059
  }
658
1060
 
659
1061
  public async getModel(): Promise<ApiCompiledModel> {
@@ -726,6 +1128,18 @@ export class Model {
726
1128
  if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
727
1129
  throw new BadRequestError("Model has no queryable entities.");
728
1130
 
1131
+ // Query boundary FIRST (the *what* axis): reject a target that isn't in
1132
+ // the package's queryable surface with a generic 404, before authorize
1133
+ // (the *who* axis) and before compilation — so a non-queryable source is
1134
+ // indistinguishable from a non-existent one and can't be probed.
1135
+ // "deferred" means the early gate couldn't pin the target; the compiled
1136
+ // backstop below settles it against the source the query actually runs.
1137
+ const boundary = this.assertQueryBoundaryEarly(
1138
+ sourceName,
1139
+ queryName,
1140
+ query,
1141
+ );
1142
+
729
1143
  // Early fast-path authorize gate (before loadQuery). Resolve the source
730
1144
  // from surface syntax; gate if it names one. This runs BEFORE compilation
731
1145
  // so the gate can't be used as a schema oracle — without it, a denied
@@ -842,6 +1256,16 @@ export class Model {
842
1256
  // AccessDeniedError stays a 403; independent of bypassFilters.
843
1257
  const compiledSource =
844
1258
  await this.resolveAuthorizeSourceFromRunnable(runnable);
1259
+
1260
+ // Boundary backstop (the *what* axis, 404) BEFORE the authorize gate
1261
+ // (the *who* axis, 403): settle "deferred" early-gate admissions against
1262
+ // the compiled run target. Skipped for "cleared" admissions — an
1263
+ // explicitly-named exported query may read hidden sources internally
1264
+ // (the author's deliberate exposure), and must not be re-denied here.
1265
+ if (boundary === "deferred") {
1266
+ this.assertQueryBoundaryCompiled(compiledSource, query);
1267
+ }
1268
+
845
1269
  // Run unless it's the redundant re-probe of the exact named source the
846
1270
  // early gate already cleared. When compiledSource is unknown/unresolved,
847
1271
  // this still runs and assertAuthorized applies the model-wide file-level
@@ -950,8 +1374,11 @@ export class Model {
950
1374
  sourceInfos: this.getSourceInfos()?.map((sourceInfo) =>
951
1375
  JSON.stringify(sourceInfo),
952
1376
  ),
953
- sources: this.sources,
954
- queries: this.queries,
1377
+ // Discovery surface: an explore lists only its export closure
1378
+ // (getSources/getQueries curate); `this.sources` stays complete for
1379
+ // enforcement and resolution.
1380
+ sources: this.getSources(),
1381
+ queries: this.getQueries(),
955
1382
  givens: this.givens,
956
1383
  } as ApiCompiledModel;
957
1384
  }
@@ -1007,6 +1434,10 @@ export class Model {
1007
1434
  modelPath: this.modelPath,
1008
1435
  malloyVersion: MALLOY_VERSION,
1009
1436
  modelInfo: JSON.stringify(this.modelInfo ?? {}),
1437
+ // Raw-notebook view is uncurated (complete `this.sources`/`this.queries`,
1438
+ // not the export-filtered `getSources`/`getQueries`): notebooks can't be
1439
+ // imported, so their in-file sources have no internal/import-only role to
1440
+ // hide — they're always public. Model files curate; notebooks don't.
1010
1441
  sources: this.modelDef && this.sources,
1011
1442
  queries: this.modelDef && this.queries,
1012
1443
  annotations: allAnnotations,
@@ -75,11 +75,19 @@ describe("service/package", () => {
75
75
  new Map([
76
76
  [
77
77
  "model1.malloy",
78
- { getPath: () => "model1.malloy" } as unknown as Model,
78
+ {
79
+ getPath: () => "model1.malloy",
80
+ setDiscoveryCuration: () => {},
81
+ setQueryBoundary: () => {},
82
+ } as unknown as Model,
79
83
  ],
80
84
  [
81
85
  "model2.malloynb",
82
- { getPath: () => "model2.malloynb" } as unknown as Model,
86
+ {
87
+ getPath: () => "model2.malloynb",
88
+ setDiscoveryCuration: () => {},
89
+ setQueryBoundary: () => {},
90
+ } as unknown as Model,
83
91
  ],
84
92
  ]),
85
93
  );
@@ -294,17 +302,21 @@ describe("service/package", () => {
294
302
  {
295
303
  getPath: () => "model1.malloy",
296
304
  getModel: () => "foo",
305
+ setDiscoveryCuration: () => {},
306
+ setQueryBoundary: () => {},
297
307
  } as unknown as Model,
298
308
  ],
299
309
  [
300
310
  "model2.malloynb",
301
311
  {
302
312
  getPath: () => "model2.malloynb",
313
+ setDiscoveryCuration: () => {},
303
314
  getNotebookError: () => {
304
315
  return {
305
316
  message: "This is the error",
306
317
  };
307
318
  },
319
+ setQueryBoundary: () => {},
308
320
  } as unknown as Model,
309
321
  ],
310
322
  ]),