@malloy-publisher/server 0.0.229 → 0.0.231

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/dist/app/api-doc.yaml +18 -9
  2. package/dist/app/assets/{EnvironmentPage-QOoHiVeJ.js → EnvironmentPage-wa_EPkwK.js} +1 -1
  3. package/dist/app/assets/{HomePage-C71GOfVW.js → HomePage-jnCrupQp.js} +1 -1
  4. package/dist/app/assets/{LightMode-CrgCAwLe.js → LightMode-DYbwNULZ.js} +1 -1
  5. package/dist/app/assets/{MainPage-BG5__FN3.js → MainPage-CuJLrPNI.js} +1 -1
  6. package/dist/app/assets/{MaterializationsPage-DE6PnrDR.js → MaterializationsPage-D_67x2ee.js} +1 -1
  7. package/dist/app/assets/{ModelPage-CcBjcbLm.js → ModelPage-D5JtAWqR.js} +1 -1
  8. package/dist/app/assets/{PackagePage-JTy3ztkB.js → PackagePage-BRwtqUSG.js} +1 -1
  9. package/dist/app/assets/{RouteError-Cymbp47a.js → RouteError-CBNNrnSD.js} +1 -1
  10. package/dist/app/assets/{ThemeEditorPage-C_nMnHr8.js → ThemeEditorPage-CTCeBneA.js} +1 -1
  11. package/dist/app/assets/{WorkbookPage-CPQu-DQx.js → WorkbookPage-SN6f1RBm.js} +1 -1
  12. package/dist/app/assets/{core-Coi3caGs.es-CSOmajHS.js → core-Dp3q5Ieu.es-CD5FvM2s.js} +1 -1
  13. package/dist/app/assets/{index-DlWCXghy.js → index-B3Nn8Vm2.js} +459 -446
  14. package/dist/app/assets/index-BLCx1EdC.js +18 -0
  15. package/dist/app/assets/{index-DxArlgRD.js → index-C_tJstcx.js} +4 -4
  16. package/dist/app/assets/{index-CcuuST2X.js → index-CfmBVB6M.js} +1 -1
  17. package/dist/app/assets/{index-CkmABCAw.js → index-DU4r7GdU.js} +435 -422
  18. package/dist/app/index.html +1 -1
  19. package/dist/package_load_worker.mjs +108 -7
  20. package/dist/server.mjs +576 -50
  21. package/package.json +12 -12
  22. package/scripts/bake-duckdb-extensions.js +5 -2
  23. package/src/config.ts +40 -0
  24. package/src/ducklake_version.spec.ts +163 -0
  25. package/src/ducklake_version.ts +153 -0
  26. package/src/errors.ts +12 -0
  27. package/src/malloy_pin_prereqs.spec.ts +25 -0
  28. package/src/mcp/skills/skills_bundle.json +1 -1
  29. package/src/server.ts +7 -0
  30. package/src/service/authorize.spec.ts +22 -0
  31. package/src/service/authorize.ts +267 -10
  32. package/src/service/authorize_integration.spec.ts +1068 -26
  33. package/src/service/connection.spec.ts +71 -0
  34. package/src/service/connection.ts +312 -25
  35. package/src/service/connection_config.spec.ts +21 -0
  36. package/src/service/connection_config.ts +15 -1
  37. package/src/service/ducklake_lazy_attach.spec.ts +110 -0
  38. package/src/service/environment.ts +43 -9
  39. package/src/service/environment_store.spec.ts +197 -12
  40. package/src/service/environment_store.ts +88 -14
  41. package/src/service/extension_fetch_policy.spec.ts +256 -0
  42. package/src/service/materialization_scheduler.spec.ts +29 -0
  43. package/src/service/materialization_service.spec.ts +119 -1
  44. package/src/service/model.spec.ts +67 -0
  45. package/src/service/model.ts +656 -31
  46. package/src/service/package.spec.ts +38 -0
  47. package/src/service/package.ts +12 -1
  48. package/src/storage/duckdb/MaterializationRepository.spec.ts +39 -2
  49. package/src/storage/duckdb/MaterializationRepository.ts +12 -0
  50. package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +256 -0
  51. package/dist/app/assets/index-CM2qhQCI.js +0 -18
@@ -4,6 +4,10 @@ import {
4
4
  Connection,
5
5
  FixedConnectionMap,
6
6
  GivenValue,
7
+ isBasicArray,
8
+ isJoined,
9
+ isRepeatedRecord,
10
+ isSourceDef,
7
11
  MalloyConfig,
8
12
  MalloyError,
9
13
  ModelDef,
@@ -13,6 +17,8 @@ import {
13
17
  QueryData,
14
18
  QueryMaterializer,
15
19
  Runtime,
20
+ type FieldDef,
21
+ type SourceDef,
16
22
  } from "@malloydata/malloy";
17
23
  import * as Malloy from "@malloydata/malloy-interfaces";
18
24
  import {
@@ -52,6 +58,7 @@ import { modelAnnotations } from "./annotations";
52
58
  import {
53
59
  collectAuthorizeExprs,
54
60
  evaluateAuthorize,
61
+ referencedGivenNames,
55
62
  validateAuthorizeProbes,
56
63
  } from "./authorize";
57
64
  import {
@@ -93,6 +100,28 @@ const MALLOY_VERSION = (
93
100
  export type ModelType = "model" | "notebook";
94
101
  type ModelConnectionInput = MalloyConfig | Map<string, Connection>;
95
102
 
103
+ /** One reachable authorize gate found by {@link Model.collectAllReachableGates}. */
104
+ type GateEntry = { label: string; exprs: string[]; selfContained: boolean };
105
+
106
+ /**
107
+ * True for a struct field that Malloy's own `isJoined()` recognizes as a join
108
+ * (`'join' in sd`) purely because it's a nested-column kind — `RecordDef`,
109
+ * `RepeatedRecordDef`, and `BasicArrayDef` all extend `JoinBase` — rather than
110
+ * an actual join to another source. `isSourceDef()` already excludes
111
+ * `'record'`/`'array'`, but that alone doesn't distinguish "ordinary
112
+ * STRUCT/ARRAY/JSON column" from genuine join/source-shape drift; this checks
113
+ * the field kinds explicitly so the two can't be conflated. These field kinds
114
+ * can never carry `#(authorize)`, so a walk must skip them, not treat them as
115
+ * an unresolvable join.
116
+ */
117
+ function isRecordOrArrayField(field: FieldDef): boolean {
118
+ return (
119
+ (field as { type?: string }).type === "record" ||
120
+ isBasicArray(field) ||
121
+ isRepeatedRecord(field)
122
+ );
123
+ }
124
+
96
125
  interface RunnableNotebookCell {
97
126
  type: "code" | "markdown";
98
127
  text: string;
@@ -149,6 +178,14 @@ export class Model {
149
178
  /** Model-wide `##(authorize)` expressions; apply to every query in the
150
179
  * model, including ad-hoc inline sources not declared in the model. */
151
180
  private fileLevelAuthorize: string[] = [];
181
+ /** Given names (`$NAME`) referenced by any authorize gate reachable
182
+ * anywhere in this model -- the file-level gate, every top-level
183
+ * source's own gate, and every gate reached transitively from a
184
+ * top-level source via join_* / query-source derivation (the same walk
185
+ * {@link assertAuthorizedForAllSources} runs at request time). Computed
186
+ * once at construction; see {@link filterGivensToModelSurface}, the only
187
+ * consumer. */
188
+ private authorizeReferencedGivenNames: Set<string> = new Set();
152
189
  /** Whether discovery accessors curate to the `export {}` closure. Pushed
153
190
  * down by the owning Package (see Package.applyDiscoveryPolicyToModels):
154
191
  * true only when the package declares `explores` in publisher.json.
@@ -238,6 +275,16 @@ export class Model {
238
275
  } catch {
239
276
  this.fileLevelAuthorize = [];
240
277
  }
278
+ // Guarded the same way as fileLevelAuthorize above: a malformed gate
279
+ // reachable only through a join/derivation must not throw out of the
280
+ // constructor (gateExprsForOwnAnnotations already fails closed per
281
+ // struct, so this can only throw on something unrelated).
282
+ try {
283
+ this.authorizeReferencedGivenNames =
284
+ this.computeAuthorizeReferencedGivenNames();
285
+ } catch {
286
+ this.authorizeReferencedGivenNames = new Set();
287
+ }
241
288
  this.modelInfo =
242
289
  modelInfo ??
243
290
  (this.modelDef ? modelDefToModelInfo(this.modelDef) : undefined);
@@ -266,6 +313,24 @@ export class Model {
266
313
  return this.filterMap.get(sourceName) ?? [];
267
314
  }
268
315
 
316
+ /**
317
+ * Given name → declared Malloy type, from this model's own given surface
318
+ * ({@link givens}). Passed to {@link evaluateAuthorize}'s self-contained
319
+ * probe fallback so it prefers the gate author's DECLARED type over
320
+ * inferring one from the caller's JS value (e.g. `$LEVEL > 3` compares
321
+ * numerically even if the caller sends `"5"`). Only reaches a given
322
+ * declared within one import hop of this model — a gate on a source
323
+ * reached through a deeper transitive import isn't on this surface, so
324
+ * that case still falls back to inferring from the value.
325
+ */
326
+ private givenDeclaredTypes(): Map<string, string> {
327
+ return new Map(
328
+ (this.givens ?? [])
329
+ .filter((g) => g.name != null && g.type != null)
330
+ .map((g) => [g.name, g.type] as [string, string]),
331
+ );
332
+ }
333
+
269
334
  /**
270
335
  * Effective authorize expressions gating a source: file-level
271
336
  * `##(authorize)` followed by the source's own `#(authorize)`, evaluated as
@@ -281,6 +346,78 @@ export class Model {
281
346
  );
282
347
  }
283
348
 
349
+ /**
350
+ * Filter caller-supplied givens down to the ones safe to forward to the
351
+ * REAL query's `getPreparedResult`/`run`. A caller may legitimately need
352
+ * to supply a value only so a joined source's `#(authorize)` gate can see
353
+ * it — that gate is evaluated separately, against the FULL unfiltered
354
+ * givens (see `assertAuthorizedForAllSources`, which runs before this
355
+ * filter is applied). Malloy's own given-resolution doesn't flatten a
356
+ * `given:` declared more than one import hop away into the entry model's
357
+ * namespace (see `docs/authorize.md`), so passing such an authorize-only
358
+ * name straight through to the real query throws ("givens: unknown
359
+ * given"). Dropping it here is safe: the query itself can't reference a
360
+ * given it doesn't declare, so there is nothing for the dropped value to
361
+ * have affected.
362
+ *
363
+ * Only drops a name that is BOTH absent from this model's own given
364
+ * surface ({@link givens}) AND referenced by an authorize gate reachable
365
+ * in this model ({@link authorizeReferencedGivenNames}) — i.e. a name
366
+ * that could only have been supplied for gate evaluation. A name that no
367
+ * gate references is left untouched even when the model doesn't surface
368
+ * it, so a genuinely unknown / typo'd / legitimately-needed-but-unsurfaced
369
+ * `where:` given still reaches the real query and fails closed via
370
+ * Malloy's own "unknown given" error, instead of being silently swallowed
371
+ * and falling back to its declared default (over-exposure).
372
+ */
373
+ private filterGivensToModelSurface(
374
+ givens: Record<string, GivenValue> | undefined,
375
+ ): Record<string, GivenValue> | undefined {
376
+ if (!givens) return givens;
377
+ const surfaceNames = new Set((this.givens ?? []).map((g) => g.name));
378
+ const filtered: Record<string, GivenValue> = {};
379
+ for (const [name, value] of Object.entries(givens)) {
380
+ const authorizeOnly =
381
+ !surfaceNames.has(name) &&
382
+ this.authorizeReferencedGivenNames.has(name);
383
+ if (!authorizeOnly) filtered[name] = value;
384
+ }
385
+ return filtered;
386
+ }
387
+
388
+ /**
389
+ * Compute {@link authorizeReferencedGivenNames}: every given name (`$NAME`)
390
+ * referenced by an authorize gate expression reachable anywhere in this
391
+ * model. Walks the file-level gate plus, for every top-level source in
392
+ * `modelDef.contents`, every gate reached from it via
393
+ * {@link collectAllReachableGates} — the exact same unified traversal
394
+ * {@link assertAuthorizedForAllSources} runs per-query, just rooted at
395
+ * every top-level source instead of one run target. Runs once at
396
+ * construction, not per request.
397
+ */
398
+ private computeAuthorizeReferencedGivenNames(): Set<string> {
399
+ const names = new Set<string>();
400
+ const addExprs = (exprs: string[]) => {
401
+ for (const expr of exprs) {
402
+ for (const name of referencedGivenNames(expr)) names.add(name);
403
+ }
404
+ };
405
+ addExprs(this.fileLevelAuthorize);
406
+ const modelDef = this.modelDef;
407
+ if (!modelDef) return names;
408
+ for (const entry of Object.values(modelDef.contents)) {
409
+ if (!isSourceDef(entry)) continue;
410
+ for (const { exprs } of this.collectAllReachableGates(
411
+ entry,
412
+ modelDef,
413
+ new Set(),
414
+ )) {
415
+ addExprs(exprs);
416
+ }
417
+ }
418
+ return names;
419
+ }
420
+
284
421
  /**
285
422
  * Whether the model declares any `#(authorize)` / `##(authorize)` gate at all
286
423
  * (file-level or on any source). Lets callers cheaply skip authorize work for
@@ -323,9 +460,34 @@ export class Model {
323
460
  sourceName: string | undefined,
324
461
  givens: Record<string, GivenValue>,
325
462
  ): Promise<void> {
326
- const exprs = this.effectiveAuthorizeFor(sourceName);
463
+ await this.assertAuthorizedExprs(
464
+ sourceName ?? "(query)",
465
+ this.effectiveAuthorizeFor(sourceName),
466
+ givens,
467
+ );
468
+ }
469
+
470
+ /**
471
+ * Core runtime gate, factored out of {@link assertAuthorized} so the
472
+ * joined-source walk ({@link assertAuthorizedForAllSources}) can evaluate a
473
+ * gate it read directly off a join field's own struct annotations, without
474
+ * going through a `this.sources` name lookup (which would miss deep
475
+ * transitively-imported and inline-extend joins — see the design doc).
476
+ *
477
+ * `selfContainedFirst`, passed through to {@link evaluateAuthorize},
478
+ * isolates a joined/derived source's gate from the entry model's own
479
+ * ambient given namespace — see {@link collectAllReachableGates}'s
480
+ * `selfContained` tag. Left `false` (ambient-first) for the run target's
481
+ * OWN source gate ({@link assertAuthorized}'s call), which is correct to
482
+ * evaluate against the entry model's ambient namespace.
483
+ */
484
+ private async assertAuthorizedExprs(
485
+ label: string,
486
+ exprs: string[],
487
+ givens: Record<string, GivenValue>,
488
+ selfContainedFirst = false,
489
+ ): Promise<void> {
327
490
  if (exprs.length === 0) return; // unrestricted
328
- const label = sourceName ?? "(query)";
329
491
  const deny = () => {
330
492
  throw new AccessDeniedError(`Access denied for source "${label}".`);
331
493
  };
@@ -336,6 +498,8 @@ export class Model {
336
498
  this.modelMaterializer!,
337
499
  exprs,
338
500
  givens,
501
+ this.givenDeclaredTypes(),
502
+ { selfContainedFirst },
339
503
  );
340
504
  } catch (err) {
341
505
  // Fail closed — e.g. a referenced given had no supplied value.
@@ -349,6 +513,455 @@ export class Model {
349
513
  if (!passed) deny();
350
514
  }
351
515
 
516
+ /**
517
+ * Gate a compiled query against every gate reachable in its struct — the
518
+ * run target's own gate PLUS every joined source's gate, recursed
519
+ * transitively (A→B→C). Closes the join-bypass: `#(authorize)` on a source
520
+ * reached only via `join_*` was previously never enforced (see
521
+ * `docs/authorize.md` "Known limitations" and the design doc's W1).
522
+ *
523
+ * Each joined source's gate is read directly from the join field's OWN
524
+ * struct annotations (the struct spread in Malloy's `join.js` preserves
525
+ * them), not from a `this.sources` name lookup — a name lookup would
526
+ * silently miss a deep transitively-imported source (absent from
527
+ * `modelDef.contents`) or an inline-extend join (`referenceID` cleared),
528
+ * leaving both bypassable. File-level `##(authorize)` still prepends to
529
+ * every joined source, so the permissive-admin idiom keeps working
530
+ * transitively. Semantics are AND across sources: any single reachable
531
+ * gate failing denies the whole query, while each source's own list of
532
+ * expressions stays an OR disjunction. Identical expr-lists are evaluated
533
+ * once.
534
+ *
535
+ * Runs UNCONDITIONALLY — NOT guarded by {@link hasAuthorize}. `hasAuthorize`
536
+ * only inspects `this.sources` (top-level `modelDef.contents` sources), so a
537
+ * gated source reached only through a cross-file/deep-transitive join is
538
+ * invisible to it and the walk below would never run. The own-source probe
539
+ * and the joined-gate walk are both cheap no-ops for a genuinely ungated
540
+ * model (empty expr lists / no joined sources to find), so there is nothing
541
+ * to save by skipping them.
542
+ */
543
+ public async assertAuthorizedForAllSources(
544
+ runnable: { getPreparedQuery(): Promise<unknown> },
545
+ givens: Record<string, GivenValue>,
546
+ ): Promise<void> {
547
+ const ownSourceName =
548
+ await this.resolveAuthorizeSourceFromRunnable(runnable);
549
+ await this.assertAuthorized(ownSourceName, givens);
550
+
551
+ const { struct, modelDef, compositeResolvedSourceDef, extendSources } =
552
+ await this.resolveRunTargetStruct(runnable);
553
+ const seen = new Set<SourceDef>();
554
+ // `struct` IS the run target itself — its own gate stays ambient-first
555
+ // (`treatAsOwnGate: true`), matching `assertAuthorized` above. Do NOT
556
+ // dedup this against that call: AND-across-sources evaluates every
557
+ // reachable source's gate independently (see the comment below).
558
+ const joinedGates = this.collectAllReachableGates(
559
+ struct,
560
+ modelDef,
561
+ seen,
562
+ true,
563
+ );
564
+ if (modelDef) {
565
+ // Sources joined LOCALLY inside the query's own `-> { join_one: ...
566
+ // }` refinement live on the pipeline segment's `extendSource`, not on
567
+ // the run target's `struct.fields` — the walk above never sees them
568
+ // from `struct` alone. Walk each the same way as everything else.
569
+ const runTargetLabel = ownSourceName ?? "(run target)";
570
+ for (const field of extendSources) {
571
+ const { resolved, denyGate } = this.classifyJoinedField(
572
+ field,
573
+ runTargetLabel,
574
+ );
575
+ if (denyGate) {
576
+ joinedGates.push(denyGate);
577
+ continue;
578
+ }
579
+ if (!resolved) continue;
580
+ joinedGates.push(
581
+ ...this.collectAllReachableGates(resolved, modelDef, seen),
582
+ );
583
+ }
584
+ }
585
+ if (compositeResolvedSourceDef && modelDef) {
586
+ // The run target itself may be a composite source (`compose(a, b)`).
587
+ // Malloy resolves it to exactly one concrete member branch per query
588
+ // (surfaced as Query.compositeResolvedSourceDef), based on which
589
+ // fields the query references — so gate everything reachable from
590
+ // that RESOLVED branch, not every member. Gating every member would
591
+ // deny access through an open branch just because a sibling branch
592
+ // happens to be locked. This branch stands in for the run target
593
+ // ITSELF (just the concrete resolved shape), so its own gate stays
594
+ // ambient-first too (`treatAsOwnGate: true`) — same reasoning as the
595
+ // `struct` walk above.
596
+ //
597
+ // `compositeResolvedSourceDef && modelDef` is not a fail-open gap for
598
+ // a runnable composite RUN TARGET: Malloy's composite resolver
599
+ // (`_resolveCompositeSources` in `@malloydata/malloy`'s
600
+ // `composite-source-utils.ts`) marks composite resolution as
601
+ // required as soon as the top-level source is itself a composite,
602
+ // before any field-usage logic runs — so it always resolves to a
603
+ // concrete member (even one no query field discriminates, it just
604
+ // picks the first candidate — see the "no field forcing a choice"
605
+ // test in authorize_integration.spec.ts) or the compile fails
606
+ // outright (no member satisfies the query). The `undefined` case
607
+ // here is only ever "the run target genuinely isn't a composite" —
608
+ // not "a composite run target resolved to nothing". (A composite
609
+ // reached via a JOIN is a different story — Malloy doesn't surface
610
+ // which member it resolved to there, which is why the `joinedSource
611
+ // .type === "composite"` branch above walks every member
612
+ // conservatively instead.)
613
+ joinedGates.push(
614
+ ...this.collectAllReachableGates(
615
+ compositeResolvedSourceDef,
616
+ modelDef,
617
+ seen,
618
+ true,
619
+ ),
620
+ );
621
+ }
622
+ // Evaluate every reachable source's gate independently — do NOT dedup by
623
+ // expression text. AND-across-sources requires a probe per source: two
624
+ // distinct sources with identical gate text must each be evaluated, or a
625
+ // non-deterministic gate (e.g. one referencing random()) would be
626
+ // under-enforced (evaluated once, reused) — a fail-open relative to
627
+ // per-source enforcement. Probes are ~microsecond one-row DuckDB queries
628
+ // and reachable gated sources are few, so there is nothing worth deduping.
629
+ // (Cycles/repeat structs are already pruned in collectAllReachableGates
630
+ // by struct identity, so joinedGates holds no literal duplicates.)
631
+ for (const { label, exprs, selfContained } of joinedGates) {
632
+ await this.assertAuthorizedExprs(label, exprs, givens, selfContained);
633
+ }
634
+ }
635
+
636
+ /**
637
+ * Resolve the run-target `SourceDef` and its `ModelDef`, for walking joined
638
+ * sources. `prepared._modelDef` is the modelDef the query actually compiled
639
+ * against (falls back to `this.modelDef`); a string `structRef` resolves
640
+ * through `modelDef.contents`. Also surfaces `compositeResolvedSourceDef` —
641
+ * when the run target is itself a composite source (`compose(a, b)`), this
642
+ * is the ONE concrete member branch Malloy resolved the query against (see
643
+ * {@link assertAuthorizedForAllSources}). Also surfaces `extendSources` —
644
+ * `FieldDef[]` pulled from every pipeline segment's `extendSource`, i.e.
645
+ * every source joined LOCALLY inside the query's own `-> { join_one: ...
646
+ * }` refinement. Such a join lives on the query pipeline's segment, not on
647
+ * the run target's own `struct.fields`, so {@link collectAllReachableGates}
648
+ * walking `struct.fields` alone would never see it. Returns `undefined`s /
649
+ * an empty array if any of these can't be resolved — callers treat that as
650
+ * "no joins to check" rather than denying, since
651
+ * {@link assertAuthorizedForAllSources}'s own-source gate above is still the
652
+ * authoritative deny for an unresolvable target.
653
+ */
654
+ private async resolveRunTargetStruct(runnable: {
655
+ getPreparedQuery(): Promise<unknown>;
656
+ }): Promise<{
657
+ struct: SourceDef | undefined;
658
+ modelDef: ModelDef | undefined;
659
+ compositeResolvedSourceDef: SourceDef | undefined;
660
+ extendSources: FieldDef[];
661
+ }> {
662
+ try {
663
+ const prepared = (await runnable.getPreparedQuery()) as {
664
+ _query?: {
665
+ structRef?: unknown;
666
+ compositeResolvedSourceDef?: SourceDef;
667
+ pipeline?: { extendSource?: FieldDef[] }[];
668
+ };
669
+ _modelDef?: ModelDef;
670
+ };
671
+ const modelDef = prepared._modelDef ?? this.modelDef;
672
+ if (!modelDef)
673
+ return {
674
+ struct: undefined,
675
+ modelDef: undefined,
676
+ compositeResolvedSourceDef: undefined,
677
+ extendSources: [],
678
+ };
679
+ const structRef = prepared._query?.structRef;
680
+ const struct =
681
+ typeof structRef === "string"
682
+ ? modelDef.contents[structRef]
683
+ : structRef;
684
+ const extendSources = (prepared._query?.pipeline ?? []).flatMap(
685
+ (segment) => segment.extendSource ?? [],
686
+ );
687
+ return {
688
+ struct:
689
+ struct && typeof struct === "object"
690
+ ? (struct as SourceDef)
691
+ : undefined,
692
+ modelDef,
693
+ compositeResolvedSourceDef:
694
+ prepared._query?.compositeResolvedSourceDef,
695
+ extendSources,
696
+ };
697
+ } catch {
698
+ // Not fail-open: if getPreparedQuery() throws here, execution's own
699
+ // getPreparedResult()/run() (same compilation) throws too, so no data
700
+ // is returned. Safety depends on execution sharing this compilation.
701
+ return {
702
+ struct: undefined,
703
+ modelDef: undefined,
704
+ compositeResolvedSourceDef: undefined,
705
+ extendSources: [],
706
+ };
707
+ }
708
+ }
709
+
710
+ /**
711
+ * Effective authorize exprs read directly off a struct's OWN block
712
+ * annotations (file-level `##(authorize)` ++ its own `#(authorize)`). Used
713
+ * by the joined-source and composite-member walks below, which read a
714
+ * struct's gate straight off its own annotations rather than through a
715
+ * `this.sources` name lookup (a name lookup only covers top-level
716
+ * `modelDef.contents` sources and would miss these).
717
+ *
718
+ * Fails CLOSED: `extractSourcesFromModelDef` only validates authorize
719
+ * annotations for top-level `modelDef.contents` sources at model load, so
720
+ * a malformed gate on a source reachable ONLY through a join or a
721
+ * composite member is never probed there — a parse failure here can't be
722
+ * assumed unreachable/already-validated. Force denial (a single
723
+ * unsatisfiable `"false"` expr) rather than treating the parse failure as
724
+ * "no gate" (fail-open).
725
+ */
726
+ private gateExprsForOwnAnnotations(struct: SourceDef): string[] {
727
+ const ownNotes = (struct.annotations?.blockNotes ?? []).map(
728
+ (note) => note.text,
729
+ );
730
+ try {
731
+ return [
732
+ ...this.fileLevelAuthorize,
733
+ ...collectAuthorizeExprs(ownNotes),
734
+ ];
735
+ } catch {
736
+ return ["false"];
737
+ }
738
+ }
739
+
740
+ /**
741
+ * Classify one struct field found while walking a join site — shared by
742
+ * every call site in this file that iterates a `FieldDef[]` looking for
743
+ * joins (`collectAllReachableGates`'s own `struct.fields` walk, the
744
+ * `extendSources` walk, and the query-source inner-join walk), so the
745
+ * record/array-skip + drift-deny logic below lives in exactly one place.
746
+ *
747
+ * Returns:
748
+ * - `{ resolved }` — `field` is a genuine joined `SourceDef`; caller
749
+ * recurses into it via {@link collectAllReachableGates};
750
+ * - `{}` (both empty) — `field` isn't a join at all, or is a
751
+ * record/array-typed nested column ({@link isRecordOrArrayField}) that
752
+ * merely LOOKS like a join to `isJoined()`; skip it, there's no gate to
753
+ * find;
754
+ * - `{ denyGate }` — `field` is joined but resolved to neither of the
755
+ * above: real join/source-shape drift. Log loudly and return a
756
+ * synthetic always-false gate entry rather than silently `continue`-ing
757
+ * past it (fail OPEN), which is exactly the join-bypass this walk
758
+ * exists to close.
759
+ */
760
+ private classifyJoinedField(
761
+ field: FieldDef,
762
+ parentLabel: string,
763
+ ): { resolved?: SourceDef; denyGate?: GateEntry } {
764
+ if (!isJoined(field)) return {};
765
+ if (isRecordOrArrayField(field)) return {};
766
+ if (!isSourceDef(field)) {
767
+ logger.error(
768
+ "authorize: joined field failed to resolve to a walkable SourceDef; denying rather than silently skipping its gate (possible Malloy struct-shape drift)",
769
+ {
770
+ modelPath: this.modelPath,
771
+ parentSource: parentLabel,
772
+ fieldName: (field as { name?: string }).name,
773
+ fieldType: (field as { type?: string }).type,
774
+ },
775
+ );
776
+ return {
777
+ denyGate: {
778
+ label: `${parentLabel} (unresolvable joined source)`,
779
+ exprs: ["false"],
780
+ selfContained: false,
781
+ },
782
+ };
783
+ }
784
+ return { resolved: field as SourceDef };
785
+ }
786
+
787
+ /**
788
+ * Canonical entry point for "every gate reachable from `struct`" — the
789
+ * single walk every call site in this file uses, replacing what used to
790
+ * be a hand-composed sequence of narrower collectors per call site (the
791
+ * inconsistency between those sequences is what let two derivations
792
+ * launder a locked base's gate away — see the composite-branch and
793
+ * query-local-join call sites in {@link assertAuthorizedForAllSources}).
794
+ * Collects, for `struct` and recursively for everything below:
795
+ * - its own annotations ({@link gateExprsForOwnAnnotations});
796
+ * - every joined source (`struct.fields`), including every member of a
797
+ * joined composite (`compose(a, b)` reached via `join_*`) — Malloy
798
+ * doesn't surface which branch a JOINED composite resolved to (unlike
799
+ * a run target's own composite resolution, which the caller handles
800
+ * by passing the resolved branch in as `struct` directly), so a joined
801
+ * composite is walked conservatively: every member gated, any one
802
+ * failing denies the whole query;
803
+ * - if `struct` is itself query-derived (`source: x is y -> {...}`), the
804
+ * base it derives from (`query.structRef`, resolved the same way
805
+ * {@link resolveRunTargetStruct} resolves a run target's structRef) —
806
+ * a `QuerySourceDef`'s own `.fields`/`.annotations` reflect the
807
+ * DERIVED shape, not `y`'s gate, so without this the derivation
808
+ * launders the base's gate away;
809
+ * - if `struct` is itself query-derived, its own inner-pipeline
810
+ * `join_one`s (`-> { extend: { join_one: locked ... } ... }`), which
811
+ * live on the derivation's pipeline segment, not on `struct.fields`
812
+ * and not reachable via `query.structRef` — mirrors the run target's
813
+ * own `extendSources` handling in {@link assertAuthorizedForAllSources},
814
+ * applied to a query-source's `query.pipeline` instead of the run
815
+ * query's;
816
+ * - if `struct` is itself query-derived AND its base is a composite
817
+ * (`source: x is compose(a, b) -> {...}`), the ONE resolved member
818
+ * branch Malloy picked for THIS derivation (`query.compositeResolvedSourceDef`)
819
+ * — mirrors the run target's own composite-resolution handling in
820
+ * {@link assertAuthorizedForAllSources}, applied to a query-source's
821
+ * own resolution instead of the run query's.
822
+ * Every case above recurses through this SAME function, so a derivation,
823
+ * a join, and a composite member compose uniformly no matter how deep
824
+ * (a query-source over a joined query-source, a chained derivation, a
825
+ * composite member that is itself query-derived, etc).
826
+ *
827
+ * `seen` (struct-identity keyed) is shared across the whole walk by the
828
+ * caller, guarding cycles and repeat structs — a struct already visited
829
+ * anywhere in the walk is not walked again.
830
+ *
831
+ * `QuerySourceDef` isn't re-exported from the package root (same
832
+ * situation as `given.ts`'s `MalloyGiven` duck type), so query-source
833
+ * detection checks `.type` and reaches `.query.structRef`/`.query.pipeline`
834
+ * through a local shape rather than importing the real type.
835
+ *
836
+ * Each returned entry carries `selfContained`, telling the caller which
837
+ * order {@link evaluateAuthorize} should try for that gate (see
838
+ * `assertAuthorizedExprs`'s `selfContainedFirst`). `treatAsOwnGate` — true
839
+ * only for the run target's own struct and its own resolved composite
840
+ * branch (the two call sites in {@link assertAuthorizedForAllSources} that
841
+ * represent the run target ITSELF, not something reached by walking a
842
+ * join/derivation) — marks `struct`'s OWN top-level annotations entry
843
+ * `selfContained: false` (ambient-first, matching {@link assertAuthorized}'s
844
+ * treatment of the same gate). Every other entry — everything reached by
845
+ * recursing into a joined field, a composite member, a query-source's
846
+ * base, or an inner pipeline join, no matter how deep — is tagged
847
+ * `selfContained: true`: those are gates that live in a DIFFERENT source
848
+ * (possibly a different file/given-namespace) than the run target, so
849
+ * evaluating them ambiently against the entry model's own namespace risks
850
+ * a name collision silently granting access off the wrong given (see
851
+ * `evaluateAuthorize`'s `selfContainedFirst` doc).
852
+ */
853
+ private collectAllReachableGates(
854
+ struct: SourceDef | undefined,
855
+ modelDef: ModelDef | undefined,
856
+ seen: Set<SourceDef> = new Set(),
857
+ treatAsOwnGate = false,
858
+ ): GateEntry[] {
859
+ if (!struct || !modelDef || seen.has(struct)) return [];
860
+ seen.add(struct);
861
+
862
+ const results: GateEntry[] = [];
863
+ const label = (struct as { as?: string }).as ?? struct.name;
864
+ const ownExprs = this.gateExprsForOwnAnnotations(struct);
865
+ if (ownExprs.length > 0) {
866
+ results.push({
867
+ label,
868
+ exprs: ownExprs,
869
+ selfContained: !treatAsOwnGate,
870
+ });
871
+ }
872
+
873
+ for (const field of struct.fields as FieldDef[]) {
874
+ const { resolved, denyGate } = this.classifyJoinedField(field, label);
875
+ if (denyGate) {
876
+ results.push(denyGate);
877
+ continue;
878
+ }
879
+ if (!resolved) continue;
880
+ const joinedSource = resolved;
881
+ results.push(
882
+ ...this.collectAllReachableGates(joinedSource, modelDef, seen),
883
+ );
884
+ // A JOINED source that is itself a composite (`compose(a, b)`
885
+ // reached via `join_*`) is walked conservatively: unlike a run
886
+ // target's own composite resolution (handled by the caller passing
887
+ // the resolved branch in as `struct` directly, never as a joined
888
+ // field), Malloy doesn't surface which member a JOINED composite
889
+ // resolved to, so every member is gated — any member's gate
890
+ // failing denies the whole query.
891
+ if (joinedSource.type === "composite") {
892
+ const members = (
893
+ joinedSource as SourceDef & { sources: SourceDef[] }
894
+ ).sources;
895
+ for (const member of members) {
896
+ results.push(
897
+ ...this.collectAllReachableGates(member, modelDef, seen),
898
+ );
899
+ }
900
+ }
901
+ }
902
+
903
+ const duck = struct as unknown as {
904
+ type: string;
905
+ query?: {
906
+ structRef?: SourceDef | string;
907
+ compositeResolvedSourceDef?: SourceDef;
908
+ pipeline?: { extendSource?: FieldDef[] }[];
909
+ };
910
+ };
911
+ if (duck.type === "query_source") {
912
+ const ref = duck.query?.structRef;
913
+ const base = typeof ref === "string" ? modelDef.contents[ref] : ref;
914
+ if (base && isSourceDef(base)) {
915
+ results.push(
916
+ ...this.collectAllReachableGates(
917
+ base as SourceDef,
918
+ modelDef,
919
+ seen,
920
+ ),
921
+ );
922
+ }
923
+ // A query-source's own base may itself be a composite
924
+ // (`source: qs is compose(a, b) -> {...}`) — Malloy resolves that
925
+ // composite to exactly one concrete member branch for THIS
926
+ // query-source's derivation (surfaced the same way as a run
927
+ // target's own composite resolution, see
928
+ // assertAuthorizedForAllSources), carried on the query-source's
929
+ // OWN `query.compositeResolvedSourceDef`, not on `query.structRef`
930
+ // (the raw composite) or `query.pipeline`. Without walking it, a
931
+ // query-source derived from a locked composite member laundered
932
+ // that member's gate away. Recursing through this same function
933
+ // means a query-source nested at any depth (query-source over
934
+ // composite over query-source, etc) is covered uniformly.
935
+ const resolved = duck.query?.compositeResolvedSourceDef;
936
+ if (resolved) {
937
+ results.push(
938
+ ...this.collectAllReachableGates(resolved, modelDef, seen),
939
+ );
940
+ }
941
+ const innerJoins = (duck.query?.pipeline ?? []).flatMap(
942
+ (segment) => segment.extendSource ?? [],
943
+ );
944
+ for (const field of innerJoins) {
945
+ const { resolved: innerJoinSource, denyGate: innerJoinDenyGate } =
946
+ this.classifyJoinedField(field, label);
947
+ if (innerJoinDenyGate) {
948
+ results.push(innerJoinDenyGate);
949
+ continue;
950
+ }
951
+ if (!innerJoinSource) continue;
952
+ results.push(
953
+ ...this.collectAllReachableGates(
954
+ innerJoinSource,
955
+ modelDef,
956
+ seen,
957
+ ),
958
+ );
959
+ }
960
+ }
961
+
962
+ return results;
963
+ }
964
+
352
965
  /**
353
966
  * Gate ad-hoc compile/query text by the named source it targets. Resolves the
354
967
  * source from surface syntax (`extractRunTargetSourceName`) and applies the
@@ -369,17 +982,16 @@ export class Model {
369
982
  * Gate a compiled query by the source it actually reads, resolved from the
370
983
  * prepared query's `structRef` (authoritative — survives named-query and
371
984
  * multi-statement indirection that surface syntax misses, e.g. the executed
372
- * `run:` statement isn't the first one). Used as the `/compile` backstop once
373
- * a runnable exists.
985
+ * `run:` statement isn't the first one), PLUS every source reached
986
+ * transitively via join_* (see assertAuthorizedForAllSources). Used as the
987
+ * `/compile` backstop once a runnable exists, so `/compile` inherits the
988
+ * same join-bypass protection as the query path.
374
989
  */
375
990
  public async assertAuthorizedForRunnable(
376
991
  runnable: { getPreparedQuery(): Promise<unknown> },
377
992
  givens: Record<string, GivenValue>,
378
993
  ): Promise<void> {
379
- await this.assertAuthorized(
380
- await this.resolveAuthorizeSourceFromRunnable(runnable),
381
- givens,
382
- );
994
+ await this.assertAuthorizedForAllSources(runnable, givens);
383
995
  }
384
996
 
385
997
  /**
@@ -1290,18 +1902,22 @@ export class Model {
1290
1902
  this.assertQueryBoundaryCompiled(compiledSource, query);
1291
1903
  }
1292
1904
 
1293
- // Run unless it's the redundant re-probe of the exact named source the
1294
- // early gate already cleared. When compiledSource is unknown/unresolved,
1295
- // this still runs and assertAuthorized applies the model-wide file-level
1296
- // gate via effectiveAuthorizeFor. Note: on this path an ad-hoc inline
1905
+ // Gate the compiled run target's own source PLUS every source reached
1906
+ // transitively via join_* (assertAuthorizedForAllSources) this MUST
1907
+ // run unconditionally, not just when compiledSource !== earlySource:
1908
+ // the common `run: joiner -> {...}` case has an ungated top-level source
1909
+ // (so compiledSource === earlySource and the own-source re-probe really
1910
+ // would be redundant), but a joined source's gate has never been checked
1911
+ // yet on this request. The walk runs UNCONDITIONALLY (do NOT re-add a
1912
+ // hasAuthorize() guard here — it only sees top-level sources, so guarding
1913
+ // on it re-opens the deep-import join bypass); it is a cheap no-op for an
1914
+ // ungated model. When compiledSource is unknown/unresolved,
1915
+ // the own-source half still applies the model-wide file-level gate via
1916
+ // effectiveAuthorizeFor. Note: on this path an ad-hoc inline
1297
1917
  // `duckdb.sql(...)` query is rejected by restricted mode (the raw-SQL
1298
1918
  // ban from loadRestrictedQuery above) before it can run, so the
1299
1919
  // raw-warehouse bypass is closed by restricted mode — not by this gate.
1300
- // This fallback's job is to apply the file-level gate to permitted ad-hoc
1301
- // forms (declared-source references) whose source can't be named.
1302
- if (!(compiledSource && compiledSource === earlySource)) {
1303
- await this.assertAuthorized(compiledSource, givens ?? {});
1304
- }
1920
+ await this.assertAuthorizedForAllSources(runnable, givens ?? {});
1305
1921
 
1306
1922
  const maxRows = getMaxQueryRows();
1307
1923
  const maxBytes = getMaxResponseBytes();
@@ -1319,17 +1935,26 @@ export class Model {
1319
1935
  let rowLimit = 0;
1320
1936
  let executionTime = 0;
1321
1937
  let queryResults;
1938
+ // Givens supplied only so a joined source's authorize gate could see
1939
+ // them (checked above, against the full unfiltered set) must not reach
1940
+ // the real query if this model doesn't itself surface them — see
1941
+ // filterGivensToModelSurface.
1942
+ const querySurfaceGivens = this.filterGivensToModelSurface(givens);
1322
1943
  try {
1323
1944
  rowLimit = resolveModelQueryRowLimit(
1324
- (await runnable.getPreparedResult({ givens, buildManifest }))
1325
- .resultExplore.limit,
1945
+ (
1946
+ await runnable.getPreparedResult({
1947
+ givens: querySurfaceGivens,
1948
+ buildManifest,
1949
+ })
1950
+ ).resultExplore.limit,
1326
1951
  { defaultLimit: getDefaultQueryRowLimit(), maxRows },
1327
1952
  );
1328
1953
  executionTime = performance.now() - startTime;
1329
1954
 
1330
1955
  queryResults = await runnable.run({
1331
1956
  rowLimit,
1332
- givens,
1957
+ givens: querySurfaceGivens,
1333
1958
  abortSignal,
1334
1959
  buildManifest,
1335
1960
  });
@@ -1552,16 +2177,13 @@ export class Model {
1552
2177
 
1553
2178
  // Authorize gate — only cells that actually run a query touch data, so
1554
2179
  // gate exactly those (a source-def / import cell has no runnable and
1555
- // accesses nothing). Resolve the source from the COMPILED cell query
1556
- // (authoritative survives `run: <named query>` cells the text regex
1557
- // misses); assertAuthorized applies the source's gate, or the model-wide
1558
- // file-level gate for an unknown/inline source. Before the execution try
1559
- // below so AccessDeniedError stays a 403; independent of bypassFilters.
2180
+ // accesses nothing). Gates the COMPILED cell query's own source (the
2181
+ // model-wide file-level gate for an unknown/inline source) PLUS every
2182
+ // source reached transitively via join_* see
2183
+ // assertAuthorizedForAllSources. Before the execution try below so
2184
+ // AccessDeniedError stays a 403; independent of bypassFilters.
1560
2185
  if (cell.runnable) {
1561
- const authorizeSource = await this.resolveAuthorizeSourceFromRunnable(
1562
- cell.runnable,
1563
- );
1564
- await this.assertAuthorized(authorizeSource, givens ?? {});
2186
+ await this.assertAuthorizedForAllSources(cell.runnable, givens ?? {});
1565
2187
  }
1566
2188
 
1567
2189
  // For code cells, execute the runnable if available
@@ -1599,10 +2221,13 @@ export class Model {
1599
2221
  // Per-query freshness gate (see getQueryResults): the same
1600
2222
  // freshness-filtered manifest gates notebook-cell queries.
1601
2223
  const buildManifest = this.resolveFreshBuildManifest();
2224
+ // See getQueryResults / filterGivensToModelSurface: the gate
2225
+ // above already saw the full unfiltered givens.
2226
+ const cellSurfaceGivens = this.filterGivensToModelSurface(givens);
1602
2227
  const rowLimit = resolveModelQueryRowLimit(
1603
2228
  (
1604
2229
  await runnableToExecute.getPreparedResult({
1605
- givens,
2230
+ givens: cellSurfaceGivens,
1606
2231
  buildManifest,
1607
2232
  })
1608
2233
  ).resultExplore.limit,
@@ -1613,7 +2238,7 @@ export class Model {
1613
2238
  );
1614
2239
  const result = await runnableToExecute.run({
1615
2240
  rowLimit,
1616
- givens,
2241
+ givens: cellSurfaceGivens,
1617
2242
  abortSignal,
1618
2243
  buildManifest,
1619
2244
  });