@malloy-publisher/server 0.0.205 → 0.0.207

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/dist/app/api-doc.yaml +394 -395
  2. package/dist/app/assets/{EnvironmentPage-CAge6UHD.js → EnvironmentPage-BScgHmkw.js} +1 -1
  3. package/dist/app/assets/{HomePage-DhTe8qpa.js → HomePage-CGedji_w.js} +1 -1
  4. package/dist/app/assets/{MainPage-CeTxxGex.js → MainPage-DWfF4jXW.js} +2 -2
  5. package/dist/app/assets/{MaterializationsPage-CpDHB70t.js → MaterializationsPage-B9PDlk8c.js} +1 -1
  6. package/dist/app/assets/{ModelPage-D9sSMb75.js → ModelPage-BiNOgK_e.js} +1 -1
  7. package/dist/app/assets/{PackagePage-LRqQWrFY.js → PackagePage-DAN5V7gu.js} +1 -1
  8. package/dist/app/assets/{RouteError-xT6kuCNw.js → RouteError-CEnIzuKN.js} +1 -1
  9. package/dist/app/assets/{WorkbookPage-DsIh9svZ.js → WorkbookPage-gA1ceqHP.js} +1 -1
  10. package/dist/app/assets/{core-C2sQrwVu.es-Bjem0hym.js → core-AOmIKwkc.es-Dclu1Fga.js} +1 -1
  11. package/dist/app/assets/{index-BdOZDcce.js → index-DGGe8UpP.js} +1 -1
  12. package/dist/app/assets/{index-RX3QOTde.js → index-DtlPzNxc.js} +127 -127
  13. package/dist/app/assets/{index-DHHAcY5o.js → index-uu6UpHd2.js} +1 -1
  14. package/dist/app/assets/{index.umd-D2WH3D-f.js → index.umd-DDq93YX4.js} +1 -1
  15. package/dist/app/index.html +1 -1
  16. package/dist/instrumentation.mjs +18 -8
  17. package/dist/package_load_worker.mjs +19 -2
  18. package/dist/server.mjs +1272 -1299
  19. package/package.json +1 -1
  20. package/src/constants.ts +12 -0
  21. package/src/controller/materialization.controller.ts +79 -35
  22. package/src/controller/package.controller.spec.ts +179 -0
  23. package/src/controller/package.controller.ts +60 -73
  24. package/src/dto/package.dto.ts +16 -1
  25. package/src/errors.spec.ts +12 -0
  26. package/src/errors.ts +18 -0
  27. package/src/health.spec.ts +34 -1
  28. package/src/instrumentation.ts +33 -17
  29. package/src/materialization_metrics.ts +121 -0
  30. package/src/package_load/package_load_pool.ts +7 -1
  31. package/src/package_load/package_load_worker.ts +44 -4
  32. package/src/package_load/protocol.ts +7 -1
  33. package/src/server-old.ts +7 -149
  34. package/src/server.ts +9 -188
  35. package/src/service/authorize_integration.spec.ts +67 -0
  36. package/src/service/environment.ts +270 -12
  37. package/src/service/environment_store.spec.ts +0 -81
  38. package/src/service/environment_store.ts +0 -23
  39. package/src/service/explore_visibility.spec.ts +434 -0
  40. package/src/service/exports_probe.spec.ts +107 -0
  41. package/src/service/manifest_loader.spec.ts +99 -0
  42. package/src/service/manifest_loader.ts +95 -0
  43. package/src/service/materialization_service.spec.ts +584 -500
  44. package/src/service/materialization_service.ts +839 -657
  45. package/src/service/model.ts +419 -15
  46. package/src/service/package.spec.ts +14 -2
  47. package/src/service/package.ts +339 -29
  48. package/src/service/package_rollback.spec.ts +190 -0
  49. package/src/service/package_worker_path.spec.ts +223 -0
  50. package/src/service/query_boundary.spec.ts +470 -0
  51. package/src/storage/DatabaseInterface.ts +35 -57
  52. package/src/storage/StorageManager.mock.ts +0 -9
  53. package/src/storage/StorageManager.ts +7 -290
  54. package/src/storage/duckdb/DuckDBRepository.ts +2 -35
  55. package/src/storage/duckdb/MaterializationRepository.ts +52 -27
  56. package/src/storage/duckdb/schema.ts +4 -20
  57. package/tests/integration/materialization/manifest_binding.integration.spec.ts +194 -0
  58. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +369 -264
  59. package/tests/unit/duckdb/legacy_schema_migration.test.ts +7 -4
  60. package/src/controller/manifest.controller.ts +0 -38
  61. package/src/service/manifest_service.spec.ts +0 -206
  62. package/src/service/manifest_service.ts +0 -117
  63. package/src/service/materialized_table_gc.spec.ts +0 -384
  64. package/src/service/materialized_table_gc.ts +0 -231
  65. package/src/storage/duckdb/DuckDBManifestStore.ts +0 -70
  66. package/src/storage/duckdb/ManifestRepository.ts +0 -120
  67. package/src/storage/duckdb/manifest_store.spec.ts +0 -133
  68. package/src/storage/ducklake/DuckLakeManifestStore.ts +0 -155
  69. package/tests/unit/storage/StorageManager.test.ts +0 -166
@@ -48,8 +48,6 @@ type PackageConnectionInput =
48
48
  | Map<string, Connection>
49
49
  | (() => MalloyConfig);
50
50
 
51
- const ENABLE_LIST_MODEL_COMPILATION = true;
52
-
53
51
  export class Package {
54
52
  private environmentName: string;
55
53
  private packageName: string;
@@ -58,6 +56,17 @@ export class Package {
58
56
  private models: Map<string, Model> = new Map();
59
57
  private packagePath: string;
60
58
  private malloyConfig: MalloyConfig;
59
+ // Build-manifest binding state (Malloy Persistence v0). When bound, these
60
+ // entries (buildId -> { tableName }) are what served queries use to route
61
+ // persist sources to their materialized physical tables; they are also reused
62
+ // by the /compile preview so previewed SQL matches executed SQL. Surfaced on
63
+ // /status (via getPackageMetadata) so the control plane can confirm a worker
64
+ // actually bound the distributed manifest instead of inferring it from logs.
65
+ private buildManifestEntries: BuildManifest["entries"] | undefined;
66
+ private manifestBindingStatus: "unbound" | "bound" | "live_fallback" =
67
+ "unbound";
68
+ private manifestEntryCount = 0;
69
+ private boundManifestUri: string | null = null;
61
70
  private static meter = metrics.getMeter("publisher");
62
71
  private static packageLoadHistogram = this.meter.createHistogram(
63
72
  "malloy_package_load_duration",
@@ -83,6 +92,64 @@ export class Package {
83
92
  this.databases = databases;
84
93
  this.models = models;
85
94
  this.malloyConfig = malloyConfig;
95
+ this.applyDiscoveryPolicyToModels();
96
+ this.applyQueryBoundaryToModels();
97
+ }
98
+
99
+ /**
100
+ * Push the discovery-curation policy down onto each Model. Curation (file
101
+ * listing via `explores` and within-file `export {}` filtering) is enabled
102
+ * only when `explores` is declared in publisher.json — absent/empty
103
+ * `explores` preserves legacy listings. Re-derived on reload and metadata
104
+ * PATCH (the inputs can change there).
105
+ */
106
+ /** True when the package opts into curated discovery via a non-empty
107
+ * `explores`. Single source of truth so the curation/boundary/listing
108
+ * derivations can't drift out of sync. */
109
+ private exploresDeclared(): boolean {
110
+ const explores = this.packageMetadata.explores;
111
+ return !!(explores && explores.length > 0);
112
+ }
113
+
114
+ /** The declared explore set, or null when discovery is uncurated. */
115
+ private exploreSet(): Set<string> | null {
116
+ const explores = this.packageMetadata.explores;
117
+ return explores && explores.length > 0 ? new Set(explores) : null;
118
+ }
119
+
120
+ private applyDiscoveryPolicyToModels(): void {
121
+ const curationEnabled = this.exploresDeclared();
122
+ for (const model of this.models.values()) {
123
+ model.setDiscoveryCuration(curationEnabled);
124
+ }
125
+ }
126
+
127
+ /**
128
+ * Push the package-level query-boundary policy down onto each Model so the
129
+ * query chokepoints can enforce it without a back-reference to the Package:
130
+ * `Model.getQueryResults` (the HTTP query route and the MCP tool) and the
131
+ * `/compile` path (via `assertQueryBoundaryForRunnable`). Derived once here
132
+ * (and on reload) rather than per query: the policy only changes when the
133
+ * manifest is (re)read.
134
+ *
135
+ * Policy: queryable == discoverable. The boundary is inert unless `explores`
136
+ * is declared (no curated surface ⇒ nothing to restrict) AND
137
+ * `queryableSources` is "declared" (the default; "all" decouples the axes).
138
+ * When active, a model file is a query entry point only if it is listed in
139
+ * `explores`; within-file curation (`export {}`) is read off each Model.
140
+ */
141
+ private applyQueryBoundaryToModels(): void {
142
+ const exploresDeclared = this.exploresDeclared();
143
+ const exploreSet = this.exploreSet();
144
+ const mode =
145
+ this.packageMetadata.queryableSources === "all" ? "all" : "declared";
146
+ for (const [modelPath, model] of this.models) {
147
+ model.setQueryBoundary({
148
+ mode,
149
+ exploresDeclared,
150
+ isQueryEntryPoint: exploreSet ? exploreSet.has(modelPath) : true,
151
+ });
152
+ }
86
153
  }
87
154
 
88
155
  static async create(
@@ -251,6 +318,9 @@ export class Package {
251
318
  name: outcome.packageMetadata.name,
252
319
  description: outcome.packageMetadata.description,
253
320
  resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}`,
321
+ explores: outcome.packageMetadata.explores,
322
+ queryableSources: outcome.packageMetadata.queryableSources,
323
+ manifestLocation: outcome.packageMetadata.manifestLocation ?? null,
254
324
  };
255
325
 
256
326
  // Build live `Model`s from worker output. Any per-model compile
@@ -272,10 +342,18 @@ export class Package {
272
342
  // cleans the package directory.
273
343
  throw err;
274
344
  }
275
- models.set(
276
- sm.modelPath,
277
- Model.fromSerialized(packageName, packagePath, malloyConfig, sm),
345
+ const model = Model.fromSerialized(
346
+ packageName,
347
+ packagePath,
348
+ malloyConfig,
349
+ sm,
278
350
  );
351
+ // Validate renderer tags on the main thread (the renderer is too heavy
352
+ // to load inside the pure-CPU package-load worker). A misconfigured tag
353
+ // throws a ModelCompilationError (424), aborting the whole load like any
354
+ // other per-model compile error above.
355
+ await model.validateRenderTags();
356
+ models.set(sm.modelPath, model);
279
357
  }
280
358
 
281
359
  const endTime = performance.now();
@@ -289,7 +367,7 @@ export class Package {
289
367
  duration: formatDuration(executionTime),
290
368
  });
291
369
 
292
- return new Package(
370
+ const pkg = new Package(
293
371
  environmentName,
294
372
  packageName,
295
373
  packagePath,
@@ -298,6 +376,21 @@ export class Package {
298
376
  models,
299
377
  malloyConfig,
300
378
  );
379
+
380
+ // Fail-safe at load: a bad explores entry doesn't fail the package
381
+ // (its models still load and listModels hides the unmatched entry — it
382
+ // never falls back to listing everything). Warn so the misconfig is
383
+ // visible; the publish path rejects it outright (see package.controller).
384
+ const invalidMsg = pkg.formatInvalidExplores();
385
+ if (invalidMsg) {
386
+ logger.warn(`Package ${packageName} has invalid explores`, {
387
+ packageName,
388
+ detail: invalidMsg,
389
+ });
390
+ }
391
+ pkg.logEmptyDiscoveryWarnings();
392
+
393
+ return pkg;
301
394
  }
302
395
 
303
396
  public getPackageName(): string {
@@ -305,7 +398,171 @@ export class Package {
305
398
  }
306
399
 
307
400
  public getPackageMetadata(): ApiPackage {
308
- return this.packageMetadata;
401
+ // Overlay the server-computed fields onto the stored metadata: the
402
+ // explores misconfig warnings (loading is fail-safe — the package still
403
+ // serves with the bad entry hidden — so this is the only non-log signal
404
+ // that it's broken) and the manifest-binding state (so /status reflects
405
+ // whether persist sources are actually routed to materialized tables).
406
+ // Always returns a copy so these overlays never mutate the stored
407
+ // metadata; the binding fields are authoritative from the private state.
408
+ //
409
+ // `name` is the registered package name (the environment's identity for
410
+ // this package), not the value read from the package's own manifest —
411
+ // those can differ (e.g. a package installed under a different name than
412
+ // its publisher.json declares). `listPackages` already overrides it to
413
+ // the registered name; surfacing it here keeps the single-package GET
414
+ // consistent without relying on a returned-reference mutation.
415
+ const metadata: ApiPackage = {
416
+ ...this.packageMetadata,
417
+ name: this.packageName,
418
+ manifestBindingStatus: this.manifestBindingStatus,
419
+ manifestEntryCount: this.manifestEntryCount,
420
+ boundManifestUri: this.boundManifestUri,
421
+ };
422
+ const warnings = this.exploreWarnings();
423
+ if (warnings.length > 0) {
424
+ metadata.exploresWarnings = warnings;
425
+ }
426
+ return metadata;
427
+ }
428
+
429
+ /**
430
+ * The currently-bound build-manifest entries (buildId -> { tableName }), or
431
+ * undefined when the package is serving live. Reused by the /compile preview
432
+ * so previewed SQL gets the same persist-source -> physical-table routing as
433
+ * execution.
434
+ */
435
+ public getBuildManifestEntries(): BuildManifest["entries"] | undefined {
436
+ return this.buildManifestEntries;
437
+ }
438
+
439
+ /**
440
+ * Record the URI whose manifest is currently bound to the served models. May
441
+ * differ from `manifestLocation` after an in-memory auto-load following a
442
+ * materialization build (no URI), in which case it stays null.
443
+ */
444
+ public setBoundManifestUri(uri: string | null): void {
445
+ this.boundManifestUri = uri;
446
+ }
447
+
448
+ /**
449
+ * Mark that a configured `manifestLocation` could not be fetched/bound, so
450
+ * the package is serving live despite intending to be materialized-routed.
451
+ */
452
+ public markManifestBindFailed(): void {
453
+ this.manifestBindingStatus = "live_fallback";
454
+ }
455
+
456
+ /**
457
+ * Store the entries applied by the latest (re)bind for reuse (/compile) and
458
+ * observability (/status). An empty map means the manifest was cleared, so
459
+ * the package reverts to live (unbound).
460
+ */
461
+ private recordManifestBinding(
462
+ buildManifest: BuildManifest["entries"],
463
+ ): void {
464
+ const count = Object.keys(buildManifest).length;
465
+ this.buildManifestEntries = count > 0 ? buildManifest : undefined;
466
+ this.manifestEntryCount = count;
467
+ this.manifestBindingStatus = count > 0 ? "bound" : "unbound";
468
+ if (count === 0) {
469
+ this.boundManifestUri = null;
470
+ }
471
+ }
472
+
473
+ /**
474
+ * Declared `explores` (publisher.json) that don't resolve to a real
475
+ * `.malloy` model in this package, each with an actionable reason. Empty
476
+ * when explores is absent/empty or every entry resolves.
477
+ *
478
+ * The listing already fails safe — a non-resolving entry matches no model in
479
+ * `listModels`, so it hides rather than exposes. This surfaces *why*, so the
480
+ * load path can warn and the publish path can reject (see package.controller).
481
+ */
482
+ public getInvalidExplores(
483
+ exploresOverride?: string[],
484
+ ): { entry: string; reason: string }[] {
485
+ const declared = exploresOverride ?? this.packageMetadata.explores;
486
+ if (!declared || declared.length === 0) return [];
487
+ const malloyModels = new Set(
488
+ Array.from(this.models.keys()).filter((p) =>
489
+ p.endsWith(MODEL_FILE_SUFFIX),
490
+ ),
491
+ );
492
+ const problems: { entry: string; reason: string }[] = [];
493
+ for (const entry of declared) {
494
+ if (entry.endsWith(NOTEBOOK_FILE_SUFFIX)) {
495
+ problems.push({
496
+ entry,
497
+ reason:
498
+ `notebooks are always public and cannot be explores. ` +
499
+ `Fix: remove it, and list a ${MODEL_FILE_SUFFIX} model file instead.`,
500
+ });
501
+ } else if (!malloyModels.has(entry)) {
502
+ problems.push({
503
+ entry,
504
+ reason:
505
+ `file not found in the package. Fix: list a ${MODEL_FILE_SUFFIX} ` +
506
+ `file relative to the package root (e.g. "index.malloy").`,
507
+ });
508
+ }
509
+ }
510
+ return problems;
511
+ }
512
+
513
+ /** One actionable message per invalid entry (empty when all resolve). */
514
+ public exploreWarnings(exploresOverride?: string[]): string[] {
515
+ return this.getInvalidExplores(exploresOverride).map(
516
+ (p) =>
517
+ `Invalid explores entry '${p.entry}' in ${PACKAGE_MANIFEST_NAME}: ${p.reason}`,
518
+ );
519
+ }
520
+
521
+ /**
522
+ * The {@link exploreWarnings} joined into one string, or "" if none.
523
+ * Newline-separated so multiple invalid entries stay one-per-line in the
524
+ * 400 message rather than running together.
525
+ */
526
+ public formatInvalidExplores(exploresOverride?: string[]): string {
527
+ return this.exploreWarnings(exploresOverride).join("\n");
528
+ }
529
+
530
+ /**
531
+ * One message per LISTED model whose discovery surface is empty because it
532
+ * is import-only (imports other files, declares/re-exports nothing). Such a
533
+ * model renders a blank page, which reads as broken; the fix is an explicit
534
+ * re-export. Log-only (see loadViaWorker/reloadAllModels) — deliberately
535
+ * NOT part of exploreWarnings, which is strict-at-publish: import-only
536
+ * files are a legitimate pattern and must not block a publish. Hidden
537
+ * (non-listed) models are skipped — nobody browses them, so an empty
538
+ * surface there is just normal plumbing.
539
+ */
540
+ public emptyDiscoveryWarnings(): string[] {
541
+ const exploreSet = this.exploreSet();
542
+ const warnings: string[] = [];
543
+ for (const [modelPath, model] of this.models) {
544
+ if (!modelPath.endsWith(MODEL_FILE_SUFFIX)) continue;
545
+ if (exploreSet && !exploreSet.has(modelPath)) continue;
546
+ if (model.hasEmptyDiscoverySurface()) {
547
+ warnings.push(
548
+ `Model "${modelPath}" is listed but exposes nothing: it only ` +
549
+ `imports other files and re-exports none of their sources. ` +
550
+ `Add e.g. 'export { source_name }' to surface sources on ` +
551
+ `this model.`,
552
+ );
553
+ }
554
+ }
555
+ return warnings;
556
+ }
557
+
558
+ /** Log {@link emptyDiscoveryWarnings}; shared by load and reload. */
559
+ private logEmptyDiscoveryWarnings(): void {
560
+ for (const warning of this.emptyDiscoveryWarnings()) {
561
+ logger.warn(`Package ${this.packageName} has a blank-looking model`, {
562
+ packageName: this.packageName,
563
+ detail: warning,
564
+ });
565
+ }
309
566
  }
310
567
 
311
568
  public listDatabases(): ApiDatabase[] {
@@ -404,18 +661,67 @@ export class Package {
404
661
  ),
405
662
  );
406
663
  } else {
407
- nextModels.set(
408
- sm.modelPath,
409
- Model.fromSerialized(
410
- this.packageName,
411
- this.packagePath,
412
- this.malloyConfig,
413
- sm,
414
- ),
664
+ const model = Model.fromSerialized(
665
+ this.packageName,
666
+ this.packagePath,
667
+ this.malloyConfig,
668
+ sm,
669
+ { buildManifest },
415
670
  );
671
+ // Validate renderer tags here too (loadViaWorker does it for the
672
+ // create path). Reload keeps per-model placeholders rather than
673
+ // aborting the whole package, so a render-tag error is recorded as
674
+ // this model's compilationError instead of thrown.
675
+ try {
676
+ await model.validateRenderTags();
677
+ nextModels.set(sm.modelPath, model);
678
+ } catch (renderErr) {
679
+ const err =
680
+ renderErr instanceof Error
681
+ ? renderErr
682
+ : new Error(String(renderErr));
683
+ logger.warn("Render-tag validation failed during reload", {
684
+ packageName: this.packageName,
685
+ modelPath: sm.modelPath,
686
+ error: err.message,
687
+ });
688
+ nextModels.set(
689
+ sm.modelPath,
690
+ Model.fromCompilationError(
691
+ this.packageName,
692
+ sm.modelPath,
693
+ sm.modelType,
694
+ err,
695
+ ),
696
+ );
697
+ }
416
698
  }
417
699
  }
418
700
  this.models = nextModels;
701
+ // A reload re-reads publisher.json in the worker; pick up any change to
702
+ // the explore set and query-boundary mode so listModels()/the gate
703
+ // reflect edited explores without a full Package.create.
704
+ this.packageMetadata.explores = outcome.packageMetadata.explores;
705
+ this.packageMetadata.queryableSources =
706
+ outcome.packageMetadata.queryableSources;
707
+ this.packageMetadata.manifestLocation =
708
+ outcome.packageMetadata.manifestLocation ?? null;
709
+ this.applyDiscoveryPolicyToModels();
710
+ this.applyQueryBoundaryToModels();
711
+ // Remember what we just bound so /compile can route identically and
712
+ // /status can report the binding. An empty map reverts to live (unbound).
713
+ this.recordManifestBinding(buildManifest);
714
+ // Re-run the fail-safe warning against the refreshed model set: an edit
715
+ // to publisher.json that introduces a bad entry should surface in the
716
+ // logs on reload too, not only at initial load (loadViaWorker).
717
+ const invalidMsg = this.formatInvalidExplores();
718
+ if (invalidMsg) {
719
+ logger.warn(`Package ${this.packageName} has invalid explores`, {
720
+ packageName: this.packageName,
721
+ detail: invalidMsg,
722
+ });
723
+ }
724
+ this.logEmptyDiscoveryWarnings();
419
725
  }
420
726
 
421
727
  public async getModelFileText(modelPath: string): Promise<string> {
@@ -427,22 +733,27 @@ export class Package {
427
733
  }
428
734
 
429
735
  public async listModels(): Promise<ApiModel[]> {
736
+ // When `explores` is declared in publisher.json, only those models
737
+ // form the public surface; every other .malloy file still compiles for
738
+ // import/join resolution but is hidden from the listing. Absent/empty →
739
+ // every model is listed (backward-compatible default). Notebooks are
740
+ // unaffected (see listNotebooks) — they are always public.
741
+ const exploreSet = this.exploreSet();
430
742
  const values = await Promise.all(
431
743
  Array.from(this.models.keys())
432
744
  .filter((modelPath) => {
433
- return modelPath.endsWith(MODEL_FILE_SUFFIX);
745
+ if (!modelPath.endsWith(MODEL_FILE_SUFFIX)) return false;
746
+ return exploreSet ? exploreSet.has(modelPath) : true;
434
747
  })
435
748
  .map(async (modelPath) => {
436
749
  let error: string | undefined;
437
- if (ENABLE_LIST_MODEL_COMPILATION) {
438
- try {
439
- await this.models.get(modelPath)?.getModel();
440
- } catch (modelError) {
441
- error =
442
- modelError instanceof Error
443
- ? modelError.message
444
- : undefined;
445
- }
750
+ try {
751
+ await this.models.get(modelPath)?.getModel();
752
+ } catch (modelError) {
753
+ error =
754
+ modelError instanceof Error
755
+ ? modelError.message
756
+ : undefined;
446
757
  }
447
758
  return {
448
759
  environmentName: this.environmentName,
@@ -462,10 +773,7 @@ export class Package {
462
773
  return modelPath.endsWith(NOTEBOOK_FILE_SUFFIX);
463
774
  })
464
775
  .map(async (modelPath) => {
465
- let error: Error | undefined;
466
- if (ENABLE_LIST_MODEL_COMPILATION) {
467
- error = this.models.get(modelPath)?.getNotebookError();
468
- }
776
+ const error = this.models.get(modelPath)?.getNotebookError();
469
777
  return {
470
778
  environmentName: this.environmentName,
471
779
  packageName: this.packageName,
@@ -632,5 +940,7 @@ export class Package {
632
940
 
633
941
  public setPackageMetadata(packageMetadata: ApiPackage) {
634
942
  this.packageMetadata = packageMetadata;
943
+ this.applyDiscoveryPolicyToModels();
944
+ this.applyQueryBoundaryToModels();
635
945
  }
636
946
  }
@@ -0,0 +1,190 @@
1
+ import { afterEach, beforeEach, describe, expect, it } from "bun:test";
2
+ import * as fs from "fs/promises";
3
+ import * as os from "os";
4
+ import * as path from "path";
5
+
6
+ import { BadRequestError } from "../errors";
7
+ import { Environment } from "./environment";
8
+
9
+ /**
10
+ * Regression tests for explores-validation ROLLBACK, guarding against the two
11
+ * data-loss bugs a code review found:
12
+ *
13
+ * 1. No-location create rollback must NOT delete a pre-existing user
14
+ * directory. The controller now `unloadPackage`s (evict from memory, keep
15
+ * files) instead of `deletePackage` (which renames + rm's the tree).
16
+ *
17
+ * 2. A location update/reinstall with invalid explores must roll back to the
18
+ * PREVIOUS tree rather than swapping the rejected one in and 400-ing after.
19
+ * Validation now happens inside `installPackage`'s swap window.
20
+ *
21
+ * Both run against a real `Environment` + real `Package.create` over temp dirs.
22
+ */
23
+ describe("explores-validation rollback (real Environment)", () => {
24
+ let rootDir: string;
25
+ let envPath: string;
26
+
27
+ const GOOD_MODEL = `source: ones is duckdb.sql("SELECT 1 as x")\n`;
28
+ const BAD_MODEL = `source: twos is duckdb.sql("SELECT 2 as y")\n`;
29
+
30
+ async function writePackageDir(
31
+ dir: string,
32
+ manifest: Record<string, unknown>,
33
+ model: string,
34
+ ): Promise<void> {
35
+ await fs.mkdir(dir, { recursive: true });
36
+ await fs.writeFile(
37
+ path.join(dir, "publisher.json"),
38
+ JSON.stringify({
39
+ name: "pkg",
40
+ description: "rollback fixture",
41
+ ...manifest,
42
+ }),
43
+ );
44
+ await fs.writeFile(path.join(dir, "model.malloy"), model);
45
+ }
46
+
47
+ async function copyDir(src: string, dst: string): Promise<void> {
48
+ await fs.mkdir(dst, { recursive: true });
49
+ await fs.cp(src, dst, { recursive: true });
50
+ }
51
+
52
+ beforeEach(async () => {
53
+ rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-rollback-"));
54
+ envPath = path.join(rootDir, "env");
55
+ await fs.mkdir(envPath, { recursive: true });
56
+ });
57
+
58
+ afterEach(async () => {
59
+ await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
60
+ });
61
+
62
+ it("Fix 1: unloadPackage evicts from memory but keeps the on-disk directory", async () => {
63
+ // Mimics the no-location create path: a pre-existing package directory is
64
+ // registered, then rolled back. unloadPackage must not touch the files.
65
+ const env = await Environment.create("testEnv", envPath, []);
66
+ const pkgDir = path.join(envPath, "pkg");
67
+ await writePackageDir(pkgDir, {}, GOOD_MODEL);
68
+
69
+ await env.addPackage("pkg");
70
+ expect(env.getPackageStatus("pkg")).toBeDefined();
71
+
72
+ await env.unloadPackage("pkg");
73
+
74
+ // Evicted from memory...
75
+ expect(env.getPackageStatus("pkg")).toBeUndefined();
76
+ // ...but the user's files are untouched.
77
+ const modelText = await fs.readFile(
78
+ path.join(pkgDir, "model.malloy"),
79
+ "utf-8",
80
+ );
81
+ expect(modelText).toBe(GOOD_MODEL);
82
+ });
83
+
84
+ it("Fix 2: a rejected location reinstall restores the previous tree", async () => {
85
+ const env = await Environment.create("testEnv", envPath, []);
86
+
87
+ const goodFixture = path.join(rootDir, "good");
88
+ const badFixture = path.join(rootDir, "bad");
89
+ await writePackageDir(goodFixture, {}, GOOD_MODEL);
90
+ await writePackageDir(
91
+ badFixture,
92
+ { explores: ["missing.malloy"] },
93
+ BAD_MODEL,
94
+ );
95
+
96
+ // First install: the good tree is served.
97
+ await env.installPackage("pkg", (stagingPath) =>
98
+ copyDir(goodFixture, stagingPath),
99
+ );
100
+ expect(await env.getModelFileText("pkg", "model.malloy")).toBe(
101
+ GOOD_MODEL,
102
+ );
103
+
104
+ // Reinstall with an invalid-explores tree, validated inside the swap.
105
+ await expect(
106
+ env.installPackage(
107
+ "pkg",
108
+ (stagingPath) => copyDir(badFixture, stagingPath),
109
+ (pkg) => pkg.formatInvalidExplores(),
110
+ ),
111
+ ).rejects.toBeInstanceOf(BadRequestError);
112
+
113
+ // The previous (good) tree is still the one on disk — the rejected tree
114
+ // was rolled back, not swapped in.
115
+ expect(await env.getModelFileText("pkg", "model.malloy")).toBe(
116
+ GOOD_MODEL,
117
+ );
118
+ });
119
+
120
+ it("a valid location reinstall still succeeds (no false rollback)", async () => {
121
+ const env = await Environment.create("testEnv", envPath, []);
122
+ const goodFixture = path.join(rootDir, "good");
123
+ const nextFixture = path.join(rootDir, "next");
124
+ await writePackageDir(goodFixture, {}, GOOD_MODEL);
125
+ await writePackageDir(
126
+ nextFixture,
127
+ { explores: ["model.malloy"] },
128
+ BAD_MODEL,
129
+ );
130
+
131
+ await env.installPackage("pkg", (stagingPath) =>
132
+ copyDir(goodFixture, stagingPath),
133
+ );
134
+ await env.installPackage(
135
+ "pkg",
136
+ (stagingPath) => copyDir(nextFixture, stagingPath),
137
+ (pkg) => pkg.formatInvalidExplores(),
138
+ );
139
+ expect(await env.getModelFileText("pkg", "model.malloy")).toBe(BAD_MODEL);
140
+ });
141
+
142
+ it("updatePackage normalizes a ./-prefixed body explores (no false 400, persists normalized)", async () => {
143
+ // API-body explores must go through the same normalization the worker
144
+ // applies to on-disk explores, so `./model.malloy` validates and persists
145
+ // as `model.malloy` rather than being rejected with a misleading 404.
146
+ const env = await Environment.create("testEnv", envPath, []);
147
+ const pkgDir = path.join(envPath, "pkg");
148
+ await writePackageDir(pkgDir, {}, GOOD_MODEL);
149
+ await env.addPackage("pkg");
150
+
151
+ const updated = await env.updatePackage("pkg", {
152
+ name: "pkg",
153
+ explores: ["./model.malloy"],
154
+ });
155
+
156
+ // Accepted (no BadRequestError) and stored in normalized form...
157
+ expect(updated.explores).toEqual(["model.malloy"]);
158
+ // ...both in memory and on disk.
159
+ const manifest = JSON.parse(
160
+ await fs.readFile(path.join(pkgDir, "publisher.json"), "utf-8"),
161
+ );
162
+ expect(manifest.explores).toEqual(["model.malloy"]);
163
+ });
164
+
165
+ it("compileSource rejects a notebook path; a model path still compiles", async () => {
166
+ // /compile appends source to the target MODEL's content for context; a
167
+ // notebook isn't a valid target and must be rejected up front (not left to
168
+ // a confusing downstream parse error).
169
+ const env = await Environment.create("testEnv", envPath, []);
170
+ const pkgDir = path.join(envPath, "pkg");
171
+ await writePackageDir(pkgDir, {}, GOOD_MODEL);
172
+ await fs.writeFile(
173
+ path.join(pkgDir, "report.malloynb"),
174
+ `>>>markdown\n# Report\n`,
175
+ );
176
+ await env.addPackage("pkg");
177
+
178
+ await expect(
179
+ env.compileSource("pkg", "report.malloynb", "run: ones"),
180
+ ).rejects.toBeInstanceOf(BadRequestError);
181
+
182
+ // The .malloy model still compiles ad-hoc source (no regression).
183
+ const { problems } = await env.compileSource(
184
+ "pkg",
185
+ "model.malloy",
186
+ "run: ones -> { select: x }",
187
+ );
188
+ expect(problems.some((p) => p.severity === "error")).toBe(false);
189
+ });
190
+ });