@malloy-publisher/server 0.0.204 → 0.0.206

package/src/service/package.ts

@@ -83,6 +83,64 @@ export class Package {
       this.databases = databases;
       this.models = models;
       this.malloyConfig = malloyConfig;
+      this.applyDiscoveryPolicyToModels();
+      this.applyQueryBoundaryToModels();
+   }
+
+   /**
+    * Push the discovery-curation policy down onto each Model. Curation (file
+    * listing via `explores` and within-file `export {}` filtering) is enabled
+    * only when `explores` is declared in publisher.json — absent/empty
+    * `explores` preserves legacy listings. Re-derived on reload and metadata
+    * PATCH (the inputs can change there).
+    */
+   /** True when the package opts into curated discovery via a non-empty
+    *  `explores`. Single source of truth so the curation/boundary/listing
+    *  derivations can't drift out of sync. */
+   private exploresDeclared(): boolean {
+      const explores = this.packageMetadata.explores;
+      return !!(explores && explores.length > 0);
+   }
+
+   /** The declared explore set, or null when discovery is uncurated. */
+   private exploreSet(): Set<string> | null {
+      const explores = this.packageMetadata.explores;
+      return explores && explores.length > 0 ? new Set(explores) : null;
+   }
+
+   private applyDiscoveryPolicyToModels(): void {
+      const curationEnabled = this.exploresDeclared();
+      for (const model of this.models.values()) {
+         model.setDiscoveryCuration(curationEnabled);
+      }
+   }
+
+   /**
+    * Push the package-level query-boundary policy down onto each Model so the
+    * query chokepoints can enforce it without a back-reference to the Package:
+    * `Model.getQueryResults` (the HTTP query route and the MCP tool) and the
+    * `/compile` path (via `assertQueryBoundaryForRunnable`). Derived once here
+    * (and on reload) rather than per query: the policy only changes when the
+    * manifest is (re)read.
+    *
+    * Policy: queryable == discoverable. The boundary is inert unless `explores`
+    * is declared (no curated surface ⇒ nothing to restrict) AND
+    * `queryableSources` is "declared" (the default; "all" decouples the axes).
+    * When active, a model file is a query entry point only if it is listed in
+    * `explores`; within-file curation (`export {}`) is read off each Model.
+    */
+   private applyQueryBoundaryToModels(): void {
+      const exploresDeclared = this.exploresDeclared();
+      const exploreSet = this.exploreSet();
+      const mode =
+         this.packageMetadata.queryableSources === "all" ? "all" : "declared";
+      for (const [modelPath, model] of this.models) {
+         model.setQueryBoundary({
+            mode,
+            exploresDeclared,
+            isQueryEntryPoint: exploreSet ? exploreSet.has(modelPath) : true,
+         });
+      }
    }
 
    static async create(
@@ -138,13 +196,24 @@ export class Package {
             malloy_package_name: packageName,
             status,
          });
-         // Clean up package directory on failure
+         // Clean up the package directory on failure, but NOT when packagePath
+         // is an in-place mount symlink (watch mode). Removing it would unmount
+         // the package, so a transient compile error from a half-typed model
+         // saved mid-edit would brick the package until a restart. The symlink
+         // points at the user's live source, which is left untouched; the next
+         // save recompiles against it.
          try {
-            await fs.rm(packagePath, {
-               recursive: true,
-               force: true,
-            });
-            logger.info(`Cleaned up failed package directory: ${packagePath}`);
+            const stat = await fs.lstat(packagePath).catch(() => null);
+            if (stat?.isSymbolicLink()) {
+               logger.info(
+                  `Skipping cleanup of symlinked package path on failure: ${packagePath}`,
+               );
+            } else {
+               await fs.rm(packagePath, { recursive: true, force: true });
+               logger.info(
+                  `Cleaned up failed package directory: ${packagePath}`,
+               );
+            }
          } catch (cleanupError) {
             logger.warn(`Failed to clean up package directory ${packagePath}`, {
                error: cleanupError,
@@ -240,6 +309,9 @@ export class Package {
          name: outcome.packageMetadata.name,
          description: outcome.packageMetadata.description,
          resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}`,
+         explores: outcome.packageMetadata.explores,
+         queryableSources: outcome.packageMetadata.queryableSources,
+         manifestLocation: outcome.packageMetadata.manifestLocation ?? null,
       };
 
       // Build live `Model`s from worker output. Any per-model compile
@@ -261,10 +333,18 @@ export class Package {
             // cleans the package directory.
             throw err;
          }
-         models.set(
-            sm.modelPath,
-            Model.fromSerialized(packageName, packagePath, malloyConfig, sm),
+         const model = Model.fromSerialized(
+            packageName,
+            packagePath,
+            malloyConfig,
+            sm,
          );
+         // Validate renderer tags on the main thread (the renderer is too heavy
+         // to load inside the pure-CPU package-load worker). A misconfigured tag
+         // throws a ModelCompilationError (424), aborting the whole load like any
+         // other per-model compile error above.
+         await model.validateRenderTags();
+         models.set(sm.modelPath, model);
       }
 
       const endTime = performance.now();
@@ -278,7 +358,7 @@ export class Package {
          duration: formatDuration(executionTime),
       });
 
-      return new Package(
+      const pkg = new Package(
          environmentName,
          packageName,
          packagePath,
@@ -287,6 +367,21 @@ export class Package {
          models,
          malloyConfig,
       );
+
+      // Fail-safe at load: a bad explores entry doesn't fail the package
+      // (its models still load and listModels hides the unmatched entry — it
+      // never falls back to listing everything). Warn so the misconfig is
+      // visible; the publish path rejects it outright (see package.controller).
+      const invalidMsg = pkg.formatInvalidExplores();
+      if (invalidMsg) {
+         logger.warn(`Package ${packageName} has invalid explores`, {
+            packageName,
+            detail: invalidMsg,
+         });
+      }
+      pkg.logEmptyDiscoveryWarnings();
+
+      return pkg;
    }
 
    public getPackageName(): string {
@@ -294,7 +389,109 @@ export class Package {
    }
 
    public getPackageMetadata(): ApiPackage {
-      return this.packageMetadata;
+      // Surface explores misconfig so consumers/UI can show it (loading is
+      // fail-safe — the package still serves with the bad entry hidden — so this
+      // is the only non-log signal that it's broken). Computed fresh against the
+      // current models; absent when everything resolves. Returns a copy in that
+      // case so the added field never mutates the stored metadata.
+      const warnings = this.exploreWarnings();
+      if (warnings.length === 0) return this.packageMetadata;
+      return { ...this.packageMetadata, exploresWarnings: warnings };
+   }
+
+   /**
+    * Declared `explores` (publisher.json) that don't resolve to a real
+    * `.malloy` model in this package, each with an actionable reason. Empty
+    * when explores is absent/empty or every entry resolves.
+    *
+    * The listing already fails safe — a non-resolving entry matches no model in
+    * `listModels`, so it hides rather than exposes. This surfaces *why*, so the
+    * load path can warn and the publish path can reject (see package.controller).
+    */
+   public getInvalidExplores(
+      exploresOverride?: string[],
+   ): { entry: string; reason: string }[] {
+      const declared = exploresOverride ?? this.packageMetadata.explores;
+      if (!declared || declared.length === 0) return [];
+      const malloyModels = new Set(
+         Array.from(this.models.keys()).filter((p) =>
+            p.endsWith(MODEL_FILE_SUFFIX),
+         ),
+      );
+      const problems: { entry: string; reason: string }[] = [];
+      for (const entry of declared) {
+         if (entry.endsWith(NOTEBOOK_FILE_SUFFIX)) {
+            problems.push({
+               entry,
+               reason:
+                  `notebooks are always public and cannot be explores. ` +
+                  `Fix: remove it, and list a ${MODEL_FILE_SUFFIX} model file instead.`,
+            });
+         } else if (!malloyModels.has(entry)) {
+            problems.push({
+               entry,
+               reason:
+                  `file not found in the package. Fix: list a ${MODEL_FILE_SUFFIX} ` +
+                  `file relative to the package root (e.g. "index.malloy").`,
+            });
+         }
+      }
+      return problems;
+   }
+
+   /** One actionable message per invalid entry (empty when all resolve). */
+   public exploreWarnings(exploresOverride?: string[]): string[] {
+      return this.getInvalidExplores(exploresOverride).map(
+         (p) =>
+            `Invalid explores entry '${p.entry}' in ${PACKAGE_MANIFEST_NAME}: ${p.reason}`,
+      );
+   }
+
+   /**
+    * The {@link exploreWarnings} joined into one string, or "" if none.
+    * Newline-separated so multiple invalid entries stay one-per-line in the
+    * 400 message rather than running together.
+    */
+   public formatInvalidExplores(exploresOverride?: string[]): string {
+      return this.exploreWarnings(exploresOverride).join("\n");
+   }
+
+   /**
+    * One message per LISTED model whose discovery surface is empty because it
+    * is import-only (imports other files, declares/re-exports nothing). Such a
+    * model renders a blank page, which reads as broken; the fix is an explicit
+    * re-export. Log-only (see loadViaWorker/reloadAllModels) — deliberately
+    * NOT part of exploreWarnings, which is strict-at-publish: import-only
+    * files are a legitimate pattern and must not block a publish. Hidden
+    * (non-listed) models are skipped — nobody browses them, so an empty
+    * surface there is just normal plumbing.
+    */
+   public emptyDiscoveryWarnings(): string[] {
+      const exploreSet = this.exploreSet();
+      const warnings: string[] = [];
+      for (const [modelPath, model] of this.models) {
+         if (!modelPath.endsWith(MODEL_FILE_SUFFIX)) continue;
+         if (exploreSet && !exploreSet.has(modelPath)) continue;
+         if (model.hasEmptyDiscoverySurface()) {
+            warnings.push(
+               `Model "${modelPath}" is listed but exposes nothing: it only ` +
+                  `imports other files and re-exports none of their sources. ` +
+                  `Add e.g. 'export { source_name }' to surface sources on ` +
+                  `this model.`,
+            );
+         }
+      }
+      return warnings;
+   }
+
+   /** Log {@link emptyDiscoveryWarnings}; shared by load and reload. */
+   private logEmptyDiscoveryWarnings(): void {
+      for (const warning of this.emptyDiscoveryWarnings()) {
+         logger.warn(`Package ${this.packageName} has a blank-looking model`, {
+            packageName: this.packageName,
+            detail: warning,
+         });
+      }
    }
 
    public listDatabases(): ApiDatabase[] {
@@ -393,18 +590,63 @@ export class Package {
                ),
             );
          } else {
-            nextModels.set(
-               sm.modelPath,
-               Model.fromSerialized(
-                  this.packageName,
-                  this.packagePath,
-                  this.malloyConfig,
-                  sm,
-               ),
+            const model = Model.fromSerialized(
+               this.packageName,
+               this.packagePath,
+               this.malloyConfig,
+               sm,
             );
+            // Validate renderer tags here too (loadViaWorker does it for the
+            // create path). Reload keeps per-model placeholders rather than
+            // aborting the whole package, so a render-tag error is recorded as
+            // this model's compilationError instead of thrown.
+            try {
+               await model.validateRenderTags();
+               nextModels.set(sm.modelPath, model);
+            } catch (renderErr) {
+               const err =
+                  renderErr instanceof Error
+                     ? renderErr
+                     : new Error(String(renderErr));
+               logger.warn("Render-tag validation failed during reload", {
+                  packageName: this.packageName,
+                  modelPath: sm.modelPath,
+                  error: err.message,
+               });
+               nextModels.set(
+                  sm.modelPath,
+                  Model.fromCompilationError(
+                     this.packageName,
+                     sm.modelPath,
+                     sm.modelType,
+                     err,
+                  ),
+               );
+            }
          }
       }
       this.models = nextModels;
+      // A reload re-reads publisher.json in the worker; pick up any change to
+      // the explore set and query-boundary mode so listModels()/the gate
+      // reflect edited explores without a full Package.create.
+      this.packageMetadata.explores = outcome.packageMetadata.explores;
+      this.packageMetadata.queryableSources =
+         outcome.packageMetadata.queryableSources;
+      this.packageMetadata.manifestLocation =
+         outcome.packageMetadata.manifestLocation ?? null;
+      this.applyDiscoveryPolicyToModels();
+      this.applyQueryBoundaryToModels();
+      // Re-run the fail-safe warning against the refreshed model set: an edit
+      // to publisher.json that introduces a bad entry should surface in the
+      // logs on reload too, not only at initial load (loadViaWorker).
+      const invalidMsg = this.formatInvalidExplores();
+      if (invalidMsg) {
+         logger.warn(`Package ${this.packageName} has invalid explores`, {
+            packageName: this.packageName,
+            detail: invalidMsg,
+         });
+      }
+      this.logEmptyDiscoveryWarnings();
    }
 
    public async getModelFileText(modelPath: string): Promise<string> {
@@ -416,10 +658,17 @@ export class Package {
    }
 
    public async listModels(): Promise<ApiModel[]> {
+      // When `explores` is declared in publisher.json, only those models
+      // form the public surface; every other .malloy file still compiles for
+      // import/join resolution but is hidden from the listing. Absent/empty →
+      // every model is listed (backward-compatible default). Notebooks are
+      // unaffected (see listNotebooks) — they are always public.
+      const exploreSet = this.exploreSet();
       const values = await Promise.all(
          Array.from(this.models.keys())
             .filter((modelPath) => {
-               return modelPath.endsWith(MODEL_FILE_SUFFIX);
+               if (!modelPath.endsWith(MODEL_FILE_SUFFIX)) return false;
+               return exploreSet ? exploreSet.has(modelPath) : true;
             })
             .map(async (modelPath) => {
                let error: string | undefined;
@@ -621,5 +870,7 @@ export class Package {
 
    public setPackageMetadata(packageMetadata: ApiPackage) {
       this.packageMetadata = packageMetadata;
+      this.applyDiscoveryPolicyToModels();
+      this.applyQueryBoundaryToModels();
    }
 }

package/src/service/package_rollback.spec.ts

@@ -0,0 +1,190 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import * as fs from "fs/promises";
+import * as os from "os";
+import * as path from "path";
+
+import { BadRequestError } from "../errors";
+import { Environment } from "./environment";
+
+/**
+ * Regression tests for explores-validation ROLLBACK, guarding against the two
+ * data-loss bugs a code review found:
+ *
+ *  1. No-location create rollback must NOT delete a pre-existing user
+ *     directory. The controller now `unloadPackage`s (evict from memory, keep
+ *     files) instead of `deletePackage` (which renames + rm's the tree).
+ *
+ *  2. A location update/reinstall with invalid explores must roll back to the
+ *     PREVIOUS tree rather than swapping the rejected one in and 400-ing after.
+ *     Validation now happens inside `installPackage`'s swap window.
+ *
+ * Both run against a real `Environment` + real `Package.create` over temp dirs.
+ */
+describe("explores-validation rollback (real Environment)", () => {
+   let rootDir: string;
+   let envPath: string;
+
+   const GOOD_MODEL = `source: ones is duckdb.sql("SELECT 1 as x")\n`;
+   const BAD_MODEL = `source: twos is duckdb.sql("SELECT 2 as y")\n`;
+
+   async function writePackageDir(
+      dir: string,
+      manifest: Record<string, unknown>,
+      model: string,
+   ): Promise<void> {
+      await fs.mkdir(dir, { recursive: true });
+      await fs.writeFile(
+         path.join(dir, "publisher.json"),
+         JSON.stringify({
+            name: "pkg",
+            description: "rollback fixture",
+            ...manifest,
+         }),
+      );
+      await fs.writeFile(path.join(dir, "model.malloy"), model);
+   }
+
+   async function copyDir(src: string, dst: string): Promise<void> {
+      await fs.mkdir(dst, { recursive: true });
+      await fs.cp(src, dst, { recursive: true });
+   }
+
+   beforeEach(async () => {
+      rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-rollback-"));
+      envPath = path.join(rootDir, "env");
+      await fs.mkdir(envPath, { recursive: true });
+   });
+
+   afterEach(async () => {
+      await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
+   });
+
+   it("Fix 1: unloadPackage evicts from memory but keeps the on-disk directory", async () => {
+      // Mimics the no-location create path: a pre-existing package directory is
+      // registered, then rolled back. unloadPackage must not touch the files.
+      const env = await Environment.create("testEnv", envPath, []);
+      const pkgDir = path.join(envPath, "pkg");
+      await writePackageDir(pkgDir, {}, GOOD_MODEL);
+
+      await env.addPackage("pkg");
+      expect(env.getPackageStatus("pkg")).toBeDefined();
+
+      await env.unloadPackage("pkg");
+
+      // Evicted from memory...
+      expect(env.getPackageStatus("pkg")).toBeUndefined();
+      // ...but the user's files are untouched.
+      const modelText = await fs.readFile(
+         path.join(pkgDir, "model.malloy"),
+         "utf-8",
+      );
+      expect(modelText).toBe(GOOD_MODEL);
+   });
+
+   it("Fix 2: a rejected location reinstall restores the previous tree", async () => {
+      const env = await Environment.create("testEnv", envPath, []);
+
+      const goodFixture = path.join(rootDir, "good");
+      const badFixture = path.join(rootDir, "bad");
+      await writePackageDir(goodFixture, {}, GOOD_MODEL);
+      await writePackageDir(
+         badFixture,
+         { explores: ["missing.malloy"] },
+         BAD_MODEL,
+      );
+
+      // First install: the good tree is served.
+      await env.installPackage("pkg", (stagingPath) =>
+         copyDir(goodFixture, stagingPath),
+      );
+      expect(await env.getModelFileText("pkg", "model.malloy")).toBe(
+         GOOD_MODEL,
+      );
+
+      // Reinstall with an invalid-explores tree, validated inside the swap.
+      await expect(
+         env.installPackage(
+            "pkg",
+            (stagingPath) => copyDir(badFixture, stagingPath),
+            (pkg) => pkg.formatInvalidExplores(),
+         ),
+      ).rejects.toBeInstanceOf(BadRequestError);
+
+      // The previous (good) tree is still the one on disk — the rejected tree
+      // was rolled back, not swapped in.
+      expect(await env.getModelFileText("pkg", "model.malloy")).toBe(
+         GOOD_MODEL,
+      );
+   });
+
+   it("a valid location reinstall still succeeds (no false rollback)", async () => {
+      const env = await Environment.create("testEnv", envPath, []);
+      const goodFixture = path.join(rootDir, "good");
+      const nextFixture = path.join(rootDir, "next");
+      await writePackageDir(goodFixture, {}, GOOD_MODEL);
+      await writePackageDir(
+         nextFixture,
+         { explores: ["model.malloy"] },
+         BAD_MODEL,
+      );
+
+      await env.installPackage("pkg", (stagingPath) =>
+         copyDir(goodFixture, stagingPath),
+      );
+      await env.installPackage(
+         "pkg",
+         (stagingPath) => copyDir(nextFixture, stagingPath),
+         (pkg) => pkg.formatInvalidExplores(),
+      );
+      expect(await env.getModelFileText("pkg", "model.malloy")).toBe(BAD_MODEL);
+   });
+
+   it("updatePackage normalizes a ./-prefixed body explores (no false 400, persists normalized)", async () => {
+      // API-body explores must go through the same normalization the worker
+      // applies to on-disk explores, so `./model.malloy` validates and persists
+      // as `model.malloy` rather than being rejected with a misleading 404.
+      const env = await Environment.create("testEnv", envPath, []);
+      const pkgDir = path.join(envPath, "pkg");
+      await writePackageDir(pkgDir, {}, GOOD_MODEL);
+      await env.addPackage("pkg");
+
+      const updated = await env.updatePackage("pkg", {
+         name: "pkg",
+         explores: ["./model.malloy"],
+      });
+
+      // Accepted (no BadRequestError) and stored in normalized form...
+      expect(updated.explores).toEqual(["model.malloy"]);
+      // ...both in memory and on disk.
+      const manifest = JSON.parse(
+         await fs.readFile(path.join(pkgDir, "publisher.json"), "utf-8"),
+      );
+      expect(manifest.explores).toEqual(["model.malloy"]);
+   });
+
+   it("compileSource rejects a notebook path; a model path still compiles", async () => {
+      // /compile appends source to the target MODEL's content for context; a
+      // notebook isn't a valid target and must be rejected up front (not left to
+      // a confusing downstream parse error).
+      const env = await Environment.create("testEnv", envPath, []);
+      const pkgDir = path.join(envPath, "pkg");
+      await writePackageDir(pkgDir, {}, GOOD_MODEL);
+      await fs.writeFile(
+         path.join(pkgDir, "report.malloynb"),
+         `>>>markdown\n# Report\n`,
+      );
+      await env.addPackage("pkg");
+
+      await expect(
+         env.compileSource("pkg", "report.malloynb", "run: ones"),
+      ).rejects.toBeInstanceOf(BadRequestError);
+
+      // The .malloy model still compiles ad-hoc source (no regression).
+      const { problems } = await env.compileSource(
+         "pkg",
+         "model.malloy",
+         "run: ones -> { select: x }",
+      );
+      expect(problems.some((p) => p.severity === "error")).toBe(false);
+   });
+});