@malloy-publisher/server 0.0.229 → 0.0.230

package/src/service/environment.ts

@@ -149,6 +149,17 @@ export class Environment {
     * the only lasting record. Read by EnvironmentStore.getStatus.
     */
    private failedPackages: Map<string, string> = new Map();
+   /**
+    * Why a configured package never reached the disk, keyed by package name.
+    *
+    * Separate from {@link failedPackages} because it is the more specific
+    * answer and has to win. A package whose location failed to mount is still
+    * seeded SERVING by addEnvironment, so its later lazy load fails on the
+    * manifest that was never copied and `listPackages` records that instead.
+    * Reporting "Package manifest ... does not exist." for what was really a
+    * typo'd `location` sends the reader hunting in the wrong place.
+    */
+   private mountErrors: Map<string, string> = new Map();
    private malloyConfig: EnvironmentMalloyConfig;
    private connectionMutex = new Mutex();
    private retiredConnectionGenerations =
@@ -472,10 +483,15 @@ export class Environment {
                );
             }
 
-            // Authorize backstop (the *who* axis, 403). Only run when the model
-            // declares gates so ungated compiles don't pay for the extra
-            // final-query compile.
-            if (queryMaterializer && gateModel?.hasAuthorize()) {
+            // Authorize backstop (the *who* axis, 403). NOT guarded by
+            // hasAuthorize(): that only inspects top-level modelDef.contents
+            // sources, so a gated source reached only via a cross-file/deep
+            // join is invisible to it and this backstop would silently never
+            // run for such a model — the same bypass assertAuthorizedForRunnable
+            // itself closes on the query path (see model.ts
+            // assertAuthorizedForAllSources). The own-source probe and joined-
+            // gate walk it runs are cheap no-ops for a genuinely ungated model.
+            if (queryMaterializer && gateModel) {
                await gateModel.assertAuthorizedForRunnable(
                   queryMaterializer,
                   givens ?? {},
@@ -923,7 +939,7 @@ export class Environment {
          this.setPackageStatus(packageName, PackageStatus.SERVING);
          // It loaded, so any earlier failure is stale. A package that failed at
          // boot can be fixed on disk and reloaded without a restart.
-         this.failedPackages.delete(packageName);
+         this.clearPackageLoadFailure(packageName);
          logger.debug(`Successfully loaded package ${packageName}`);
 
          return _package;
@@ -1008,7 +1024,7 @@ export class Environment {
       // Same reasoning as the load and install paths: it is serving now, so an
       // earlier boot failure is stale. Without this, a package fixed on disk
       // and re-added keeps its loadError for the life of the process.
-      this.failedPackages.delete(packageName);
+      this.clearPackageLoadFailure(packageName);
       return this.packages.get(packageName);
    }
 
@@ -1198,7 +1214,7 @@ export class Environment {
          this.packages.set(packageName, newPackage);
          this.setPackageStatus(packageName, PackageStatus.SERVING);
          // Publishing a fixed package clears the boot failure it replaces.
-         this.failedPackages.delete(packageName);
+         this.clearPackageLoadFailure(packageName);
 
          if (oldPackage) {
             this.retireConnectionGeneration(`package ${packageName}`, () =>
@@ -1547,9 +1563,27 @@ export class Environment {
       return this.packageStatuses.get(packageName);
    }
 
+   /**
+    * Record why a configured package's location never mounted, so /status can
+    * name the real cause instead of the missing-manifest fallout it produces.
+    * Called by EnvironmentStore right after the environment is created.
+    */
+   public setPackageMountError(packageName: string, message: string): void {
+      this.mountErrors.set(packageName, message);
+   }
+
+   /** Forget any recorded failure for a package, whatever its cause. */
+   private clearPackageLoadFailure(packageName: string): void {
+      this.failedPackages.delete(packageName);
+      this.mountErrors.delete(packageName);
+   }
+
    /** Packages configured for this environment that did not load, and why. */
    public getFailedPackages(): ReadonlyMap<string, string> {
-      return this.failedPackages;
+      if (this.mountErrors.size === 0) return this.failedPackages;
+      // Mount errors last, so the specific cause overwrites the generic
+      // manifest error that the un-mounted package produces on its lazy load.
+      return new Map([...this.failedPackages, ...this.mountErrors]);
    }
 
    public setPackageStatus(packageName: string, status: PackageStatus): void {
@@ -1574,7 +1608,7 @@ export class Environment {
          // the controller still drops its config row. Leaving the entry would
          // make getStatus report a loadError for a package that is no longer
          // configured, which is the one thing that channel must not do.
-         this.failedPackages.delete(packageName);
+         this.clearPackageLoadFailure(packageName);
 
          const _package = this.packages.get(packageName);
          if (!_package) {

package/src/service/environment_store.spec.ts

@@ -1,5 +1,6 @@
 import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test";
 import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "fs";
+import { promises as fsPromises } from "fs";
 import * as path from "path";
 import * as sinon from "sinon";
 import { components } from "../api";
@@ -470,6 +471,169 @@ describe("EnvironmentStore Service", () => {
       expect("loadErrors" in status).toBe(false);
    });
 
+   it("keeps good packages serving when a sibling's location is bad", async () => {
+      // The headline case: one bad location in an environment must not take
+      // down its healthy siblings. Two good packages plus one whose location
+      // does not exist -> both good ones serve, the bad one is a per-package
+      // loadError, and the environment is intact.
+      const goodA = path.join(serverRootPath, "good-a");
+      const goodB = path.join(serverRootPath, "good-b");
+      for (const dir of [goodA, goodB]) {
+         mkdirSync(dir, { recursive: true });
+         writeFileSync(
+            path.join(dir, "publisher.json"),
+            JSON.stringify({ name: path.basename(dir) }),
+         );
+      }
+      writeFileSync(
+         path.join(serverRootPath, "publisher.config.json"),
+         JSON.stringify({
+            environments: [
+               {
+                  name: projectName,
+                  packages: [
+                     { name: "good-a", location: goodA },
+                     { name: "bad", location: "/non/existent/path" },
+                     { name: "good-b", location: goodB },
+                  ],
+                  connections: [],
+               },
+            ],
+         }),
+      );
+
+      const newEnvironmentStore = new EnvironmentStore(serverRootPath);
+      await newEnvironmentStore.finishedInitialization;
+
+      const status = await newEnvironmentStore.getStatus();
+      expect(status.operationalState).toBe("serving");
+      expect(status.environments.map((e) => e.name)).toEqual([projectName]);
+
+      const environment = await newEnvironmentStore.getEnvironment(projectName);
+      const packages = await environment.listPackages();
+      expect(packages.map((p) => p.name).sort()).toEqual(["good-a", "good-b"]);
+
+      expect(status.loadErrors).toHaveLength(1);
+      expect(status.loadErrors?.[0]?.environment).toBe(projectName);
+      expect(status.loadErrors?.[0]?.package).toBe("bad");
+      // The message has to name the location that is actually wrong. The
+      // un-mounted package also fails its lazy load on the manifest that was
+      // never copied, and reporting that instead points the reader at
+      // publisher_data/ for what is really a typo in the config.
+      expect(status.loadErrors?.[0]?.message).toContain("/non/existent/path");
+      expect(status.loadErrors?.[0]?.message).not.toContain("publisher_data");
+   });
+
+   it("keeps sibling packages serving when one sharing their location fails to extract", async () => {
+      // Packages grouped under ONE location share a single download, then each
+      // is extracted separately. A failure in one extract must not strand the
+      // siblings queued behind it: the group-level catch used to wrap the whole
+      // loop, so with three packages from one repo a throw on the second meant
+      // the third was never copied even though nothing was wrong with it.
+      const shared = path.join(serverRootPath, "shared-src");
+      mkdirSync(shared, { recursive: true });
+      writeFileSync(
+         path.join(shared, "publisher.json"),
+         JSON.stringify({ name: "shared" }),
+      );
+
+      writeFileSync(
+         path.join(serverRootPath, "publisher.config.json"),
+         JSON.stringify({
+            environments: [
+               {
+                  name: projectName,
+                  packages: [
+                     { name: "first", location: shared },
+                     { name: "middle", location: shared },
+                     { name: "last", location: shared },
+                  ],
+                  connections: [],
+               },
+            ],
+         }),
+      );
+
+      // Fail exactly one package's copy. Everything reachable from config is
+      // shared across the group (same location, hence the same source path), so
+      // the copy is the only per-package step that can realistically fail on
+      // its own — the same shape as a GitHub subdirectory that isn't there.
+      const realCp = fsPromises.cp;
+      const cpStub = sinon.stub(fsPromises, "cp").callsFake((async (
+         src: Parameters<typeof realCp>[0],
+         dest: Parameters<typeof realCp>[1],
+         opts?: Parameters<typeof realCp>[2],
+      ) => {
+         if (String(dest).endsWith(`${path.sep}middle`)) {
+            throw new Error("simulated extract failure for middle");
+         }
+         return realCp(src, dest, opts);
+      }) as typeof realCp);
+
+      try {
+         const newEnvironmentStore = new EnvironmentStore(serverRootPath);
+         await newEnvironmentStore.finishedInitialization;
+
+         const environment =
+            await newEnvironmentStore.getEnvironment(projectName);
+         const packages = await environment.listPackages();
+         // "last" is the one that regressed: it is queued behind the failure.
+         expect(packages.map((p) => p.name).sort()).toEqual(["first", "last"]);
+
+         const status = await newEnvironmentStore.getStatus();
+         expect(status.operationalState).toBe("serving");
+         expect(status.loadErrors).toHaveLength(1);
+         expect(status.loadErrors?.[0]?.package).toBe("middle");
+         expect(status.loadErrors?.[0]?.message).toContain(
+            "simulated extract failure",
+         );
+      } finally {
+         cpStub.restore();
+      }
+   });
+
+   it("does not delete the environment directory when a package name is rejected", async () => {
+      // A package name that fails the allowlist throws as the FIRST statement of
+      // the extract, so the per-package catch runs before any path for it
+      // exists. safeJoinUnderRoot deliberately allows a name that resolves to
+      // the root, so re-deriving the cleanup target from the name would hand it
+      // the whole environment directory and take the healthy siblings with it.
+      const good = path.join(serverRootPath, "good-src");
+      mkdirSync(good, { recursive: true });
+      writeFileSync(
+         path.join(good, "publisher.json"),
+         JSON.stringify({ name: "good" }),
+      );
+
+      writeFileSync(
+         path.join(serverRootPath, "publisher.config.json"),
+         JSON.stringify({
+            environments: [
+               {
+                  name: projectName,
+                  packages: [
+                     { name: "good", location: good },
+                     { name: ".", location: good },
+                  ],
+                  connections: [],
+               },
+            ],
+         }),
+      );
+
+      const newEnvironmentStore = new EnvironmentStore(serverRootPath);
+      await newEnvironmentStore.finishedInitialization;
+
+      // Boot pre-validates the names and skips the environment, so reach the
+      // lazy path, which passes the config's package names through unchecked.
+      await newEnvironmentStore.getEnvironment(projectName).catch(() => {});
+
+      const envDir = path.join(serverRootPath, "publisher_data", projectName);
+      // The environment directory and the healthy sibling must both survive.
+      expect(existsSync(envDir)).toBe(true);
+      expect(existsSync(path.join(envDir, "good"))).toBe(true);
+   });
+
    it("should not report an environment that is serving even if its database sync fails", async () => {
       // An environment reaches this.environments before addEnvironmentToDatabase
       // runs, so a throw from that tail is caught by the same handler that
@@ -824,8 +988,10 @@ describe("EnvironmentStore Service", () => {
       { timeout: 30000 },
    );
 
-   it("should handle missing project paths", async () => {
-      // Create publisher config with non-existent project path
+   it("isolates a bad package location to that package, keeping the environment", async () => {
+      // A package whose location does not exist used to abort the whole
+      // environment. It now behaves like a bad manifest: the package is
+      // dropped and reported per-package, and the environment still loads.
       const publisherConfigPath = path.join(
          serverRootPath,
          "publisher.config.json",
@@ -847,10 +1013,20 @@ describe("EnvironmentStore Service", () => {
          }),
       );
 
-      // Test that getting the project throws an error
-      await expect(
-         environmentStore.getEnvironment(projectName),
-      ).rejects.toThrow();
+      const newEnvironmentStore = new EnvironmentStore(serverRootPath);
+      await newEnvironmentStore.finishedInitialization;
+
+      // The environment loaded rather than being skipped.
+      const environment = await newEnvironmentStore.getEnvironment(projectName);
+      expect(environment.metadata.name).toBe(projectName);
+
+      // And the failure is reported at the package level, not the env level.
+      const status = await newEnvironmentStore.getStatus();
+      expect(status.operationalState).toBe("serving");
+      expect(status.loadErrors).toHaveLength(1);
+      expect(status.loadErrors?.[0]?.environment).toBe(projectName);
+      expect(status.loadErrors?.[0]?.package).toBe(projectName);
+      expect(status.loadErrors?.[0]?.message).toBeTruthy();
    });
 
    it("should handle invalid publisher config", async () => {
@@ -1095,8 +1271,10 @@ describe("Project Service Error Recovery", () => {
    });
 
    describe("Project Loading Error Recovery", () => {
-      it("should handle missing project directories gracefully", async () => {
-         // Create publisher config with missing project directory
+      it("keeps the environment when a package directory is missing, reporting per-package", async () => {
+         // Same isolation as above, for a relative-to-serverRoot missing dir:
+         // the environment loads and the missing package is a per-package
+         // loadError rather than an environment-level skip.
          const publisherConfigPath = path.join(
             serverRootPath,
             "publisher.config.json",
@@ -1121,10 +1299,17 @@ describe("Project Service Error Recovery", () => {
             }),
          );
 
-         // Test that the project store handles the missing directory
-         await expect(
-            environmentStore.getEnvironment(projectName),
-         ).rejects.toThrow();
+         const newEnvironmentStore = new EnvironmentStore(serverRootPath);
+         await newEnvironmentStore.finishedInitialization;
+
+         const environment =
+            await newEnvironmentStore.getEnvironment(projectName);
+         expect(environment.metadata.name).toBe(projectName);
+
+         const status = await newEnvironmentStore.getStatus();
+         expect(status.operationalState).toBe("serving");
+         expect(status.loadErrors).toHaveLength(1);
+         expect(status.loadErrors?.[0]?.package).toBe(projectName);
       });
 
       it(

package/src/service/environment_store.ts

@@ -1013,13 +1013,16 @@ export class EnvironmentStore {
          (environment?.packages && environment.packages.length > 0) ||
          (environmentConfig?.packages && environmentConfig.packages.length > 0);
       let absoluteEnvironmentPath: string;
+      let mountErrors: ReadonlyMap<string, string> = new Map();
       if (hasPackages) {
          const packagesToProcess =
             environment?.packages || environmentConfig?.packages || [];
-         absoluteEnvironmentPath = await this.loadEnvironmentIntoDisk(
+         const loaded = await this.loadEnvironmentIntoDisk(
             environmentName,
             packagesToProcess,
          );
+         absoluteEnvironmentPath = loaded.path;
+         mountErrors = loaded.mountErrors;
          if (absoluteEnvironmentPath.endsWith(".zip")) {
             absoluteEnvironmentPath = await this.unzipEnvironment(
                absoluteEnvironmentPath,
@@ -1038,6 +1041,12 @@ export class EnvironmentStore {
       if (!newEnvironment.metadata) newEnvironment.metadata = {};
       newEnvironment.metadata.location = absoluteEnvironmentPath;
 
+      // Attach before the SERVING seeding below: those packages are configured
+      // but un-mounted, and this is the only record of why.
+      for (const [packageName, message] of mountErrors) {
+         newEnvironment.setPackageMountError(packageName, message);
+      }
+
       this.environments.set(environmentName, newEnvironment);
       // It loaded, so any earlier failure for this name is stale. A boot
       // failure can be followed by a successful POST or lazy load of the same
@@ -1263,6 +1272,38 @@ export class EnvironmentStore {
       return location.startsWith("s3://");
    }
 
+   /**
+    * Isolate a location failure to the packages it actually affects instead of
+    * aborting the whole environment. The packages that mounted still serve;
+    * each one named here is left un-mounted, so its configured status entry
+    * (set in addEnvironment) resolves to a per-package load failure that
+    * getFailedPackages() -> /status loadErrors reports. A bad location now
+    * behaves like a bad manifest: those packages dropped, siblings unaffected.
+    *
+    * The message is kept rather than letting the lazy load speak for it. An
+    * un-mounted package fails later on the manifest that was never copied, and
+    * "Package manifest ... does not exist." points at publisher_data/ instead
+    * of the location that is actually wrong. Redacted because it reaches an
+    * HTTP response body.
+    */
+   private recordMountFailure(
+      error: unknown,
+      location: string,
+      packageNames: string[],
+      mountErrors: Map<string, string>,
+   ): void {
+      logger.error(
+         `Failed to download or mount location "${location}"`,
+         this.extractErrorDataFromError(error),
+      );
+      const message = redactPgSecrets(
+         error instanceof Error ? error.message : String(error),
+      );
+      for (const packageName of packageNames) {
+         mountErrors.set(packageName, message);
+      }
+   }
+
    private async loadEnvironmentIntoDisk(
       environmentName: string,
       packages: ApiEnvironment["packages"],
@@ -1273,6 +1314,10 @@ export class EnvironmentStore {
          PUBLISHER_DATA_DIR,
          environmentName,
       );
+      // Why each package's location failed, carried out to the Environment so
+      // /status can report the real cause rather than the missing manifest it
+      // leaves behind.
+      const mountErrors = new Map<string, string>();
 
       await fs.promises.mkdir(absoluteTargetPath, { recursive: true });
 
@@ -1333,6 +1378,11 @@ export class EnvironmentStore {
          );
          await fs.promises.mkdir(tempDownloadPath, { recursive: true });
          logger.info(`Created temporary directory: ${tempDownloadPath}`);
+         // Two failure scopes, deliberately separate. The download is shared by
+         // every package at this location, so losing it loses all of them. An
+         // extract is per package, so one bad sibling must not strand the ones
+         // after it in the loop.
+         let downloaded = true;
          try {
             // Use the existing download method for all locations
             await this.downloadOrMountLocation(
@@ -1341,11 +1391,28 @@ export class EnvironmentStore {
                environmentName,
                "shared",
             );
-            // Extract each package from the downloaded content
-            for (const _package of packagesForLocation) {
+         } catch (error) {
+            downloaded = false;
+            this.recordMountFailure(
+               error,
+               groupedLocation,
+               packagesForLocation.map((p) => p.name),
+               mountErrors,
+            );
+         }
+         // Extract each package from the downloaded content
+         for (const _package of downloaded ? packagesForLocation : []) {
+            // Declared out here so the catch can clean up the exact path this
+            // iteration created. Never re-derive it from the package name
+            // there: the name may be why we are in the catch at all, and
+            // safeJoinUnderRoot permits a name that resolves to the root, so a
+            // package called "." would hand the cleanup the whole environment
+            // directory. Undefined means nothing was created yet.
+            let absolutePackagePath: string | undefined;
+            try {
                const packageDir = _package.name;
                assertSafePackageName(packageDir);
-               const absolutePackagePath = safeJoinUnderRoot(
+               absolutePackagePath = safeJoinUnderRoot(
                   absoluteTargetPath,
                   packageDir,
                );
@@ -1481,16 +1548,23 @@ export class EnvironmentStore {
                      `Copied entire download as package "${packageDir}"`,
                   );
                }
+            } catch (error) {
+               this.recordMountFailure(
+                  error,
+                  groupedLocation,
+                  [_package.name],
+                  mountErrors,
+               );
+               // Leave nothing behind. A copy that threw part way through can
+               // leave a tree holding the manifest and models but not the data,
+               // which loads cleanly and therefore clears the very error
+               // recorded above, so the package serves incomplete content and
+               // /status says nothing. Continuing past the failure is what
+               // makes this reachable for more than one package per location.
+               if (absolutePackagePath) {
+                  await clearMountTarget(absolutePackagePath);
+               }
             }
-         } catch (error) {
-            const errorData = this.extractErrorDataFromError(error);
-            logger.error(
-               `Failed to download or mount location "${groupedLocation}"`,
-               errorData,
-            );
-            throw new PackageNotFoundError(
-               `Failed to download or mount location: ${groupedLocation}`,
-            );
          }
          try {
             // Clean up temporary download directory
@@ -1508,7 +1582,7 @@ export class EnvironmentStore {
          }
       }
 
-      return absoluteTargetPath;
+      return { path: absoluteTargetPath, mountErrors };
    }
 
    private async downloadOrMountLocation(