@malloy-publisher/server 0.0.196 → 0.0.197

package/src/pg_helpers.spec.ts

@@ -0,0 +1,226 @@
+import { afterEach, describe, expect, it } from "bun:test";
+import { ConnectionAuthError } from "./errors";
+import {
+   classifyPgError,
+   handlePgAttachError,
+   pgConnectTimeoutSeconds,
+   redactPgSecrets,
+   withPgConnectTimeout,
+} from "./pg_helpers";
+
+describe("pgConnectTimeoutSeconds", () => {
+   const ORIGINAL_TIMEOUT = process.env.PG_CONNECT_TIMEOUT_SECONDS;
+
+   afterEach(() => {
+      if (ORIGINAL_TIMEOUT === undefined) {
+         delete process.env.PG_CONNECT_TIMEOUT_SECONDS;
+      } else {
+         process.env.PG_CONNECT_TIMEOUT_SECONDS = ORIGINAL_TIMEOUT;
+      }
+   });
+
+   it("defaults to 5 when env unset", () => {
+      delete process.env.PG_CONNECT_TIMEOUT_SECONDS;
+      expect(pgConnectTimeoutSeconds()).toBe(5);
+   });
+
+   it("honors PG_CONNECT_TIMEOUT_SECONDS override", () => {
+      process.env.PG_CONNECT_TIMEOUT_SECONDS = "12";
+      expect(pgConnectTimeoutSeconds()).toBe(12);
+   });
+
+   it("falls back to 5 when env value is invalid", () => {
+      process.env.PG_CONNECT_TIMEOUT_SECONDS = "not-a-number";
+      expect(pgConnectTimeoutSeconds()).toBe(5);
+   });
+
+   it("falls back to 5 when env value is zero or negative", () => {
+      process.env.PG_CONNECT_TIMEOUT_SECONDS = "0";
+      expect(pgConnectTimeoutSeconds()).toBe(5);
+      process.env.PG_CONNECT_TIMEOUT_SECONDS = "-3";
+      expect(pgConnectTimeoutSeconds()).toBe(5);
+   });
+});
+
+describe("withPgConnectTimeout", () => {
+   it("appends to keyword form when missing", () => {
+      expect(withPgConnectTimeout("host=h dbname=d user=u password=p", 5)).toBe(
+         "host=h dbname=d user=u password=p connect_timeout=5",
+      );
+   });
+
+   it("appends to postgres: keyword form (DuckLake catalogUrl shape)", () => {
+      expect(
+         withPgConnectTimeout("postgres:host=h user=u password=p dbname=d", 5),
+      ).toBe("postgres:host=h user=u password=p dbname=d connect_timeout=5");
+   });
+
+   it("does not override a user-supplied connect_timeout in keyword form", () => {
+      expect(withPgConnectTimeout("host=h connect_timeout=30", 99)).toBe(
+         "host=h connect_timeout=30",
+      );
+   });
+
+   it("appends to URI form with no query", () => {
+      expect(withPgConnectTimeout("postgresql://u:p@h:5432/d", 5)).toBe(
+         "postgresql://u:p@h:5432/d?connect_timeout=5",
+      );
+   });
+
+   it("appends to URI form with existing query", () => {
+      expect(
+         withPgConnectTimeout("postgresql://u:p@h/d?sslmode=require", 5),
+      ).toBe("postgresql://u:p@h/d?sslmode=require&connect_timeout=5");
+   });
+
+   it("appends to URI with bare trailing ?", () => {
+      expect(withPgConnectTimeout("postgresql://h/d?", 5)).toBe(
+         "postgresql://h/d?connect_timeout=5",
+      );
+   });
+
+   it("does not double-append when URI already has connect_timeout (?-style)", () => {
+      expect(
+         withPgConnectTimeout("postgresql://h/d?connect_timeout=10", 5),
+      ).toBe("postgresql://h/d?connect_timeout=10");
+   });
+
+   it("does not double-append when URI already has connect_timeout (&-style)", () => {
+      expect(
+         withPgConnectTimeout(
+            "postgresql://h/d?sslmode=require&connect_timeout=10",
+            5,
+         ),
+      ).toBe("postgresql://h/d?sslmode=require&connect_timeout=10");
+   });
+
+   it("recognizes postgres:// (alternative scheme) as URI form", () => {
+      expect(withPgConnectTimeout("postgres://u@h/d", 5)).toBe(
+         "postgres://u@h/d?connect_timeout=5",
+      );
+   });
+});
+
+describe("redactPgSecrets", () => {
+   it("redacts bare password values", () => {
+      expect(redactPgSecrets("host=h password=hunter2 dbname=d")).toBe(
+         "host=h password=*** dbname=d",
+      );
+   });
+
+   it("redacts single-quoted password values", () => {
+      expect(redactPgSecrets("host=h password='s3 cret' dbname=d")).toBe(
+         "host=h password=*** dbname=d",
+      );
+   });
+
+   it("leaves non-secret content alone", () => {
+      expect(redactPgSecrets("user=alice dbname=billing")).toBe(
+         "user=alice dbname=billing",
+      );
+   });
+});
+
+describe("classifyPgError", () => {
+   it.each([
+      'password authentication failed for user "alice"',
+      "no pg_hba.conf entry for host",
+      'role "alice" does not exist',
+      'database "billing" does not exist',
+      "permission denied for relation foo",
+   ])("classifies '%s' as auth error", (msg) => {
+      const result = classifyPgError(new Error(msg), "PG attach");
+      expect(result).toBeInstanceOf(ConnectionAuthError);
+      expect(result?.message).toContain("PG attach:");
+   });
+
+   it("returns undefined for unrelated errors", () => {
+      expect(
+         classifyPgError(
+            new Error('relation "users" does not exist'),
+            "PG attach",
+         ),
+      ).toBeUndefined();
+      expect(
+         classifyPgError(new Error("connection reset by peer"), "PG attach"),
+      ).toBeUndefined();
+   });
+
+   it("returns undefined for non-Error values", () => {
+      expect(
+         classifyPgError("password authentication failed", "ctx"),
+      ).toBeUndefined();
+      expect(classifyPgError(undefined, "ctx")).toBeUndefined();
+   });
+
+   it("redacts embedded passwords in the wrapped message", () => {
+      const result = classifyPgError(
+         new Error(
+            "password authentication failed: tried host=h password=hunter2",
+         ),
+         "DuckLake attach",
+      );
+      expect(result?.message).toContain("password=***");
+      expect(result?.message).not.toContain("hunter2");
+   });
+});
+
+describe("handlePgAttachError", () => {
+   it("swallows 'already exists' errors", () => {
+      const outcome = handlePgAttachError(
+         new Error('database "db_x" already exists'),
+         "ctx",
+      );
+      expect(outcome.action).toBe("swallow");
+   });
+
+   it("swallows 'already attached' errors", () => {
+      const outcome = handlePgAttachError(
+         new Error("DuckLake catalog db_x is already attached"),
+         "ctx",
+      );
+      expect(outcome.action).toBe("swallow");
+   });
+
+   it("classifies libpq auth failures as ConnectionAuthError", () => {
+      const outcome = handlePgAttachError(
+         new Error('password authentication failed for user "alice"'),
+         "PG attach db_x",
+      );
+      expect(outcome.action).toBe("throw");
+      if (outcome.action === "throw") {
+         expect(outcome.error).toBeInstanceOf(ConnectionAuthError);
+         expect(outcome.error.message).toContain("PG attach db_x:");
+      }
+   });
+
+   it("passes through unrelated Error instances unchanged", () => {
+      const original = new Error("network unreachable");
+      const outcome = handlePgAttachError(original, "ctx");
+      expect(outcome.action).toBe("throw");
+      if (outcome.action === "throw") {
+         expect(outcome.error).toBe(original);
+         expect(outcome.error).not.toBeInstanceOf(ConnectionAuthError);
+      }
+   });
+
+   it("wraps non-Error throwables so callers always get an Error", () => {
+      const outcome = handlePgAttachError("a string was thrown", "ctx");
+      expect(outcome.action).toBe("throw");
+      if (outcome.action === "throw") {
+         expect(outcome.error).toBeInstanceOf(Error);
+         expect(outcome.error.message).toBe("a string was thrown");
+      }
+   });
+
+   it("prefers 'already attached' over auth classification when both keywords appear", () => {
+      // Defensive: if a future DuckDB version emits a combined message,
+      // 'already attached' wins so we don't bubble up a false auth failure
+      // on what is actually a benign idempotent re-attach.
+      const outcome = handlePgAttachError(
+         new Error("already attached; permission denied tail"),
+         "ctx",
+      );
+      expect(outcome.action).toBe("swallow");
+   });
+});

package/src/pg_helpers.ts

@@ -0,0 +1,129 @@
+// Postgres / libpq helpers shared between `service/` (user-facing
+// connections) and `storage/` (materialization-storage catalog). Lives at
+// `src/` root so neither layer takes a dependency on the other — see
+// CLAUDE.md's "Two parallel DuckLake/PG attach paths" note for why this
+// matters.
+import { ConnectionAuthError } from "./errors";
+
+// Default Postgres connect_timeout (seconds), used by the materialization
+// storage catalog ATTACH so a slow or wedged libpq handshake fails the
+// caller in seconds instead of stalling the worker until the K8s liveness
+// probe trips.
+//
+// libpq enforces a documented minimum of 2 seconds — values below 2
+// effectively round up to ~2s wall clock.
+export function pgConnectTimeoutSeconds(): number {
+   const raw = process.env.PG_CONNECT_TIMEOUT_SECONDS;
+   if (!raw) return 5;
+   const parsed = Number.parseInt(raw, 10);
+   return Number.isFinite(parsed) && parsed > 0 ? parsed : 5;
+}
+
+// libpq accepts both keyword=value form ("host=h dbname=d") and URI form
+// ("postgresql://u:p@h/d?param=v"). The materialization-storage catalogUrl
+// can also arrive as `postgres:<keyword=value>` (no `//`). We detect URI
+// form (with `//`) so we know whether to append a new parameter using
+// `?`/`&` or a leading space.
+const URI_FORM_RE = /^[a-z][a-z0-9+.-]*:\/\//i;
+
+// Match an existing connect_timeout key in either form. URI form uses
+// `?key=` or `&key=`; keyword form uses whitespace separation or start-of-
+// string. Without the `[?&]` alternatives a URI-form user-supplied timeout
+// would be missed and we'd double-append, producing an invalid URL.
+const HAS_CONNECT_TIMEOUT_RE = /[?&\s]connect_timeout=|^connect_timeout=/;
+
+// Append `connect_timeout=N` to a libpq-compatible connection string if
+// the caller hasn't already set one. Handles keyword form ("host=h ..."),
+// URI form ("postgresql://..."), and the `postgres:host=h ...` keyword
+// form with a scheme prefix used by DuckLake catalogUrls.
+export function withPgConnectTimeout(
+   connectionString: string,
+   timeout: number,
+): string {
+   if (HAS_CONNECT_TIMEOUT_RE.test(connectionString)) {
+      return connectionString;
+   }
+   if (URI_FORM_RE.test(connectionString)) {
+      // URI form: append as query parameter. `?` if no query string yet,
+      // `&` otherwise. A bare trailing `?` (empty query) gets no extra
+      // separator. We don't try to handle URL fragments — libpq URIs don't
+      // use them.
+      if (!connectionString.includes("?")) {
+         return `${connectionString}?connect_timeout=${timeout}`;
+      }
+      if (connectionString.endsWith("?")) {
+         return `${connectionString}connect_timeout=${timeout}`;
+      }
+      return `${connectionString}&connect_timeout=${timeout}`;
+   }
+   // Keyword=value form (with or without `postgres:` scheme prefix).
+   return `${connectionString} connect_timeout=${timeout}`;
+}
+
+// Redact libpq `password=...` values from a string before it goes into a
+// log line or HTTP response body. Handles bare and quoted values.
+//
+// Scope: keyword-form `password=` only. Does not touch URL-style
+// `user:pw@host` credentials, AWS keys, GCS secrets, etc.
+export function redactPgSecrets(s: string): string {
+   return s.replace(/password=('[^']*'|"[^"]*"|\S+)/gi, "password=***");
+}
+
+// Substring-match libpq error patterns that indicate a non-retryable
+// auth/permission failure. Returns a ConnectionAuthError when matched so
+// callers can fast-fail with HTTP 422 (semantically "the supplied creds
+// are bad; don't retry") instead of letting the raw error fall through to
+// a generic 500 that retry loops treat as transient.
+export function classifyPgError(
+   error: unknown,
+   context: string,
+): ConnectionAuthError | undefined {
+   if (!(error instanceof Error)) return undefined;
+   const msg = error.message;
+   const patterns = [
+      /password authentication failed/i,
+      /pg_hba\.conf/i,
+      /role ".*" does not exist/i,
+      /database ".*" does not exist/i,
+      /permission denied/i,
+   ];
+   if (!patterns.some((p) => p.test(msg))) return undefined;
+   return new ConnectionAuthError(`${context}: ${redactPgSecrets(msg)}`);
+}
+
+// Outcome of inspecting an error thrown by an `ATTACH` call:
+//   - `{ action: "swallow" }`: DuckDB reported the db is already attached
+//     (idempotent re-attach); caller should log and continue.
+//   - `{ action: "throw", error: ConnectionAuthError }`: classified as a
+//     non-retryable auth failure; caller should warn-log and throw it.
+//   - `{ action: "throw", error: <original> }`: unrecognized; caller
+//     should rethrow as-is to preserve the original cause for diagnosis.
+//
+// Extracted so the decision tree gets a direct unit test without needing
+// to stub DuckDB or run a real ATTACH.
+export type PgAttachErrorOutcome =
+   | { action: "swallow" }
+   | { action: "throw"; error: Error };
+
+export function handlePgAttachError(
+   error: unknown,
+   context: string,
+): PgAttachErrorOutcome {
+   if (
+      error instanceof Error &&
+      (error.message.includes("already exists") ||
+         error.message.includes("already attached"))
+   ) {
+      return { action: "swallow" };
+   }
+   const authErr = classifyPgError(error, context);
+   if (authErr) {
+      return { action: "throw", error: authErr };
+   }
+   if (error instanceof Error) {
+      return { action: "throw", error };
+   }
+   // Non-Error thrown values get wrapped so the catch contract stays
+   // (always throws an Error).
+   return { action: "throw", error: new Error(String(error)) };
+}

package/src/server.ts

@@ -51,6 +51,8 @@ export function normalizeQueryArray(value: unknown): string[] | undefined {
 // Parse command line arguments
 function parseArgs() {
    const args = process.argv.slice(2);
+   let sawServerRoot = false;
+   let sawConfig = false;
    for (let i = 0; i < args.length; i++) {
       const arg = args[i];
       if (arg === "--port" && args[i + 1]) {
@@ -60,8 +62,13 @@ function parseArgs() {
          process.env.PUBLISHER_HOST = args[i + 1];
          i++;
       } else if (arg === "--server_root" && args[i + 1]) {
+         sawServerRoot = true;
          process.env.SERVER_ROOT = args[i + 1];
          i++;
+      } else if (arg === "--config" && args[i + 1]) {
+         sawConfig = true;
+         process.env.PUBLISHER_CONFIG_PATH = args[i + 1];
+         i++;
       } else if (arg === "--mcp_port" && args[i + 1]) {
          process.env.MCP_PORT = args[i + 1];
          i++;
@@ -91,6 +98,9 @@ function parseArgs() {
          console.log(
             "  --server_root <path>   Root directory to serve files from (default: .)",
          );
+         console.log(
+            "  --config <path>        Path to publisher.config.json (default: <server_root>/publisher.config.json; falls back to bundled DuckDB-only sample config if missing)",
+         );
          console.log(
             "  --mcp_port <number>    Port for MCP server (default: 4040)",
          );
@@ -107,6 +117,16 @@ function parseArgs() {
          process.exit(0);
       }
    }
+   // Zero-config invocation (`npx @malloy-publisher/server`) opts in to
+   // the bundled DuckDB-only sample config so the Quick Start works
+   // without any flags. Any explicit --server_root or --config disables
+   // this — the user told us where to look. Skip in NODE_ENV=test so
+   // specs that import this module for utility helpers (e.g.
+   // db_utils.spec.ts -> normalizeQueryArray) don't get the bundled
+   // default leaked into their EnvironmentStore construction.
+   if (!sawServerRoot && !sawConfig && process.env.NODE_ENV !== "test") {
+      process.env.PUBLISHER_USE_BUNDLED_DEFAULT = "true";
+   }
 }
 
 // Parse CLI arguments before setting up constants

package/src/service/connection.spec.ts

@@ -1129,10 +1129,14 @@ describe("connection integration tests", () => {
                   ],
                   testEnvironmentPath,
                ),
-            ).rejects.toThrow(/cannot be 'duckdb'/);
+            ).rejects.toThrow(/'duckdb' is reserved/);
          });
 
          it("should reject DuckDB connections with no attachments", async () => {
+            // Env-level DuckDB connections must declare at least one
+            // attached foreign database; the empty-array case is operator
+            // confusion (the per-package "duckdb" sandbox already covers
+            // the plain-in-memory use case).
             await expect(
                createEnvironmentConnections(
                   [
@@ -1144,9 +1148,7 @@ describe("connection integration tests", () => {
                   ],
                   testEnvironmentPath,
                ),
-            ).rejects.toThrow(
-               "DuckDB connection must have at least one attached database",
-            );
+            ).rejects.toThrow(/has no attached databases/);
          });
 
          it("should reject unsupported DuckDB connector fields", async () => {

package/src/service/connection.ts

@@ -25,6 +25,7 @@ import fs from "fs/promises";
 import path from "path";
 import { components } from "../api";
 import { logAxiosError, logger } from "../logger";
+import { redactPgSecrets } from "../pg_helpers";
 import {
    assembleEnvironmentConnections,
    CoreConnectionEntry,
@@ -365,13 +366,17 @@ async function attachDuckLake(
    const pgConnString: string = buildPgConnectionString(pg);
    // Attach DuckLake with Postgres catalog and cloud storage data path in READ_ONLY mode
    // The client manages metadata - we only read from the catalogs
-   logger.info(`pgConnString: ${pgConnString}`);
+   logger.info(`pgConnString: ${redactPgSecrets(pgConnString)}`);
    const escapedPgConnString = escapeSQL(pgConnString);
-   logger.info(`Final escaped connection string: ${escapedPgConnString}`);
+   logger.info(
+      `Final escaped connection string: ${redactPgSecrets(escapedPgConnString)}`,
+   );
    const escapedBucketUrl = escapeSQL(ducklakeConfig.storage.bucketUrl);
    logger.info(`escapedBucketUrl: ${escapedBucketUrl}`);
    const attachCommand = `ATTACH OR REPLACE 'ducklake:postgres:${escapedPgConnString}' AS ${dbName} (DATA_PATH '${escapedBucketUrl}', OVERRIDE_DATA_PATH true, READ_ONLY true);`;
-   logger.info(`Attaching DuckLake database using command: ${attachCommand}`);
+   logger.info(
+      `Attaching DuckLake database using command: ${redactPgSecrets(attachCommand)}`,
+   );
    try {
       await connection.runSQL(attachCommand);
       logger.info(

package/src/service/connection_config.ts

@@ -272,7 +272,7 @@ function validateConnectionShape(connection: ApiConnection): void {
                connection.duckdbConnection.attachedDatabases ?? [];
             if (attached.length === 0) {
                throw new Error(
-                  "DuckDB connection must have at least one attached database",
+                  `DuckDB connection "${connection.name}" has no attached databases. Add at least one foreign database (BigQuery, Snowflake, Postgres, GCS, S3, Azure) to attachedDatabases, or remove this connection entirely — each package already gets a per-package DuckDB sandbox named "duckdb" automatically.`,
                );
             }
          }
@@ -359,7 +359,7 @@ export function assembleEnvironmentConnections(
 
       if (connection.name === "duckdb") {
          throw new Error(
-            "DuckDB connection name cannot be 'duckdb'; it is reserved for Publisher package sandboxes.",
+            "Connection name 'duckdb' is reserved for per-package sandboxes. Choose a different name for environment-level DuckDB connections (e.g. 'shared_duckdb').",
          );
       }
 

package/src/service/environment.ts

@@ -389,6 +389,16 @@ export class Environment {
       }
    }
 
+   /** One mutex per package name; never replace after create (avoids parallel loads). */
+   private getOrCreatePackageMutex(packageName: string): Mutex {
+      let packageMutex = this.packageMutexes.get(packageName);
+      if (packageMutex === undefined) {
+         packageMutex = new Mutex();
+         this.packageMutexes.set(packageName, packageMutex);
+      }
+      return packageMutex;
+   }
+
    public async getPackage(
       packageName: string,
       reload: boolean = false,
@@ -399,25 +409,23 @@ export class Environment {
          return _package;
       }
 
-      // We need to acquire the mutex to prevent a thundering herd of requests from creating the
-      // package multiple times.
-      let packageMutex = this.packageMutexes.get(packageName);
-      if (packageMutex?.isLocked()) {
+      // Serialize load per package name so concurrent callers share one Mutex and
+      // failed loads cannot rm the tree while another load is still scanning it.
+      const packageMutex = this.getOrCreatePackageMutex(packageName);
+
+      if (packageMutex.isLocked()) {
          logger.debug(
             `Package ${packageName} is being loaded, waiting for unlock...`,
          );
          await packageMutex.waitForUnlock();
          logger.debug(`Package ${packageName} unlocked`);
          const existingPackage = this.packages.get(packageName);
-         if (existingPackage) {
+         if (existingPackage !== undefined && !reload) {
             logger.debug(`Package ${packageName} loaded by another request`);
             return existingPackage;
          }
-         // If package still doesn't exist after unlock, it might have failed to load
-         // Continue to try loading it ourselves
+         // Reload, or prior load failed — continue under the same mutex.
       }
-      packageMutex = new Mutex();
-      this.packageMutexes.set(packageName, packageMutex);
 
       return packageMutex.runExclusive(async () => {
          // Double-check after acquiring mutex
@@ -479,24 +487,44 @@ export class Environment {
             malloyConfig: this.malloyConfig.malloyConfig,
          },
       );
-      this.setPackageStatus(packageName, PackageStatus.LOADING);
-      try {
-         this.packages.set(
-            packageName,
-            await Package.create(
-               this.environmentName,
-               packageName,
-               packagePath,
-               () => this.malloyConfig.malloyConfig,
-            ),
+
+      const packageMutex = this.getOrCreatePackageMutex(packageName);
+      if (packageMutex.isLocked()) {
+         logger.debug(
+            `Package ${packageName} is being loaded, waiting before addPackage...`,
          );
-      } catch (error) {
-         logger.error("Error adding package", { error });
-         this.deletePackageStatus(packageName);
-         throw error;
+         await packageMutex.waitForUnlock();
+         const alreadyLoaded = this.packages.get(packageName);
+         if (alreadyLoaded !== undefined) {
+            return alreadyLoaded;
+         }
       }
-      this.setPackageStatus(packageName, PackageStatus.SERVING);
-      return this.packages.get(packageName);
+
+      return packageMutex.runExclusive(async () => {
+         const existingPackage = this.packages.get(packageName);
+         if (existingPackage !== undefined) {
+            return existingPackage;
+         }
+
+         this.setPackageStatus(packageName, PackageStatus.LOADING);
+         try {
+            this.packages.set(
+               packageName,
+               await Package.create(
+                  this.environmentName,
+                  packageName,
+                  packagePath,
+                  () => this.malloyConfig.malloyConfig,
+               ),
+            );
+         } catch (error) {
+            logger.error("Error adding package", { error });
+            this.deletePackageStatus(packageName);
+            throw error;
+         }
+         this.setPackageStatus(packageName, PackageStatus.SERVING);
+         return this.packages.get(packageName);
+      });
    }
 
    private async writePackageManifest(

package/src/service/environment_store.spec.ts

@@ -355,6 +355,15 @@ describe("EnvironmentStore Service", () => {
       expect(projects.length).toBe(2);
       expect(projects.map((p) => p.name)).toContain(projectName1);
       expect(projects.map((p) => p.name)).toContain(projectName2);
+
+      // All envs initialized cleanly → status is "serving" (not
+      // "degraded") and there's no failedEnvironments key on the
+      // response. This is the happy-path companion to the
+      // "should skip a project with invalid startup connection config"
+      // test which exercises the degraded path.
+      const status = await newEnvironmentStore.getStatus();
+      expect(status.operationalState).toBe("serving");
+      expect(status.failedEnvironments).toBeUndefined();
    });
 
    it("should skip a project with invalid startup connection config", async () => {
@@ -427,6 +436,16 @@ describe("EnvironmentStore Service", () => {
       await expect(
          newEnvironmentStore.getEnvironment(invalidProjectName),
       ).rejects.toThrow();
+
+      // The skipped environment should surface in the status response
+      // so external callers (CI smoke tests, dashboards) can tell the
+      // server is only partially serving.
+      const status = await newEnvironmentStore.getStatus();
+      expect(status.operationalState).toBe("degraded");
+      expect(status.failedEnvironments).toBeDefined();
+      expect(status.failedEnvironments?.map((f) => f.name)).toContain(
+         invalidProjectName,
+      );
    });
 
    it("should handle project updates", async () => {