@malloy-publisher/server 0.0.235 → 0.0.236

package/src/service/environment.ts

@@ -1,2191 +0,0 @@
-import type { GivenValue, LogMessage } from "@malloydata/malloy";
-import { MalloyError, Runtime } from "@malloydata/malloy";
-import { publisherMeter } from "../telemetry";
-import { Mutex } from "async-mutex";
-import crypto from "crypto";
-import * as fs from "fs";
-import * as path from "path";
-import { pathToFileURL } from "url";
-import { components } from "../api";
-import {
-   API_PREFIX,
-   normalizeModelPath,
-   NOTEBOOK_FILE_SUFFIX,
-   README_NAME,
-} from "../constants";
-import {
-   BadRequestError,
-   ConnectionNotFoundError,
-   EnvironmentNotFoundError,
-   PackageNotFoundError,
-   ServiceUnavailableError,
-} from "../errors";
-import { getPersistStorageMode } from "../config";
-import { logger } from "../logger";
-import { redactPgSecrets } from "../pg_helpers";
-import { recordManifestBind } from "../materialization_metrics";
-import {
-   assertSafeEnvironmentPath,
-   assertSafePackageName,
-   assertSafeRelativeModelPath,
-   safeJoinUnderRoot,
-} from "../path_safety";
-import { FreshnessManifest, ManifestEntry } from "../storage/DatabaseInterface";
-import { URL_READER } from "../utils";
-import {
-   buildEnvironmentMalloyConfig,
-   deleteDuckLakeConnectionFile,
-   EnvironmentMalloyConfig,
-   InternalConnection,
-} from "./connection";
-import {
-   fetchManifestEntries,
-   splitManifestEntries,
-   type FetchedManifest,
-} from "./manifest_loader";
-import { ApiConnection } from "./model";
-import { Package } from "./package";
-import type { PackageMemoryGovernor } from "./package_memory_governor";
-
-/**
- * Sibling dirs under `environmentPath` used by the install/delete pipeline so
- * that long downloads do not hold the per-package mutex.
- *
- *  - `.staging/<pkg>-<uuid>/` — a download in progress. Renamed to the
- *    canonical path under the lock once complete.
- *  - `.retired/<pkg>-<uuid>/` — the previous canonical tree, atomically
- *    renamed out of the way during a swap or delete. `fs.rm`'d asynchronously
- *    after the lock is released.
- *
- * Both names start with a `.` so the package walkers (which use
- * {@link ignoreDotfiles}) skip them.
- */
-const STAGING_DIR_NAME = ".staging";
-const RETIRED_DIR_NAME = ".retired";
-
-// How long to wait for a control-plane manifest fetch during (re)bind before
-// giving up and serving live. Binding happens before a package is marked
-// SERVING, so an unreachable/slow manifest store must not block the package.
-const MANIFEST_FETCH_TIMEOUT_MS = 15_000;
-
-export enum PackageStatus {
-   LOADING = "loading",
-   SERVING = "serving",
-   UNLOADING = "unloading",
-}
-
-interface PackageInfo {
-   name: string;
-   loadTimestamp: number;
-   status: PackageStatus;
-}
-
-type ApiPackage = components["schemas"]["Package"];
-type ApiEnvironment = components["schemas"]["Environment"];
-type RetiredConnectionGeneration = {
-   label: string;
-   releaseConnections: () => Promise<void>;
-   timer?: ReturnType<typeof setTimeout>;
-};
-
-const RETIRED_CONNECTION_DRAIN_MS = 30_000;
-
-/**
- * Module-scoped admission-rejection counters. Lazy-initialized so
- * the OTel JS `ProxyMeter` cannot strand them on a NoOp instrument
- * created before the SDK MeterProvider was registered (a real risk
- * in unit tests; see comment in `query_timeout.ts`). Environment
- * name is attached as a label so dashboards can identify hot
- * environments without grepping logs.
- */
-import { type Counter } from "@opentelemetry/api";
-let queryAdmissionRejectionsCounter: Counter | null = null;
-let packageAdmissionRejectionsCounter: Counter | null = null;
-function getQueryAdmissionRejectionsCounter(): Counter {
-   if (queryAdmissionRejectionsCounter) return queryAdmissionRejectionsCounter;
-   queryAdmissionRejectionsCounter = publisherMeter().createCounter(
-      "publisher_query_admission_rejections_total",
-      {
-         description:
-            "Queries rejected with 503 because Environment.assertCanAdmitQuery() observed memory back-pressure",
-      },
-   );
-   return queryAdmissionRejectionsCounter;
-}
-function getPackageAdmissionRejectionsCounter(): Counter {
-   if (packageAdmissionRejectionsCounter) {
-      return packageAdmissionRejectionsCounter;
-   }
-   packageAdmissionRejectionsCounter = publisherMeter().createCounter(
-      "publisher_package_admission_rejections_total",
-      {
-         description:
-            "Package loads rejected with 503 because Environment.assertCanAdmitNewPackage() observed memory back-pressure",
-      },
-   );
-   return packageAdmissionRejectionsCounter;
-}
-
-/**
- * Visible for tests; production code never calls this. Resets the
- * lazy caches so a fresh MeterProvider can capture future writes.
- */
-export function resetAdmissionTelemetryForTesting(): void {
-   queryAdmissionRejectionsCounter = null;
-   packageAdmissionRejectionsCounter = null;
-}
-
-export class Environment {
-   private packages: Map<string, Package> = new Map();
-   // Lock ordering: connectionMutex (environment) MUST be acquired before any
-   // packageMutex. Connection updates may invalidate cached package
-   // MalloyConfigs and force reloads, so the environment lock is the outer one.
-   // Never acquire connectionMutex while holding a packageMutex — that's the
-   // AB/BA deadlock path.
-   private packageMutexes = new Map<string, Mutex>();
-   private packageStatuses: Map<string, PackageInfo> = new Map();
-   /**
-    * Configured packages that failed to load, keyed by name, with the reason.
-    *
-    * A load failure is not fatal: the package is omitted and its siblings serve
-    * on. It is also observable exactly once, because the failing load deletes
-    * the `packageStatuses` entry that `listPackages` enumerates, so the next
-    * listing no longer knows the package was ever configured. That makes this
-    * the only lasting record. Read by EnvironmentStore.getStatus.
-    */
-   private failedPackages: Map<string, string> = new Map();
-   /**
-    * Why a configured package never reached the disk, keyed by package name.
-    *
-    * Separate from {@link failedPackages} because it is the more specific
-    * answer and has to win. A package whose location failed to mount is still
-    * seeded SERVING by addEnvironment, so its later lazy load fails on the
-    * manifest that was never copied and `listPackages` records that instead.
-    * Reporting "Package manifest ... does not exist." for what was really a
-    * typo'd `location` sends the reader hunting in the wrong place.
-    */
-   private mountErrors: Map<string, string> = new Map();
-   private malloyConfig: EnvironmentMalloyConfig;
-   private connectionMutex = new Mutex();
-   private retiredConnectionGenerations =
-      new Set<RetiredConnectionGeneration>();
-   private apiConnections: ApiConnection[];
-   private environmentPath: string;
-   private environmentName: string;
-   // Resolves a package's latest persisted materialization manifest entries
-   // (the full map — colocated tableName entries AND `storage=` cross-connection
-   // entries), so serve routing for BOTH tiers is re-established when a package
-   // (re)loads — e.g. after a worker restart — instead of only when a build's
-   // auto-load runs. Injected by the EnvironmentStore, which owns the
-   // materialization repository. Undefined ⇒ no re-bind on load (routing then
-   // depends on a fresh build, the old behavior). See
-   // {@link rebindServeBindingsFromLocalStore}.
-   private storageBindingResolver?: (
-      packageName: string,
-   ) => Promise<Record<string, ManifestEntry>>;
-   public metadata: ApiEnvironment;
-   // The shared memory governor that consults process RSS. Optional —
-   // when null the gate is a no-op and the environment behaves exactly
-   // like it did before the governor was introduced. Set by
-   // EnvironmentStore.setMemoryGovernor at server start so we keep the
-   // governor as the single owner of the back-pressure boolean.
-   private memoryGovernor: PackageMemoryGovernor | null = null;
-
-   /** Absolute path on disk where this environment's package files live. */
-   public getEnvironmentPath(): string {
-      return this.environmentPath;
-   }
-
-   /** This environment's name (the canonical key used by the API/service). */
-   public getEnvironmentName(): string {
-      return this.environmentName;
-   }
-
-   constructor(
-      environmentName: string,
-      environmentPath: string,
-      malloyConfig: EnvironmentMalloyConfig,
-      apiConnections: InternalConnection[],
-   ) {
-      // Sanitizer barrier: every downstream `path.join(this.environmentPath,
-      // …)` site (including the static `sweepStaleInstallDirs` sweep) gets a
-      // value that has cleared an allowlist check at the gate.
-      assertSafeEnvironmentPath(environmentPath);
-      this.environmentName = environmentName;
-      this.environmentPath = environmentPath;
-      this.malloyConfig = malloyConfig;
-      this.apiConnections = apiConnections;
-      this.metadata = {
-         resource: `${API_PREFIX}/environments/${this.environmentName}`,
-         name: this.environmentName,
-         location: this.environmentPath,
-      };
-      void this.reloadEnvironmentMetadata();
-   }
-
-   private async writeEnvironmentReadme(readme?: string): Promise<void> {
-      if (readme === undefined) return;
-
-      const readmePath = path.join(this.environmentPath, "README.md");
-
-      try {
-         await fs.promises.writeFile(readmePath, readme, "utf-8");
-         logger.info(
-            `Updated README.md for environment ${this.environmentName}`,
-         );
-      } catch (err) {
-         logger.error(`Failed to write README.md`, { error: err });
-         throw new Error(`Failed to update environment README`);
-      }
-   }
-
-   public async update(payload: ApiEnvironment) {
-      if (payload.readme !== undefined) {
-         this.metadata.readme = payload.readme;
-         await this.writeEnvironmentReadme(payload.readme);
-      }
-
-      // Handle connections update
-      // TODO: Update environment connections should have its own API endpoint
-      if (payload.connections) {
-         const payloadConnections = payload.connections;
-         await this.runConnectionUpdateExclusive(async () => {
-            logger.info(
-               `Updating ${payloadConnections.length} connections for environment ${this.environmentName}`,
-            );
-            const isUpdateConnectionRequest = true;
-            const nextMalloyConfig = buildEnvironmentMalloyConfig(
-               payloadConnections,
-               this.environmentPath,
-               isUpdateConnectionRequest,
-            );
-
-            this.updateConnections(nextMalloyConfig);
-
-            logger.info(
-               `Successfully updated connections for environment ${this.environmentName}`,
-               {
-                  apiConnections: this.apiConnections.length,
-                  internalConnections: this.apiConnections.length,
-               },
-            );
-         });
-      }
-
-      return this;
-   }
-
-   static async create(
-      environmentName: string,
-      environmentPath: string,
-      connections: ApiConnection[],
-   ): Promise<Environment> {
-      assertSafeEnvironmentPath(environmentPath);
-      if (!(await fs.promises.stat(environmentPath))?.isDirectory()) {
-         throw new EnvironmentNotFoundError(
-            `Environment path ${environmentPath} not found`,
-         );
-      }
-
-      logger.info(`Creating environment with connection configuration`);
-      const malloyConfig = buildEnvironmentMalloyConfig(
-         connections,
-         environmentPath,
-      );
-
-      logger.info(
-         `Loaded ${malloyConfig.apiConnections.length} connections for environment ${environmentName}`,
-         {
-            connections: malloyConfig.apiConnections.map((c) => ({
-               name: c.name,
-               type: c.type,
-            })),
-         },
-      );
-
-      const environment = new Environment(
-         environmentName,
-         environmentPath,
-         malloyConfig,
-         malloyConfig.apiConnections,
-      );
-
-      // Best-effort: a previous run may have crashed mid-install or
-      // mid-delete and left orphan dirs under .staging/ or .retired/.
-      // Run against the validated constructor argument so the sink path
-      // here does NOT route through `this` (which CodeQL conservatively
-      // treats as tainted because other methods on this class touch
-      // request-derived `packageName` values).
-      await Environment.sweepStaleInstallDirs(environmentPath);
-
-      return environment;
-   }
-
-   public async reloadEnvironmentMetadata(): Promise<ApiEnvironment> {
-      let readme = "";
-      try {
-         readme = (
-            await fs.promises.readFile(
-               safeJoinUnderRoot(this.environmentPath, README_NAME),
-            )
-         ).toString();
-      } catch {
-         // Readme not found, so we'll just return an empty string
-      }
-      this.metadata = {
-         ...this.metadata,
-         resource: `${API_PREFIX}/environments/${this.environmentName}`,
-         name: this.environmentName,
-         readme,
-      };
-      return this.metadata;
-   }
-
-   public async compileSource(
-      packageName: string,
-      modelName: string,
-      source: string,
-      includeSql: boolean = false,
-      givens?: Record<string, GivenValue>,
-   ): Promise<{ problems: LogMessage[]; sql?: string }> {
-      assertSafePackageName(packageName);
-      assertSafeRelativeModelPath(modelName);
-      // /compile appends the submitted source to the TARGET MODEL's content for
-      // namespace context. A notebook (.malloynb) is markdown + cells, not a
-      // model, so compiling against it only yields a confusing parse error —
-      // reject it up front with an actionable message. (Notebooks remain public
-      // for discovery/query; this is specific to the compile context.)
-      if (modelName.endsWith(NOTEBOOK_FILE_SUFFIX)) {
-         throw new BadRequestError(
-            `Cannot compile against a notebook ("${modelName}"). ` +
-               `/compile takes a .malloy model path for namespace context.`,
-         );
-      }
-      // Hold the per-package mutex for the duration of every disk read —
-      // both the explicit `fs.readFile(modelPath)` below and the implicit
-      // import resolution that `runtime.loadModel` does through the URL
-      // reader. This is mutually exclusive with `installPackage`'s Phase 2
-      // rename swap and with `deletePackage`'s rename-to-retired, so a
-      // compile can never observe a half-rewritten tree. The slow Phase 1
-      // download happens outside this lock, so a multi-second clone does
-      // not block compiles.
-      return this.withPackageLock(packageName, async () => {
-         // Sanitized join: input segments are allowlisted above; the
-         // resolve-and-contain check here is the secondary guard CodeQL's
-         // path-injection sanitizer recognises.
-         const modelPath = safeJoinUnderRoot(
-            this.environmentPath,
-            packageName,
-            modelName,
-         );
-         // Place the virtual file in the model's directory so relative imports
-         // resolve correctly. Use `pathToFileURL` rather than hand-prefixing
-         // `file://`: on Windows the latter produces a malformed URL
-         // (`file://D:\Temp\…`) that round-trips differently than the URL the
-         // Malloy runtime synthesizes from the same path, breaking the
-         // intercepting reader's string comparison below and falling through
-         // to disk for a virtual file that doesn't exist.
-         const modelDir = path.dirname(modelPath);
-         const virtualUrl = pathToFileURL(
-            path.join(modelDir, "__compile_check.malloy"),
-         );
-         const virtualUri = virtualUrl.toString();
-
-         // Read the full model file so the submitted source inherits the model's
-         // complete namespace — imports, source definitions, queries, etc.
-         let modelContent = "";
-         try {
-            modelContent = await fs.promises.readFile(modelPath, "utf8");
-         } catch {
-            // If the model file can't be read, proceed with empty content
-            // and let compilation surface any errors naturally.
-         }
-         const fullSource = modelContent
-            ? `${modelContent}\n${source}`
-            : source;
-
-         // Create a URL Reader that serves the source string for the virtual file,
-         // but falls back to the disk for everything else (imports).
-         const interceptingReader = {
-            readURL: async (url: URL) => {
-               if (url.toString() === virtualUri) {
-                  return fullSource;
-               }
-               return URL_READER.readURL(url);
-            },
-         };
-
-         // Use the locked variant — we already hold the per-package mutex.
-         const pkg = await this._loadOrGetPackageLocked(packageName);
-
-         // Authorize gate: /compile is compile-only, but it can still act
-         // as a schema oracle (a denied caller learns a gated source's columns
-         // from compile errors) and, with includeSql, leak its SQL. Gate the
-         // named source the submitted text targets BEFORE compiling — mirrors
-         // the query path's early surface-syntax gate. Unnamed/inline source
-         // text resolves to undefined, so only the model-wide file-level gate
-         // applies. The gate runs against the package's cached Model (its
-         // `given:` block + authorize annotations), independent of the virtual
-         // compile below. If the model isn't loaded, there's nothing to enforce
-         // and compilation surfaces its own error.
-         const gateModel = pkg.getModel(modelName);
-         if (gateModel) {
-            // Query boundary first (the *what* axis): /compile compiles ad-hoc
-            // text against a model, so gate it like an ad-hoc query — a
-            // non-`explores` model file, or text whose surface-resolved target
-            // is a non-curated model source (under queryableSources:
-            // "declared"), is rejected with a generic 404 before compilation
-            // can leak schema/SQL. Text the early gate can't pin is settled by
-            // the compiled backstop below.
-            gateModel.assertQueryBoundaryEarly(undefined, undefined, source);
-            await gateModel.assertAuthorizedForText(source, givens ?? {});
-         }
-
-         // Initialize Runtime with the package's active MalloyConfig so compile
-         // checks see the same package-scoped duckdb as execution. This runtime
-         // borrows the package config; the package/environment lifecycle owns release.
-         // Thread the package's bound build manifest (when present) so the
-         // /compile preview routes persist sources to their materialized tables
-         // exactly like execution does — otherwise includeSql=true would always
-         // show base-table SQL and diverge from what executeQuery actually runs.
-         const boundManifestEntries = pkg.getBuildManifestEntries();
-         const runtime = new Runtime({
-            urlReader: interceptingReader,
-            config: pkg.getMalloyConfig(),
-            buildManifest: boundManifestEntries
-               ? { entries: boundManifestEntries, strict: false }
-               : undefined,
-         });
-
-         // Attempt to compile
-         try {
-            const modelMaterializer = runtime.loadModel(virtualUrl);
-            const model = await modelMaterializer.getModel();
-
-            // Resolve the final query's materializer once (if there is one).
-            let queryMaterializer: ReturnType<
-               typeof modelMaterializer.loadFinalQuery
-            > | null = null;
-            try {
-               queryMaterializer = modelMaterializer.loadFinalQuery();
-            } catch {
-               // No runnable query (e.g. only source definitions) — nothing to
-               // gate or extract beyond the early text gate already applied.
-            }
-
-            // Compiled-source backstops — run REGARDLESS of includeSql. They
-            // gate the source the COMPILED final query actually reads, closing
-            // named-query / multi-statement indirection the early surface-syntax
-            // gate misses (e.g. `run: ungated\nrun: gated` — the early gate only
-            // matches the FIRST `run:`, but the LAST statement is what executes).
-            // Compiling a gated source even without SQL is a schema oracle
-            // (field-not-found errors leak its columns), so this must not be
-            // conditional on SQL extraction. (A `source: x is gated` alias makes
-            // a new ungated source — that's the documented "extend doesn't
-            // inherit authorize" footgun, the same as the query path.)
-
-            // Boundary backstop (the *what* axis, 404) before the authorize
-            // one (the *who* axis, 403). /compile text is always ad-hoc — the
-            // early gate can only positively deny, never fully clear — so the
-            // compiled final query's run target is the authority. Self-gates
-            // internally (no-ops when the boundary is inert: "all" / no
-            // explores), so it is deliberately NOT guarded by hasAuthorize().
-            // Text that compiles only source definitions (no final query) has
-            // no run target and nothing to gate.
-            if (queryMaterializer && gateModel) {
-               await gateModel.assertQueryBoundaryForRunnable(
-                  queryMaterializer,
-                  source,
-               );
-            }
-
-            // Authorize backstop (the *who* axis, 403). NOT guarded by
-            // hasAuthorize(): that only inspects top-level modelDef.contents
-            // sources, so a gated source reached only via a cross-file/deep
-            // join is invisible to it and this backstop would silently never
-            // run for such a model — the same bypass assertAuthorizedForRunnable
-            // itself closes on the query path (see model.ts
-            // assertAuthorizedForAllSources). The own-source probe and joined-
-            // gate walk it runs are cheap no-ops for a genuinely ungated model.
-            if (queryMaterializer && gateModel) {
-               await gateModel.assertAuthorizedForRunnable(
-                  queryMaterializer,
-                  givens ?? {},
-               );
-            }
-
-            // If includeSql is requested and compilation succeeded, attempt to extract SQL
-            let sql: string | undefined;
-            if (includeSql && queryMaterializer) {
-               try {
-                  sql = await queryMaterializer.getSQL({ givens });
-               } catch (error) {
-                  // A bad caller given (unknown name, wrong-typed value, finalized
-                  // override, ...) surfaces as a Malloy `runtime-given-*` error.
-                  // Map it to a 400 rather than silently omitting `sql` (which is
-                  // indistinguishable from "no runnable query"). Duck-type on
-                  // `.code`; let a MalloyError fall to the outer catch → problems.
-                  // The `runtime-given-` prefix is pinned to Malloy's error codes
-                  // (given_binding.ts / runtime.ts, same as model.ts) — if they're
-                  // renamed upstream a bad given would silently revert to the omit
-                  // branch below, so keep the two in sync.
-                  const givenCode = (error as { code?: string })?.code;
-                  if (
-                     typeof givenCode === "string" &&
-                     givenCode.startsWith("runtime-given-")
-                  ) {
-                     throw new BadRequestError(
-                        error instanceof Error ? error.message : String(error),
-                     );
-                  }
-                  if (error instanceof MalloyError) {
-                     throw error;
-                  }
-                  // Otherwise the source may just not contain a runnable query
-                  // (e.g. only source definitions) — omit the sql field.
-               }
-            }
-
-            // If successful, return any non-fatal warnings
-            return { problems: model.problems, sql };
-         } catch (error) {
-            // If parsing/compilation fails, return the errors
-            if (error instanceof MalloyError) {
-               return { problems: error.problems };
-            }
-            // If it's a system error (e.g. file not found), throw it up
-            throw error;
-         }
-      });
-   }
-
-   public listApiConnections(): ApiConnection[] {
-      return this.apiConnections;
-   }
-
-   public getApiConnection(connectionName: string): ApiConnection {
-      const connection = this.apiConnections.find(
-         (connection) => connection.name === connectionName,
-      );
-      if (!connection) {
-         throw new ConnectionNotFoundError(
-            `Connection ${connectionName} not found`,
-         );
-      }
-      return connection;
-   }
-
-   public async getMalloyConnection(connectionName: string) {
-      return this.malloyConfig.malloyConfig.connections.lookupConnection(
-         connectionName,
-      );
-   }
-
-   public getEnvironmentMalloyConfig() {
-      return this.malloyConfig.malloyConfig;
-   }
-
-   public async runConnectionUpdateExclusive<T>(
-      fn: () => Promise<T>,
-   ): Promise<T> {
-      return this.connectionMutex.runExclusive(fn);
-   }
-
-   private retireConnectionGeneration(
-      label: string,
-      releaseConnections: () => Promise<void>,
-   ): void {
-      const generation: RetiredConnectionGeneration = {
-         label,
-         releaseConnections,
-      };
-      generation.timer = setTimeout(() => {
-         void this.releaseRetiredConnectionGeneration(generation);
-      }, RETIRED_CONNECTION_DRAIN_MS);
-      (
-         generation.timer as ReturnType<typeof setTimeout> & {
-            unref?: () => void;
-         }
-      ).unref?.();
-      this.retiredConnectionGenerations.add(generation);
-   }
-
-   private async releaseRetiredConnectionGeneration(
-      generation: RetiredConnectionGeneration,
-   ): Promise<void> {
-      if (!this.retiredConnectionGenerations.delete(generation)) return;
-
-      if (generation.timer) {
-         clearTimeout(generation.timer);
-      }
-
-      try {
-         await generation.releaseConnections();
-      } catch (error) {
-         logger.error(
-            `Error releasing retired connection generation ${generation.label}`,
-            { error },
-         );
-      }
-   }
-
-   private async releaseAllRetiredConnectionGenerations(): Promise<void> {
-      await Promise.all(
-         [...this.retiredConnectionGenerations].map((generation) =>
-            this.releaseRetiredConnectionGeneration(generation),
-         ),
-      );
-   }
-
-   /**
-    * Snapshot of the packages currently loaded in memory (does not trigger a
-    * load or reload). Used by the standalone materialization scheduler to sweep
-    * only already-loaded packages — a not-yet-loaded package is simply not
-    * scheduled until something else loads it, so the scheduler never forces a
-    * load of its own.
-    */
-   public getLoadedPackages(): Package[] {
-      return [...this.packages.values()];
-   }
-
-   public async listPackages(): Promise<ApiPackage[]> {
-      logger.debug("Listing packages", {
-         environmentPath: this.environmentPath,
-      });
-      try {
-         const packageMetadata = await Promise.all(
-            Array.from(this.packageStatuses.keys()).map(async (packageName) => {
-               try {
-                  const packageMetadata = (
-                     this.packageStatuses.get(packageName)?.status ===
-                     PackageStatus.LOADING
-                        ? undefined
-                        : await this.getPackage(packageName, false)
-                  )?.getPackageMetadata();
-                  if (packageMetadata) {
-                     packageMetadata.name = packageName;
-                  }
-                  return packageMetadata;
-               } catch (error) {
-                  logger.error(
-                     `Failed to load package: ${packageName} due to : ${error}`,
-                  );
-                  // Directory did not contain a valid package.json file -- therefore, it's not a package.
-                  // Or it timed out
-                  // Redact before this reaches getStatus: compiling a model
-                  // resolves the package's connections, so a Postgres/DuckLake
-                  // ATTACH failure surfaces here carrying the connection string
-                  // (connection.ts builds it, and redacts it before logging for
-                  // the same reason). A log line was the old destination; this
-                  // one is an HTTP response body.
-                  //
-                  // redactPgSecrets covers both keyword-form `password=` (what
-                  // buildPgConnectionString emits) and URI userinfo
-                  // (`postgres://user:pass@host`), so a URL-form connectionString
-                  // supplied verbatim in config is redacted here too.
-                  this.failedPackages.set(
-                     packageName,
-                     redactPgSecrets(
-                        error instanceof Error ? error.message : String(error),
-                     ),
-                  );
-                  return undefined;
-               }
-            }),
-         );
-         // Get rid of undefined entries (i.e, directories without publisher.json files).
-         const filteredMetadata = packageMetadata.filter(
-            (metadata) => metadata,
-         ) as ApiPackage[];
-
-         // Filter out packages that are being unloaded
-         const finalMetadata = filteredMetadata.filter((metadata) => {
-            const packageStatus = this.packageStatuses.get(metadata.name || "");
-            return packageStatus?.status !== PackageStatus.UNLOADING;
-         });
-
-         return finalMetadata;
-      } catch (error) {
-         logger.error("Error listing packages", { error });
-         console.error(error);
-         throw error;
-      }
-   }
-
-   /**
-    * One mutex per package name; never replace after create — replacing
-    * would allow two loads of the same package to run in parallel and
-    * race on the canonical directory.
-    *
-    * `deletePackage` intentionally leaves the entry behind: a
-    * subsequent re-install must serialize against any straggling
-    * readers from the deleted generation that are still inside
-    * `withPackageLock`. The map therefore grows by the count of
-    * *distinct* package names the environment has ever served, not by
-    * install churn, so for the publisher's expected workload
-    * (config-declared packages, occasional ad-hoc additions) this is
-    * bounded in practice. Long-lived deployments that create and
-    * delete unique package names indefinitely would need an explicit
-    * sweep; we'll add one if/when that pattern appears.
-    */
-   private getOrCreatePackageMutex(packageName: string): Mutex {
-      let packageMutex = this.packageMutexes.get(packageName);
-      if (packageMutex === undefined) {
-         packageMutex = new Mutex();
-         this.packageMutexes.set(packageName, packageMutex);
-      }
-      return packageMutex;
-   }
-
-   /**
-    * Run `fn` while holding the per-package mutex. This is the single
-    * synchronization primitive that protects a package directory: every
-    * code path that mutates `{environmentPath}/{packageName}/` or reads
-    * from disk under it must serialize through this lock. See the lock
-    * ordering note above the `packageMutexes` field for the wider
-    * invariant.
-    *
-    * `async-mutex` is **not reentrant** — `fn` must not call any other
-    * method that calls `withPackageLock` on the same package, or it will
-    * deadlock. Use the `_xxxLocked` variants below in that case.
-    */
-   public async withPackageLock<T>(
-      packageName: string,
-      fn: () => Promise<T>,
-   ): Promise<T> {
-      assertSafePackageName(packageName);
-      return this.getOrCreatePackageMutex(packageName).runExclusive(fn);
-   }
-
-   private allocateStagingPath(packageName: string): string {
-      return safeJoinUnderRoot(
-         this.environmentPath,
-         STAGING_DIR_NAME,
-         `${packageName}-${crypto.randomUUID()}`,
-      );
-   }
-
-   private allocateRetiredPath(packageName: string): string {
-      return safeJoinUnderRoot(
-         this.environmentPath,
-         RETIRED_DIR_NAME,
-         `${packageName}-${crypto.randomUUID()}`,
-      );
-   }
-
-   /**
-    * Best-effort sweep of `.staging/` and `.retired/` left over from a
-    * previous run (crash, OOM, etc). Safe because both dirs are managed
-    * exclusively by `installPackage` / `deletePackage`; no in-flight
-    * operation in this process can be using them yet.
-    *
-    * Static + path-as-parameter on purpose: the sink path here must
-    * derive from the validated factory argument, not from `this`,
-    * because CodeQL's path-injection query conservatively treats every
-    * field on this class as tainted (other methods on the same class
-    * receive request-derived `packageName` values).
-    */
-   public static async sweepStaleInstallDirs(
-      environmentPath: string,
-   ): Promise<void> {
-      assertSafeEnvironmentPath(environmentPath);
-      for (const dirName of [STAGING_DIR_NAME, RETIRED_DIR_NAME]) {
-         const dir = safeJoinUnderRoot(environmentPath, dirName);
-         // Inline sanitizer barriers in the precise shape CodeQL's
-         // `js/path-injection` query recognises (regex-test +
-         // `indexOf("..") !== -1` guard) so the sink right below is
-         // covered even when the call chain feeding `environmentPath`
-         // is taint-tracked from an HTTP request handler.
-         if (dir.indexOf("..") !== -1) continue;
-         if (path.basename(dir) !== dirName) continue;
-         try {
-            await fs.promises.rm(dir, { recursive: true, force: true });
-         } catch (err) {
-            logger.warn(`Failed to sweep stale ${dirName} dir at ${dir}`, {
-               error: err,
-            });
-         }
-      }
-   }
-
-   /**
-    * Attach (or detach with `null`) the memory governor that gates new
-    * package allocations. The single instance is owned by the
-    * EnvironmentStore and propagated to every Environment so the
-    * back-pressure decision is process-wide.
-    */
-   public setMemoryGovernor(governor: PackageMemoryGovernor | null): void {
-      this.memoryGovernor = governor;
-   }
-
-   /**
-    * Inject the resolver that fetches a package's latest persisted
-    * materialization entries — both tiers (see {@link storageBindingResolver}).
-    */
-   public setStorageBindingResolver(
-      resolver: (packageName: string) => Promise<Record<string, ManifestEntry>>,
-   ): void {
-      this.storageBindingResolver = resolver;
-   }
-
-   /**
-    * Re-establish a package's serve routing from its latest persisted
-    * materialization when it (re)loads, so serving survives a restart — both
-    * tiers, not just one. Serve bindings are otherwise in-memory, set only by a
-    * build's post-run auto-load, so a worker restart silently reverted a
-    * materialized source to serving live until the next build. Runs beside
-    * {@link bindManifestIfConfigured} — the same "bind serve state on load" step.
-    * Best-effort: a lookup failure logs and leaves the package serving live (a
-    * subsequent build will bind it).
-    *
-    * Both tiers are restored from the SAME persisted manifest, split by
-    * {@link splitManifestEntries}, mirroring {@link bindManifest}:
-    *  - **colocated** (same-connection) → {@link Package.bindColocatedServeManifest},
-    *    applied as a per-query `buildManifest` override at serve time (no
-    *    recompile — the load-time compile already produced flag-carrying models).
-    *    Independent of `PERSIST_STORAGE_MODE`: colocated is the v0 path and is
-    *    not gated by the storage kill switch (a plain `#@ persist` materializes
-    *    and serves even when the tier is `off`), so its routing is restored
-    *    regardless of mode.
-    *  - **storage=** (cross-connection) → {@link Package.bindStorageServeBindings},
-    *    the virtual-source transform. Restored only when the tier is not `off`:
-    *    storage serve routing requires the tier on, so an off deployment skips it.
-    *
-    * Skipped entirely when the package has a bound `manifestLocation`: the two
-    * binding producers (this local store and the host's fetched manifest, applied
-    * by {@link bindManifest}) are mutually exclusive by manifest presence, and the
-    * host is authoritative when one is set. An orchestrated publisher STILL
-    * persists its own local materialization records (the host triggers builds
-    * through it), so without this guard the local rebind — which runs AFTER
-    * {@link bindManifestIfConfigured} on load — would overwrite the host's
-    * bindings with a possibly-staler local generation.
-    */
-   private async rebindServeBindingsFromLocalStore(
-      pkg: Package,
-   ): Promise<void> {
-      if (!this.storageBindingResolver) return;
-      // Host-authoritative: a bound manifestLocation means bindManifest already
-      // supplied both tiers' serve bindings from the host's manifest; the local
-      // store must not clobber them (mutually-exclusive binding sources).
-      if (pkg.getPackageMetadata().manifestLocation) return;
-      const packageName = pkg.getPackageName();
-      try {
-         const rawEntries = await this.storageBindingResolver(packageName);
-         if (Object.keys(rawEntries).length === 0) return;
-         const { tableNameManifest, storageEntries } = splitManifestEntries(
-            rawEntries,
-            `local store (package ${packageName})`,
-         );
-         // Colocated: restore regardless of PERSIST_STORAGE_MODE (v0 path, not
-         // gated by the storage kill switch).
-         if (Object.keys(tableNameManifest).length > 0) {
-            pkg.bindColocatedServeManifest(tableNameManifest);
-         }
-         // Storage=: only meaningful when the tier is not off (serve routing to
-         // the external store requires it). Ships dark otherwise.
-         if (
-            getPersistStorageMode() !== "off" &&
-            Object.keys(storageEntries).length > 0
-         ) {
-            pkg.bindStorageServeBindings(storageEntries);
-         }
-      } catch (err) {
-         logger.warn(
-            "Failed to rebind serve bindings from local store on load",
-            {
-               packageName,
-               error: err instanceof Error ? err.message : String(err),
-            },
-         );
-      }
-   }
-
-   /**
-    * Choke-point check called from every code path that would allocate
-    * a *new* package into the in-memory map (lazy load on cache miss,
-    * explicit reload, `addPackage`). Throws HTTP 503 when the governor
-    * is back-pressured; cheap no-op when the governor is unset or
-    * happy.
-    *
-    * `allowAdmission` is the documented opt-out for read paths that
-    * genuinely cannot tolerate 503s. None of the current callers set
-    * it; the parameter exists so a future caller (e.g. a
-    * health/warmup probe) can self-document its bypass intent.
-    */
-   private assertCanAdmitNewPackage(
-      packageName: string,
-      reason: string,
-      allowAdmission: boolean,
-   ): void {
-      if (allowAdmission) return;
-      if (!this.memoryGovernor?.isBackpressured()) return;
-      // Increment *before* throwing so the metric ticks even on
-      // the not-uncommon "caught and swallowed" path. The label
-      // shape mirrors `assertCanAdmitQuery` so a dashboard panel
-      // can sum both rejection kinds by environment.
-      getPackageAdmissionRejectionsCounter().add(1, {
-         environment: this.environmentName,
-         reason,
-      });
-      throw new ServiceUnavailableError(
-         `Publisher is under memory pressure and cannot ${reason} (package "${packageName}", environment "${this.environmentName}"). Retry after the server's memory usage drops below the low-water mark (PUBLISHER_MEMORY_LOW_WATER_FRACTION of PUBLISHER_MAX_MEMORY_BYTES), or raise PUBLISHER_MAX_MEMORY_BYTES if you have headroom.`,
-      );
-   }
-
-   /**
-    * Reject incoming queries with HTTP 503 when the memory governor
-    * has tripped its high-water mark. Used by every query controller
-    * (connection SQL, model query, notebook cell, MCP `execute_query`)
-    * to shed load before the query runs — complementing
-    * {@link assertCanAdmitNewPackage}, which only fires on cache-miss
-    * package loads and so leaves already-loaded packages fully
-    * queryable under pressure. With this in place, "back-pressured"
-    * means "no new work of any kind" until the governor's low-water
-    * mark is crossed.
-    *
-    * Cheap O(1) boolean read; no allocation when happy.
-    */
-   public assertCanAdmitQuery(): void {
-      if (!this.memoryGovernor?.isBackpressured()) return;
-      // Tick first so the counter reflects every rejection even
-      // when the controller's catch block swallows the error (e.g.
-      // an MCP tool surfaces it as a content payload rather than
-      // letting it bubble to the HTTP error mapper).
-      getQueryAdmissionRejectionsCounter().add(1, {
-         environment: this.environmentName,
-      });
-      throw new ServiceUnavailableError(
-         `Publisher is under memory pressure and cannot accept new queries (environment "${this.environmentName}"). Retry after the server's memory usage drops below the low-water mark (PUBLISHER_MEMORY_LOW_WATER_FRACTION of PUBLISHER_MAX_MEMORY_BYTES), or raise PUBLISHER_MAX_MEMORY_BYTES if you have headroom.`,
-      );
-   }
-
-   public async getPackage(
-      packageName: string,
-      reload: boolean = false,
-      options: { allowAdmission?: boolean } = {},
-   ): Promise<Package> {
-      assertSafePackageName(packageName);
-      // Fast-path: serve from cache without acquiring the lock. Safe because
-      // `Package` references are immutable; the disk-reading methods that
-      // actually need protection (compileSource, getModelFileText,
-      // reloadAllModelsForPackage, ...) acquire the lock themselves.
-      //
-      // INVARIANT: callers that consume the returned Package on the fast
-      // path (notably the MCP query tools and Model.getModel()) must
-      // remain in-memory only. If any code reachable from a `Package`
-      // method ever grows new disk I/O against the canonical tree, that
-      // path needs to be bracketed by `withPackageLock`; otherwise a
-      // concurrent install/delete will race against an unlocked reader.
-      const _package = this.packages.get(packageName);
-      if (_package !== undefined && !reload) {
-         return _package;
-      }
-
-      // We are either reloading or about to lazy-load on a cache miss
-      // — both allocate a new package. This is the single choke point
-      // for admission control; controllers no longer need their own
-      // back-pressure check.
-      this.assertCanAdmitNewPackage(
-         packageName,
-         reload ? "reload a package" : "load a package",
-         options.allowAdmission === true,
-      );
-
-      return this.withPackageLock(packageName, () =>
-         this._loadOrGetPackageLocked(packageName, reload),
-      );
-   }
-
-   /**
-    * Load (or reload) a package from its canonical disk location. Assumes
-    * the caller holds the per-package mutex (via {@link withPackageLock}).
-    *
-    * Used by {@link getPackage} and by {@link compileSource} so the
-    * cache-miss path doesn't re-enter the mutex.
-    */
-   private async _loadOrGetPackageLocked(
-      packageName: string,
-      reload: boolean = false,
-   ): Promise<Package> {
-      const existingPackage = this.packages.get(packageName);
-      if (existingPackage !== undefined && !reload) {
-         return existingPackage;
-      }
-
-      this.setPackageStatus(packageName, PackageStatus.LOADING);
-
-      try {
-         logger.debug(`Loading package ${packageName}...`);
-         const packagePath = safeJoinUnderRoot(
-            this.environmentPath,
-            packageName,
-         );
-         const _package = await Package.create(
-            this.environmentName,
-            packageName,
-            packagePath,
-            () => this.malloyConfig.malloyConfig,
-         );
-         await this.bindManifestIfConfigured(_package);
-         await this.rebindServeBindingsFromLocalStore(_package);
-         if (existingPackage !== undefined && reload) {
-            this.retireConnectionGeneration(`package ${packageName}`, () =>
-               existingPackage.getMalloyConfig().shutdown("close"),
-            );
-         }
-         this.packages.set(packageName, _package);
-         this.setPackageStatus(packageName, PackageStatus.SERVING);
-         // It loaded, so any earlier failure is stale. A package that failed at
-         // boot can be fixed on disk and reloaded without a restart.
-         this.clearPackageLoadFailure(packageName);
-         logger.debug(`Successfully loaded package ${packageName}`);
-
-         return _package;
-      } catch (error) {
-         logger.error(`Failed to load package ${packageName}`, { error });
-         if (existingPackage !== undefined && reload) {
-            // A failed RELOAD must not take down a package that is already
-            // serving. The compiled model in `packages` is still the last good
-            // one (it is only replaced on success), so keep serving it and let
-            // the caller surface the error instead of evicting the package and
-            // leaving the environment with nothing to answer from.
-            this.setPackageStatus(packageName, PackageStatus.SERVING);
-         } else {
-            this.packages.delete(packageName);
-            this.packageStatuses.delete(packageName);
-         }
-         throw error;
-      }
-   }
-
-   public async addPackage(
-      packageName: string,
-      options: { allowAdmission?: boolean } = {},
-   ) {
-      assertSafePackageName(packageName);
-      const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
-      if (
-         !(await fs.promises
-            .access(packagePath)
-            .then(() => true)
-            .catch(() => false)) ||
-         !(await fs.promises.stat(packagePath))?.isDirectory()
-      ) {
-         throw new PackageNotFoundError(`Package ${packageName} not found`);
-      }
-      // 404 takes precedence over 503 so a permanent "you forgot to
-      // upload the package" failure isn't masked as a transient
-      // "retry later" — the gate runs after the existence check.
-      this.assertCanAdmitNewPackage(
-         packageName,
-         "add a new package",
-         options.allowAdmission === true,
-      );
-      logger.info(
-         `Adding package ${packageName} to environment ${this.environmentName}`,
-         {
-            packagePath,
-         },
-      );
-
-      return this.withPackageLock(packageName, () =>
-         this._addPackageLocked(packageName),
-      );
-   }
-
-   private async _addPackageLocked(
-      packageName: string,
-   ): Promise<Package | undefined> {
-      const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
-      const existingPackage = this.packages.get(packageName);
-      if (existingPackage !== undefined) {
-         return existingPackage;
-      }
-
-      this.setPackageStatus(packageName, PackageStatus.LOADING);
-      try {
-         this.packages.set(
-            packageName,
-            await Package.create(
-               this.environmentName,
-               packageName,
-               packagePath,
-               () => this.malloyConfig.malloyConfig,
-            ),
-         );
-      } catch (error) {
-         logger.error("Error adding package", { error });
-         this.deletePackageStatus(packageName);
-         throw error;
-      }
-      this.setPackageStatus(packageName, PackageStatus.SERVING);
-      // Same reasoning as the load and install paths: it is serving now, so an
-      // earlier boot failure is stale. Without this, a package fixed on disk
-      // and re-added keeps its loadError for the life of the process.
-      this.clearPackageLoadFailure(packageName);
-      return this.packages.get(packageName);
-   }
-
-   /**
-    * Replace a package on disk via stage-and-swap, then load it.
-    *
-    *  - Phase 1 (no lock): run `downloader(stagingPath)`, writing the new
-    *    content into a fresh sibling dir at `.staging/<pkg>-<uuid>/`. This
-    *    is where multi-second downloads (git clone, GCS pull, ...) happen.
-    *  - Phase 2 (lock held): atomically rename any existing canonical tree
-    *    out to `.retired/<pkg>-<uuid>/`, rename staging into the canonical
-    *    path, and run `Package.create` against the canonical path.
-    *  - Phase 3 (after lock release): retire the old package's connections
-    *    via the existing 30s drain and `fs.rm` the retired tree.
-    *
-    * Concurrent compiles / `getModelFileText` / `reloadAllModels` calls
-    * take the same mutex and so are mutually exclusive with the Phase 2
-    * swap, but they never queue behind a long Phase 1 download.
-    *
-    * On failure (Phase 1 download or Phase 2 `Package.create`), the staging
-    * dir is removed and — if we already renamed the old tree aside — the
-    * old tree is renamed back so the canonical path is restored.
-    */
-   public async installPackage(
-      packageName: string,
-      downloader: (stagingPath: string) => Promise<void>,
-      validate?: (pkg: Package) => string | undefined,
-   ): Promise<Package> {
-      assertSafePackageName(packageName);
-      const stagingPath = this.allocateStagingPath(packageName);
-      await fs.promises.mkdir(path.dirname(stagingPath), { recursive: true });
-
-      logger.debug("install.phase1.download.started", {
-         environmentName: this.environmentName,
-         packageName,
-         stagingPath,
-      });
-      const downloadStartedAt = performance.now();
-      try {
-         await downloader(stagingPath);
-      } catch (err) {
-         await fs.promises
-            .rm(stagingPath, { recursive: true, force: true })
-            .catch(() => {});
-         throw err;
-      }
-      logger.debug("install.phase1.download.completed", {
-         environmentName: this.environmentName,
-         packageName,
-         durationMs: performance.now() - downloadStartedAt,
-      });
-
-      return this.withPackageLock(packageName, async () => {
-         logger.debug("install.phase2.swap.started", {
-            environmentName: this.environmentName,
-            packageName,
-         });
-         const canonicalPath = safeJoinUnderRoot(
-            this.environmentPath,
-            packageName,
-         );
-         let retiredPath: string | undefined;
-
-         const oldPackage = this.packages.get(packageName);
-         const oldExistsOnDisk = await fs.promises
-            .access(canonicalPath)
-            .then(() => true)
-            .catch(() => false);
-
-         if (oldExistsOnDisk) {
-            retiredPath = this.allocateRetiredPath(packageName);
-            await fs.promises.mkdir(path.dirname(retiredPath), {
-               recursive: true,
-            });
-            await fs.promises.rename(canonicalPath, retiredPath);
-            logger.debug("install.phase2.retired_old", {
-               environmentName: this.environmentName,
-               packageName,
-               retiredPath,
-            });
-         }
-
-         let newPackage: Package;
-         try {
-            await fs.promises.rename(stagingPath, canonicalPath);
-
-            this.setPackageStatus(packageName, PackageStatus.LOADING);
-            newPackage = await Package.create(
-               this.environmentName,
-               packageName,
-               canonicalPath,
-               () => this.malloyConfig.malloyConfig,
-               // This tree was just staged into place by the rename above, so a
-               // failed load leaves a half-built directory that is ours to
-               // remove; the rollback below restores the previous one.
-               true,
-            );
-            // Strict-reject hook (publish/update only — reload passes no
-            // validator and stays fail-safe). Throw INSIDE the try so the
-            // catch below rolls the swap back: the just-installed tree is
-            // wiped and the retired tree (if any) is restored, so a rejected
-            // publish/update never leaves the bad tree served on disk.
-            const validationMsg = validate?.(newPackage);
-            if (validationMsg) {
-               throw new BadRequestError(validationMsg);
-            }
-            logger.debug("install.phase2.committed", {
-               environmentName: this.environmentName,
-               packageName,
-               canonicalPath,
-            });
-         } catch (err) {
-            // Rollback: clobber whatever (partial) content sits at canonical
-            // — Package.create's own failure-cleanup may have already rm'd
-            // the directory, so the most common outcome here is ENOENT.
-            // `force: true` plus the `.catch(() => {})` make this a
-            // best-effort wipe whose only job is to leave the rename-back
-            // below a clean destination. Then put the old tree back if we
-            // moved one aside.
-            await fs.promises
-               .rm(canonicalPath, { recursive: true, force: true })
-               .catch(() => {});
-            let restored = false;
-            if (retiredPath) {
-               try {
-                  await fs.promises.rename(retiredPath, canonicalPath);
-                  restored = true;
-               } catch (restoreErr) {
-                  logger.error(
-                     "Failed to restore retired package after install rollback",
-                     {
-                        error: restoreErr,
-                        retiredPath,
-                        canonicalPath,
-                     },
-                  );
-               }
-            }
-            await fs.promises
-               .rm(stagingPath, { recursive: true, force: true })
-               .catch(() => {});
-            if (oldPackage && restored) {
-               // The rollback put the old tree back and the previous package is
-               // still in `this.packages` (it is only replaced on success
-               // below), so it is genuinely still serving. Deleting its status
-               // would strand it: listPackages enumerates packageStatuses, so
-               // the package would answer getPackage while being invisible to
-               // listings and discovery until a restart.
-               this.setPackageStatus(packageName, PackageStatus.SERVING);
-            } else {
-               // Either there was nothing to fall back to (a first install), or
-               // the restore did not happen: the rename-back threw, or the old
-               // tree was never on disk to retire. The canonical path is then
-               // missing or still holds the rejected content, so the cached
-               // package no longer matches disk and must not be advertised as
-               // serving. Drop both, and keep the two maps agreeing.
-               this.deletePackageStatus(packageName);
-               if (oldPackage) {
-                  // Retire before dropping it, the same way every other eviction
-                  // here does: once it leaves this.packages, closeAllConnections
-                  // can no longer reach its MalloyConfig, so its native handles
-                  // would never be released. Retire rather than shut down
-                  // inline, because withPackageLock does not cover queries that
-                  // already took this Package from an earlier getPackage; the
-                  // drain lets those finish first.
-                  this.retireConnectionGeneration(
-                     `package ${packageName}`,
-                     () => oldPackage.getMalloyConfig().shutdown("close"),
-                  );
-               }
-               this.packages.delete(packageName);
-            }
-            logger.debug("install.phase2.rollback", {
-               environmentName: this.environmentName,
-               packageName,
-               restored,
-               errorName: err instanceof Error ? err.name : "Unknown",
-            });
-            throw err;
-         }
-
-         // Best-effort manifest bind happens after the swap commits, outside the
-         // rollback window: a manifest that can't be fetched must not undo an
-         // otherwise-successful install (the package serves live instead).
-         await this.bindManifestIfConfigured(newPackage);
-         await this.rebindServeBindingsFromLocalStore(newPackage);
-
-         this.packages.set(packageName, newPackage);
-         this.setPackageStatus(packageName, PackageStatus.SERVING);
-         // Publishing a fixed package clears the boot failure it replaces.
-         this.clearPackageLoadFailure(packageName);
-
-         if (oldPackage) {
-            this.retireConnectionGeneration(`package ${packageName}`, () =>
-               oldPackage.getMalloyConfig().shutdown("close"),
-            );
-         }
-
-         if (retiredPath) {
-            const pathToClean = retiredPath;
-            setImmediate(() => {
-               logger.debug("install.phase3.retired_cleanup", {
-                  environmentName: this.environmentName,
-                  packageName,
-                  retiredPath: pathToClean,
-               });
-               void fs.promises
-                  .rm(pathToClean, { recursive: true, force: true })
-                  .catch((err) => {
-                     logger.warn(
-                        `Failed to clean up retired package directory ${pathToClean}`,
-                        { error: err },
-                     );
-                  });
-            });
-         }
-
-         return newPackage;
-      });
-   }
-
-   /**
-    * Reload every model in a package against the supplied build manifest,
-    * holding the per-package mutex for the duration of the disk reads.
-    * Replaces direct `Package.reloadAllModels` calls from outside
-    * `Environment`.
-    *
-    * Skips the recompile when there is nothing to substitute now AND nothing was
-    * substituted before — the same condition {@link bindManifest} applies to a
-    * pure-storage manifest flip, and for the same reason: a same-connection
-    * `tableName` manifest is resolved at COMPILE time, so an empty one over an
-    * already-empty one changes nothing a recompile could express. The previous
-    * state has to be checked too, because a package that just dropped its last
-    * colocated entry still has to recompile to revert the substitution.
-    *
-    * This matters because every materialization run lands here. Without the
-    * guard a `storage=`-only package — or one with no persist sources at all —
-    * paid a full package recompile per run: measured at ~1.1MB of RSS per run on
-    * the production image, never reclaimed, against a `main` build that reclaims.
-    */
-   public async reloadAllModelsForPackage(
-      packageName: string,
-      manifest: FreshnessManifest,
-   ): Promise<void> {
-      assertSafePackageName(packageName);
-      return this.withPackageLock(packageName, async () => {
-         const pkg = this.packages.get(packageName);
-         if (!pkg) {
-            throw new PackageNotFoundError(
-               `Package ${packageName} is not loaded`,
-            );
-         }
-         const has = Object.keys(manifest).length > 0;
-         const had = pkg.hasBoundTableNameManifest();
-         if (!has && !had) return;
-         await pkg.reloadAllModels(manifest);
-      });
-   }
-
-   /**
-    * Bind a package's `storage=` serve bindings from a build's FULL manifest
-    * entries (carrying `storageConnectionName` + captured `schema`), so a query
-    * against a materialized-into-storage source can be routed through the
-    * virtual-source serve transform. Distinct from
-    * {@link reloadAllModelsForPackage}, which binds the tableName-only manifest
-    * for same-connection persistence. No-op (logged) if the package isn't
-    * loaded — best-effort, like the post-build auto-load itself.
-    */
-   public async bindPackageStorageServeBindings(
-      packageName: string,
-      entries: Record<string, ManifestEntry>,
-   ): Promise<void> {
-      assertSafePackageName(packageName);
-      return this.withPackageLock(packageName, async () => {
-         const pkg = this.packages.get(packageName);
-         if (!pkg) {
-            logger.warn(
-               "Cannot bind storage serve bindings: package not loaded",
-               { packageName },
-            );
-            return;
-         }
-         // Host-authoritative: when a manifestLocation is bound, the host's
-         // manifest is the sole source of storage serve bindings (see
-         // bindManifest + rebindServeBindingsFromLocalStore' matching guard). This
-         // method is the LOCAL-store binding path (post-build auto-load and the
-         // post-delete rebind); on an orchestrated deployment the publisher
-         // still writes its own local materialization records, so without this
-         // guard a build's auto-load or a routine retire of a superseded record
-         // would clobber the host's bindings with a possibly-staler local
-         // generation. Standalone (no manifestLocation) is unaffected.
-         if (pkg.getPackageMetadata().manifestLocation) {
-            logger.debug(
-               "Skipping local-store storage serve binding: manifestLocation " +
-                  "is bound (host authoritative)",
-               { packageName },
-            );
-            return;
-         }
-         pkg.bindStorageServeBindings(entries);
-      });
-   }
-
-   /**
-    * Re-establish a package's COLOCATED (same-connection) serve routing from a
-    * FreshnessManifest, holding the package lock — the colocated analogue of
-    * {@link bindPackageStorageServeBindings}, used by the post-delete rebind so a
-    * reclaimed colocated table is not left routed. No-op (logged) if the package
-    * isn't loaded, and skipped when a `manifestLocation` is bound (host
-    * authoritative, same as the storage variant). An empty manifest reverts to
-    * serving live.
-    *
-    * Unlike the storage tier — and unlike the load-time colocated rebind
-    * ({@link Package.bindColocatedServeManifest}, which restores routing onto
-    * FRESHLY-compiled models with a per-query overlay, no recompile) — this path
-    * runs against models that a build's auto-load already recompiled with the
-    * substitution BAKED IN. Clearing the per-query overlay would not strip a
-    * baked substitution, so reverting/rebinding here requires a recompile
-    * ({@link Package.reloadAllModels}), exactly as {@link bindManifest} does for
-    * colocated. Skipped when there is nothing to substitute AND nothing was
-    * previously substituted to clear (no needless recompile for a storage-only or
-    * never-materialized package).
-    */
-   public async bindPackageColocatedServeManifest(
-      packageName: string,
-      entries: FreshnessManifest,
-   ): Promise<void> {
-      assertSafePackageName(packageName);
-      return this.withPackageLock(packageName, async () => {
-         const pkg = this.packages.get(packageName);
-         if (!pkg) {
-            logger.warn(
-               "Cannot bind colocated serve manifest: package not loaded",
-               { packageName },
-            );
-            return;
-         }
-         if (pkg.getPackageMetadata().manifestLocation) {
-            logger.debug(
-               "Skipping local-store colocated serve binding: manifestLocation " +
-                  "is bound (host authoritative)",
-               { packageName },
-            );
-            return;
-         }
-         const hasColocated = Object.keys(entries).length > 0;
-         const hadColocated = pkg.hasBoundTableNameManifest();
-         if (hasColocated || hadColocated) {
-            await pkg.reloadAllModels(entries);
-         }
-      });
-   }
-
-   /**
-    * If the freshly-loaded package declares a `manifestLocation`, fetch the
-    * control-plane-computed build manifest and rebind its models so persist
-    * references resolve to the materialized tables. Best-effort: a fetch/bind
-    * failure logs a warning and leaves the package serving live (the models are
-    * already loaded without a manifest). Callers must hold the package lock —
-    * this rebinds `pkg` in place rather than re-entering {@link withPackageLock}.
-    */
-   private async bindManifestIfConfigured(pkg: Package): Promise<void> {
-      const manifestLocation = pkg.getPackageMetadata().manifestLocation;
-      if (!manifestLocation) {
-         return;
-      }
-      await this.bindManifest(pkg, manifestLocation);
-   }
-
-   /** Fetch + bind a specific manifest URI onto an already-loaded package. */
-   private async bindManifest(
-      pkg: Package,
-      manifestLocation: string,
-   ): Promise<void> {
-      const packageName = pkg.getPackageName();
-      try {
-         // Bind runs before a package is marked SERVING, so a slow/unreachable
-         // manifest store must not block serving indefinitely — bound is the
-         // intended state, live is the degraded fallback. Race the fetch against
-         // a timeout and fall back to live on either failure.
-         const { tableNameManifest, storageEntries } =
-            await this.fetchManifestEntriesWithTimeout(manifestLocation);
-
-         // Tier split. Storage entries (cross-connection) bind as serve bindings
-         // WITHOUT a recompile — they apply to the already-compiled models via
-         // Model.setServeBindings. The host is the authoritative producer here,
-         // so this also supersedes the local-store rebind (see
-         // rebindServeBindingsFromLocalStore, which no-ops when manifestLocation
-         // is set).
-         // Bind whenever there ARE storage entries OR there WERE (bindStorage-
-         // ServeBindings with the now-empty set clears them): a manifest whose
-         // storage entries vanished must drop the old bindings, not leave them
-         // routing at a table the host no longer vouches for. Mirrors the
-         // hadColocated guard below.
-         const hasStorage = Object.keys(storageEntries).length > 0;
-         const hadStorage = pkg.hasStorageServeBindings();
-         if (hasStorage || hadStorage) {
-            pkg.bindStorageServeBindings(storageEntries);
-         }
-
-         // colocated entries drive the same-connection tableName substitution, which
-         // is resolved at COMPILE time — so they require a reloadAllModels
-         // recompile (v0's existing cost). Skip the recompile for a pure-storage
-         // manifest flip: only when there is nothing to substitute AND nothing
-         // previously substituted to clear (otherwise a package that just dropped
-         // its last colocated entry must still recompile to revert it).
-         const hasColocated = Object.keys(tableNameManifest).length > 0;
-         const hadColocated = pkg.hasBoundTableNameManifest();
-         if (hasColocated || hadColocated) {
-            await pkg.reloadAllModels(tableNameManifest);
-         }
-
-         pkg.setBoundManifestUri(manifestLocation);
-         recordManifestBind("success");
-         logger.info("Bound build manifest to package", {
-            environmentName: this.environmentName,
-            packageName,
-            manifestLocation,
-            tableNameEntryCount: Object.keys(tableNameManifest).length,
-            storageEntryCount: Object.keys(storageEntries).length,
-            recompiled: hasColocated || hadColocated,
-         });
-      } catch (err) {
-         pkg.markManifestBindFailed();
-         const timedOut =
-            err instanceof Error && err.message.includes("Timed out after");
-         recordManifestBind(timedOut ? "timeout" : "failure");
-         logger.warn("Failed to bind build manifest; serving live", {
-            environmentName: this.environmentName,
-            packageName,
-            manifestLocation,
-            timedOut,
-            error: err instanceof Error ? err.message : String(err),
-         });
-      }
-   }
-
-   /**
-    * Fetch manifest entries, rejecting if the fetch exceeds
-    * {@link MANIFEST_FETCH_TIMEOUT_MS}. Keeps {@link bindManifest}'s bind-before-
-    * serve guarantee from stalling on an unreachable manifest store.
-    */
-   private async fetchManifestEntriesWithTimeout(
-      manifestLocation: string,
-   ): Promise<FetchedManifest> {
-      let timer: ReturnType<typeof setTimeout> | undefined;
-      const timeout = new Promise<never>((_, reject) => {
-         timer = setTimeout(
-            () =>
-               reject(
-                  new Error(
-                     `Timed out after ${MANIFEST_FETCH_TIMEOUT_MS}ms fetching manifest ${manifestLocation}`,
-                  ),
-               ),
-            MANIFEST_FETCH_TIMEOUT_MS,
-         );
-      });
-      try {
-         return await Promise.race([
-            fetchManifestEntries(manifestLocation),
-            timeout,
-         ]);
-      } finally {
-         if (timer) {
-            clearTimeout(timer);
-         }
-      }
-   }
-
-   /**
-    * Read a model's source text from disk, holding the per-package mutex
-    * so the read is serialized against {@link installPackage} /
-    * {@link deletePackage} / {@link updatePackage}.
-    */
-   public async getModelFileText(
-      packageName: string,
-      modelPath: string,
-   ): Promise<string> {
-      assertSafePackageName(packageName);
-      assertSafeRelativeModelPath(modelPath);
-      return this.withPackageLock(packageName, async () => {
-         const pkg = this.packages.get(packageName);
-         if (!pkg) {
-            throw new PackageNotFoundError(
-               `Package ${packageName} is not loaded`,
-            );
-         }
-         return pkg.getModelFileText(modelPath);
-      });
-   }
-
-   private async writePackageManifest(
-      packageName: string,
-      metadata: {
-         name: string;
-         description?: string;
-         explores?: string[];
-         queryableSources?: "declared" | "all";
-         manifestLocation?: string | null;
-         scope?: ApiPackage["scope"];
-         materialization?: ApiPackage["materialization"];
-      },
-   ): Promise<void> {
-      const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
-      const manifestPath = safeJoinUnderRoot(packagePath, "publisher.json");
-
-      try {
-         // Read existing manifest
-         let existingManifest: Record<string, unknown> = {};
-         try {
-            const content = await fs.promises.readFile(manifestPath, "utf-8");
-            existingManifest = JSON.parse(content);
-         } catch (_err) {
-            logger.warn(`Could not read manifest for ${packageName}`);
-         }
-
-         const onDiskMaterialization =
-            existingManifest.materialization !== null &&
-            typeof existingManifest.materialization === "object" &&
-            !Array.isArray(existingManifest.materialization)
-               ? (existingManifest.materialization as Record<string, unknown>)
-               : undefined;
-
-         // Scope has two homes: `materialization.scope` (canonical) and the
-         // manifest root (deprecated). The server writes BOTH, in sync, for as
-         // long as the root form is supported:
-         //
-         //  - writing only the root would author a manifest this build's loader
-         //    refuses, since a root that disagrees with an existing envelope is a
-         //    conflict (see resolvePackageScope);
-         //  - writing only the envelope would silently downgrade a package read
-         //    by an older publisher, which knows only the root and would default
-         //    to `package` — cross-version table reuse for a package declared
-         //    `version`.
-         //
-         // A caller can only express scope through the top-level `scope` field
-         // (the wire materialization block has no `scope`), so an envelope value
-         // already on disk is preserved rather than dropped by a materialization
-         // PATCH that says nothing about it.
-         const resolvedScope =
-            metadata.scope ??
-            (onDiskMaterialization?.scope as ApiPackage["scope"] | undefined) ??
-            (existingManifest.scope as ApiPackage["scope"] | undefined);
-
-         // A materialization PATCH replaces the block wholesale, which is right
-         // for schedule and freshness — they are the policy the caller is
-         // setting, and they are mutually exclusive with each other.
-         // `queryMetadata` is orthogonal to both: a client setting a schedule
-         // has no reason to re-send the package's tags, and dropping them
-         // silently untags every statement the package's builds issue. So it is
-         // preserved on omission, like `scope` above; an explicit null still
-         // clears it, which keeps the block expressible.
-         const preservedQueryMetadata =
-            metadata.materialization !== undefined &&
-            metadata.materialization?.queryMetadata === undefined &&
-            onDiskMaterialization?.queryMetadata !== undefined
-               ? { queryMetadata: onDiskMaterialization.queryMetadata }
-               : {};
-
-         const materializationBase: Record<string, unknown> | undefined =
-            metadata.materialization !== undefined
-               ? { ...metadata.materialization, ...preservedQueryMetadata }
-               : onDiskMaterialization !== undefined
-                 ? { ...onDiskMaterialization }
-                 : undefined;
-         const materializationBlock =
-            resolvedScope !== undefined
-               ? { ...(materializationBase ?? {}), scope: resolvedScope }
-               : materializationBase;
-
-         // Update with new metadata. `explores`/`queryableSources` are only
-         // overwritten when the caller explicitly provides them; otherwise the
-         // existing on-disk value is preserved via the spread (an undefined here
-         // must not erase it).
-         const updatedManifest = {
-            ...existingManifest,
-            name: metadata.name,
-            description: metadata.description,
-            ...(metadata.explores !== undefined
-               ? { explores: metadata.explores }
-               : {}),
-            ...(metadata.queryableSources !== undefined
-               ? { queryableSources: metadata.queryableSources }
-               : {}),
-            ...(metadata.manifestLocation !== undefined
-               ? { manifestLocation: metadata.manifestLocation }
-               : {}),
-            ...(resolvedScope !== undefined ? { scope: resolvedScope } : {}),
-            ...(materializationBlock !== undefined
-               ? { materialization: materializationBlock }
-               : {}),
-         };
-
-         // Write back to file
-         await fs.promises.writeFile(
-            manifestPath,
-            JSON.stringify(updatedManifest, null, 2),
-            "utf-8",
-         );
-
-         logger.info(`Updated publisher.json for ${packageName}`);
-      } catch (error) {
-         logger.error(`Failed to update publisher.json`, { error });
-         throw new Error(`Failed to update package manifest`);
-      }
-   }
-
-   public async updatePackage(packageName: string, body: ApiPackage) {
-      assertSafePackageName(packageName);
-      return this.withPackageLock(packageName, async () => {
-         const _package = this.packages.get(packageName);
-         if (!_package) {
-            throw new PackageNotFoundError(`Package ${packageName} not found`);
-         }
-         if (body.name) {
-            _package.setName(body.name);
-         }
-         // Preserve `explores` across a metadata PATCH. `setPackageMetadata`
-         // replaces the whole object, so a name/description-only update must
-         // carry the existing discovery surface through — otherwise the
-         // in-memory `explores` is wiped and `listModels()` silently starts
-         // serving every model until the next reload. When the body explicitly
-         // carries `explores`, honor the new set instead.
-         const existing = _package.getPackageMetadata();
-         // Normalize API-body explores through the same helper the worker uses
-         // for on-disk explores, so `["./index.malloy"]` / backslash paths
-         // validate and persist identically regardless of input channel (no
-         // misleading publish-time 400, no publish-vs-reload divergence).
-         const normalizedExplores = body.explores?.map(normalizeModelPath);
-         const explores =
-            normalizedExplores !== undefined
-               ? normalizedExplores
-               : existing.explores;
-         const queryableSources =
-            body.queryableSources !== undefined
-               ? body.queryableSources
-               : existing.queryableSources;
-         // Preserve the existing manifestLocation unless the body explicitly
-         // sets it (including to null, which clears it and reverts to live).
-         const manifestLocation =
-            body.manifestLocation !== undefined
-               ? body.manifestLocation
-               : existing.manifestLocation;
-         // Persist `scope` and `materialization` (the schedule cron) are
-         // editable via the API — both are writable in the schema. When the
-         // body carries a value, apply it; otherwise preserve the
-         // manifest-derived one (a name/description-only PATCH must not wipe
-         // them, and the control plane must not misread the gap as a removal).
-         // Changing the schedule re-arms the standalone scheduler on its next
-         // tick — no reload needed.
-         //
-         // A *null* scope/materialization is treated the same as omitted
-         // (preserve), NOT a wipe: the control plane's post-build rebind PATCH
-         // carries only name/location/manifestLocation, but a client that
-         // serializes unset fields as explicit null must not thereby trip the
-         // policy gate below (a rejection there fails the orchestrated run) or
-         // reset the persisted policy. `manifestLocation` is deliberately
-         // different — null there means "clear" (revert to live), which the
-         // caller's orchestrated build path relies on.
-         const scopeProvided = body.scope != null;
-         const materializationProvided = body.materialization != null;
-         const editingPolicy = scopeProvided || materializationProvided;
-         const scope = scopeProvided ? body.scope : existing.scope;
-         const materialization = materializationProvided
-            ? body.materialization
-            : existing.materialization;
-         _package.setPackageMetadata({
-            name: body.name,
-            description: body.description,
-            resource: body.resource,
-            location: body.location,
-            explores,
-            queryableSources,
-            manifestLocation,
-            materialization,
-            scope,
-         });
-
-         // Strict-reject, symmetric with the publish path
-         // (package.controller.addPackage): validate the resulting explores
-         // against the live model set and restore the prior metadata before
-         // rejecting, so a bad update neither persists nor mutates the served
-         // surface. When the body edits the persistence policy (scope /
-         // materialization), also enforce the same scope/schedule/freshness/cron
-         // rules a publish enforces — but only then, so a description-only PATCH
-         // on a package with a pre-existing (load-tolerated) policy warning is
-         // not newly rejected. Cron validity is one of these rules
-         // (persistencePolicyWarnings Rule 4), so publish, PATCH, load, and the
-         // scheduler all enforce it identically.
-         const policyMsg = editingPolicy
-            ? _package.formatInvalidPersistencePolicy()
-            : "";
-         const invalidMsg = [_package.formatInvalidExplores(), policyMsg]
-            .filter(Boolean)
-            .join("\n");
-         if (invalidMsg) {
-            _package.setPackageMetadata(existing);
-            throw new BadRequestError(invalidMsg);
-         }
-
-         await this.writePackageManifest(packageName, {
-            name: packageName,
-            description: body.description,
-            explores: normalizedExplores,
-            queryableSources: body.queryableSources,
-            manifestLocation: body.manifestLocation,
-            // Only write when explicitly provided (non-null): mirrors the
-            // null-as-absent rule above, so a rebind PATCH neither wipes the
-            // persisted policy nor writes a stray `scope: null`.
-            scope: scopeProvided ? body.scope : undefined,
-            materialization: materializationProvided
-               ? body.materialization
-               : undefined,
-         });
-
-         // When the body changes manifestLocation, apply it now so the new
-         // binding takes effect without a separate reload: a URI rebinds models
-         // to the materialized tables; null/empty reverts the package to live.
-         if (body.manifestLocation !== undefined) {
-            if (body.manifestLocation) {
-               await this.bindManifest(_package, body.manifestLocation);
-            } else {
-               // Revert to live: drop the colocated tableName substitution AND the
-               // cross-connection storage serve bindings the prior bindManifest
-               // applied, so no query still routes to a materialized table after
-               // the operator explicitly cleared the manifest.
-               await _package.reloadAllModels({});
-               _package.bindStorageServeBindings({});
-            }
-         }
-
-         return _package.getPackageMetadata();
-      });
-   }
-
-   public getPackageStatus(packageName: string): PackageInfo | undefined {
-      return this.packageStatuses.get(packageName);
-   }
-
-   /**
-    * Packages this environment is actually serving: registered statuses minus
-    * any recorded as failed or un-mounted. Disjoint from getFailedPackages()
-    * by construction, so the readiness line's packages= and load_errors=
-    * cannot double-count a package that is seeded SERVING at boot and only
-    * pruned later by a side-effect load (which a transient DB or memory-
-    * pressure error can skip). Cheap: no package load is triggered.
-    */
-   public getServingPackageCount(): number {
-      const failed = this.getFailedPackages();
-      let serving = 0;
-      for (const name of this.packageStatuses.keys()) {
-         if (!failed.has(name)) serving += 1;
-      }
-      return serving;
-   }
-
-   /**
-    * Record why a configured package's location never mounted, so /status can
-    * name the real cause instead of the missing-manifest fallout it produces.
-    * Called by EnvironmentStore right after the environment is created.
-    */
-   public setPackageMountError(packageName: string, message: string): void {
-      this.mountErrors.set(packageName, message);
-   }
-
-   /** Forget any recorded failure for a package, whatever its cause. */
-   private clearPackageLoadFailure(packageName: string): void {
-      this.failedPackages.delete(packageName);
-      this.mountErrors.delete(packageName);
-   }
-
-   /** Packages configured for this environment that did not load, and why. */
-   public getFailedPackages(): ReadonlyMap<string, string> {
-      if (this.mountErrors.size === 0) return this.failedPackages;
-      // Mount errors last, so the specific cause overwrites the generic
-      // manifest error that the un-mounted package produces on its lazy load.
-      return new Map([...this.failedPackages, ...this.mountErrors]);
-   }
-
-   public setPackageStatus(packageName: string, status: PackageStatus): void {
-      const currentStatus = this.packageStatuses.get(packageName);
-      this.packageStatuses.set(packageName, {
-         name: packageName,
-         loadTimestamp: currentStatus?.loadTimestamp || Date.now(),
-         status: status,
-      });
-   }
-
-   public deletePackageStatus(packageName: string): void {
-      this.packageStatuses.delete(packageName);
-   }
-
-   public async deletePackage(packageName: string): Promise<void> {
-      assertSafePackageName(packageName);
-      return this.withPackageLock(packageName, async () => {
-         // Clear the load failure before the early return, not after it. A
-         // package that failed to load is not in `packages` (the load catch
-         // evicts it), so deleting it takes the early return every time, while
-         // the controller still drops its config row. Leaving the entry would
-         // make getStatus report a loadError for a package that is no longer
-         // configured, which is the one thing that channel must not do.
-         this.clearPackageLoadFailure(packageName);
-
-         const _package = this.packages.get(packageName);
-         if (!_package) {
-            return;
-         }
-         const packageStatus = this.packageStatuses.get(packageName);
-
-         // The mutex now serializes load/install/compile against delete, so
-         // the LOADING-state guard is mostly vestigial — left in place for
-         // backwards-compatible error messaging in case anything bypasses
-         // the lock.
-         if (packageStatus?.status === PackageStatus.LOADING) {
-            logger.error("Package loading. Can't unload.", {
-               environmentName: this.environmentName,
-               packageName,
-            });
-            throw new Error(
-               "Package loading. Can't unload. " +
-                  this.environmentName +
-                  " " +
-                  packageName,
-            );
-         } else if (packageStatus?.status === PackageStatus.SERVING) {
-            this.setPackageStatus(packageName, PackageStatus.UNLOADING);
-         }
-
-         // Retire the package's connections via the existing 30s drain so
-         // any in-flight queries that already acquired a connection finish
-         // before the underlying duckdb handle is released.
-         this.retireConnectionGeneration(`package ${packageName}`, () =>
-            _package.getMalloyConfig().shutdown("close"),
-         );
-
-         // Atomically rename the canonical tree out of the way so no reader
-         // can stat into it after the lock is released. The actual fs.rm is
-         // deferred to setImmediate to keep the lock-hold time at one
-         // rename rather than a (potentially slow) recursive remove.
-         const canonicalPath = safeJoinUnderRoot(
-            this.environmentPath,
-            packageName,
-         );
-         const retiredPath = this.allocateRetiredPath(packageName);
-         let renamed = false;
-         try {
-            await fs.promises.mkdir(path.dirname(retiredPath), {
-               recursive: true,
-            });
-            await fs.promises.rename(canonicalPath, retiredPath);
-            renamed = true;
-         } catch (err) {
-            logger.error(
-               "Error renaming package directory to retired during unload",
-               {
-                  error: err,
-                  environmentName: this.environmentName,
-                  packageName,
-               },
-            );
-         }
-
-         this.packages.delete(packageName);
-         this.packageStatuses.delete(packageName);
-
-         if (renamed) {
-            setImmediate(() => {
-               void fs.promises
-                  .rm(retiredPath, { recursive: true, force: true })
-                  .catch((err) => {
-                     logger.warn(
-                        `Failed to clean up retired package directory ${retiredPath}`,
-                        { error: err },
-                     );
-                  });
-            });
-         }
-      });
-   }
-
-   /**
-    * Evict a package from the in-memory caches WITHOUT touching its on-disk
-    * directory — the non-destructive counterpart to {@link deletePackage}.
-    *
-    * Used to roll back a no-location `addPackage` (which registers a
-    * *pre-existing*, user-owned directory) when post-load validation rejects
-    * it: deleting the tree there would destroy content the publisher never
-    * created. This still drains and closes the connections the just-created
-    * `Package` opened, so the duckdb handle isn't leaked.
-    */
-   public async unloadPackage(packageName: string): Promise<void> {
-      assertSafePackageName(packageName);
-      return this.withPackageLock(packageName, async () => {
-         const _package = this.packages.get(packageName);
-         if (!_package) {
-            return;
-         }
-         if (
-            this.packageStatuses.get(packageName)?.status ===
-            PackageStatus.SERVING
-         ) {
-            this.setPackageStatus(packageName, PackageStatus.UNLOADING);
-         }
-         // Same 30s connection drain as deletePackage — just no fs rename/rm.
-         this.retireConnectionGeneration(`package ${packageName}`, () =>
-            _package.getMalloyConfig().shutdown("close"),
-         );
-         this.packages.delete(packageName);
-         this.packageStatuses.delete(packageName);
-      });
-   }
-
-   public updateConnections(
-      malloyConfig: EnvironmentMalloyConfig,
-      _apiConnections?: ApiConnection[],
-      afterPreviousRelease?: () => Promise<void>,
-   ): void {
-      const previousMalloyConfig = this.malloyConfig;
-      this.malloyConfig = malloyConfig;
-      this.apiConnections = malloyConfig.apiConnections;
-
-      if (previousMalloyConfig !== malloyConfig) {
-         this.retireConnectionGeneration(
-            `environment ${this.environmentName}`,
-            async () => {
-               await previousMalloyConfig.releaseConnections();
-               await afterPreviousRelease?.();
-            },
-         );
-      } else {
-         void afterPreviousRelease?.();
-      }
-   }
-
-   public async deleteConnection(connectionName: string): Promise<void> {
-      const index = this.apiConnections.findIndex(
-         (conn) => conn.name === connectionName,
-      );
-
-      if (index !== -1) {
-         this.apiConnections.splice(index, 1);
-      }
-
-      if (index !== -1) {
-         logger.info(
-            `Removed connection ${connectionName} from environment ${this.environmentName}`,
-         );
-      } else {
-         logger.warn(
-            `Connection ${connectionName} not found in environment ${this.environmentName}`,
-         );
-      }
-   }
-
-   public async closeAllConnections(): Promise<void> {
-      // Release the package-scoped MalloyConfigs (each holds the package's own
-      // sandbox `duckdb` connection) before tearing down the environment config
-      // they wrap. Without this, hard unload leaks per-package DuckDB handles.
-      const packageReleases = await Promise.allSettled(
-         Array.from(this.packages.values(), (pkg) =>
-            pkg.getMalloyConfig().shutdown("close"),
-         ),
-      );
-      for (const result of packageReleases) {
-         if (result.status === "rejected") {
-            logger.error(
-               `Error closing package connections for environment ${this.environmentName}`,
-               { error: result.reason },
-            );
-         }
-      }
-      this.packages.clear();
-      this.packageStatuses.clear();
-
-      try {
-         await this.malloyConfig.releaseConnections();
-      } catch (error) {
-         logger.error(
-            `Error closing connections for environment ${this.environmentName}`,
-            { error },
-         );
-      }
-      await this.releaseAllRetiredConnectionGenerations();
-
-      this.apiConnections = [];
-
-      logger.info(
-         `Closed all connections for environment ${this.environmentName}`,
-      );
-   }
-
-   public async serialize(): Promise<ApiEnvironment> {
-      return {
-         ...this.metadata,
-         connections: this.listApiConnections(),
-         packages: await this.listPackages(),
-      };
-   }
-
-   public async deleteDuckDBConnection(connectionName: string): Promise<void> {
-      const duckdbPath = path.join(
-         this.environmentPath,
-         `${connectionName}.duckdb`,
-      );
-      try {
-         await fs.promises.rm(duckdbPath, { force: true });
-         logger.info(
-            `Removed DuckDB connection file ${connectionName} from environment ${this.environmentName}`,
-         );
-      } catch (error) {
-         logger.error(
-            `Failed to remove DuckDB connection file ${connectionName} from environment ${this.environmentName}`,
-            { error },
-         );
-      }
-   }
-
-   public async deleteDuckLakeConnection(
-      connectionName: string,
-   ): Promise<void> {
-      await deleteDuckLakeConnectionFile(connectionName, this.environmentPath);
-      logger.info(
-         `Removed DuckLake connection ${connectionName} from environment ${this.environmentName}`,
-      );
-   }
-}
-
-/**
- * Extracts the preamble from a Malloy model file — the leading block of
- * `##!` pragmas, `import` statements, blank lines, and comments that appear
- * before any `source:`, `query:`, or `run:` definition. This allows a
- * submitted query to inherit the model's import context.
- */
-export async function extractPreamble(modelPath: string): Promise<string> {
-   try {
-      const content = await fs.promises.readFile(modelPath, "utf8");
-      return extractPreambleFromSource(content);
-   } catch {
-      // If the model file can't be read, return empty preamble
-      // and let the compilation surface any import errors naturally.
-      return "";
-   }
-}
-
-/**
- * Extracts the preamble from Malloy source text. Exported for testing.
- */
-export function extractPreambleFromSource(content: string): string {
-   const lines = content.split("\n");
-   const preambleLines: string[] = [];
-
-   for (const line of lines) {
-      const trimmed = line.trim();
-      // Stop at the first source/query/run definition
-      if (
-         trimmed.startsWith("source:") ||
-         trimmed.startsWith("query:") ||
-         trimmed.startsWith("run:")
-      ) {
-         break;
-      }
-      preambleLines.push(line);
-   }
-
-   return preambleLines.join("\n").trimEnd();
-}