@malloy-publisher/server 0.0.199 → 0.0.201

package/src/service/environment.ts

@@ -1,5 +1,6 @@
 import type { GivenValue, LogMessage } from "@malloydata/malloy";
 import { MalloyError, Runtime } from "@malloydata/malloy";
+import { metrics } from "@opentelemetry/api";
 import { Mutex } from "async-mutex";
 import crypto from "crypto";
 import * as fs from "fs";
@@ -69,6 +70,49 @@ type RetiredConnectionGeneration = {
 
 const RETIRED_CONNECTION_DRAIN_MS = 30_000;
 
+/**
+ * Module-scoped admission-rejection counters. Lazy-initialized so
+ * the OTel JS `ProxyMeter` cannot strand them on a NoOp instrument
+ * created before the SDK MeterProvider was registered (a real risk
+ * in unit tests; see comment in `query_timeout.ts`). Environment
+ * name is attached as a label so dashboards can identify hot
+ * environments without grepping logs.
+ */
+import { type Counter } from "@opentelemetry/api";
+let queryAdmissionRejectionsCounter: Counter | null = null;
+let packageAdmissionRejectionsCounter: Counter | null = null;
+function getQueryAdmissionRejectionsCounter(): Counter {
+   if (queryAdmissionRejectionsCounter) return queryAdmissionRejectionsCounter;
+   queryAdmissionRejectionsCounter = metrics
+      .getMeter("publisher")
+      .createCounter("publisher_query_admission_rejections_total", {
+         description:
+            "Queries rejected with 503 because Environment.assertCanAdmitQuery() observed memory back-pressure",
+      });
+   return queryAdmissionRejectionsCounter;
+}
+function getPackageAdmissionRejectionsCounter(): Counter {
+   if (packageAdmissionRejectionsCounter) {
+      return packageAdmissionRejectionsCounter;
+   }
+   packageAdmissionRejectionsCounter = metrics
+      .getMeter("publisher")
+      .createCounter("publisher_package_admission_rejections_total", {
+         description:
+            "Package loads rejected with 503 because Environment.assertCanAdmitNewPackage() observed memory back-pressure",
+      });
+   return packageAdmissionRejectionsCounter;
+}
+
+/**
+ * Visible for tests; production code never calls this. Resets the
+ * lazy caches so a fresh MeterProvider can capture future writes.
+ */
+export function resetAdmissionTelemetryForTesting(): void {
+   queryAdmissionRejectionsCounter = null;
+   packageAdmissionRejectionsCounter = null;
+}
+
 export class Environment {
    private packages: Map<string, Package> = new Map();
    // Lock ordering: connectionMutex (environment) MUST be acquired before any
@@ -176,6 +220,7 @@ export class Environment {
       environmentPath: string,
       connections: ApiConnection[],
    ): Promise<Environment> {
+      assertSafeEnvironmentPath(environmentPath);
       if (!(await fs.promises.stat(environmentPath))?.isDirectory()) {
          throw new EnvironmentNotFoundError(
             `Environment path ${environmentPath} not found`,
@@ -191,7 +236,10 @@ export class Environment {
       logger.info(
          `Loaded ${malloyConfig.apiConnections.length} connections for environment ${environmentName}`,
          {
-            apiConnections: malloyConfig.apiConnections,
+            connections: malloyConfig.apiConnections.map((c) => ({
+               name: c.name,
+               type: c.type,
+            })),
          },
       );
 
@@ -218,7 +266,7 @@ export class Environment {
       try {
          readme = (
             await fs.promises.readFile(
-               path.join(this.environmentPath, README_NAME),
+               safeJoinUnderRoot(this.environmentPath, README_NAME),
             )
          ).toString();
       } catch {
@@ -579,8 +627,43 @@ export class Environment {
    ): void {
       if (allowAdmission) return;
       if (!this.memoryGovernor?.isBackpressured()) return;
+      // Increment *before* throwing so the metric ticks even on
+      // the not-uncommon "caught and swallowed" path. The label
+      // shape mirrors `assertCanAdmitQuery` so a dashboard panel
+      // can sum both rejection kinds by environment.
+      getPackageAdmissionRejectionsCounter().add(1, {
+         environment: this.environmentName,
+         reason,
+      });
       throw new ServiceUnavailableError(
-         `Publisher is under memory pressure and cannot ${reason} (package "${packageName}", environment "${this.environmentName}"). Retry after the server's memory usage drops below the configured low-water mark.`,
+         `Publisher is under memory pressure and cannot ${reason} (package "${packageName}", environment "${this.environmentName}"). Retry after the server's memory usage drops below the low-water mark (PUBLISHER_MEMORY_LOW_WATER_FRACTION of PUBLISHER_MAX_MEMORY_BYTES), or raise PUBLISHER_MAX_MEMORY_BYTES if you have headroom.`,
+      );
+   }
+
+   /**
+    * Reject incoming queries with HTTP 503 when the memory governor
+    * has tripped its high-water mark. Used by every query controller
+    * (connection SQL, model query, notebook cell, MCP `execute_query`)
+    * to shed load before the query runs — complementing
+    * {@link assertCanAdmitNewPackage}, which only fires on cache-miss
+    * package loads and so leaves already-loaded packages fully
+    * queryable under pressure. With this in place, "back-pressured"
+    * means "no new work of any kind" until the governor's low-water
+    * mark is crossed.
+    *
+    * Cheap O(1) boolean read; no allocation when happy.
+    */
+   public assertCanAdmitQuery(): void {
+      if (!this.memoryGovernor?.isBackpressured()) return;
+      // Tick first so the counter reflects every rejection even
+      // when the controller's catch block swallows the error (e.g.
+      // an MCP tool surfaces it as a content payload rather than
+      // letting it bubble to the HTTP error mapper).
+      getQueryAdmissionRejectionsCounter().add(1, {
+         environment: this.environmentName,
+      });
+      throw new ServiceUnavailableError(
+         `Publisher is under memory pressure and cannot accept new queries (environment "${this.environmentName}"). Retry after the server's memory usage drops below the low-water mark (PUBLISHER_MEMORY_LOW_WATER_FRACTION of PUBLISHER_MAX_MEMORY_BYTES), or raise PUBLISHER_MAX_MEMORY_BYTES if you have headroom.`,
       );
    }
 
@@ -696,7 +779,6 @@ export class Environment {
          `Adding package ${packageName} to environment ${this.environmentName}`,
          {
             packagePath,
-            malloyConfig: this.malloyConfig.malloyConfig,
          },
       );
 
@@ -762,6 +844,12 @@ export class Environment {
       const stagingPath = this.allocateStagingPath(packageName);
       await fs.promises.mkdir(path.dirname(stagingPath), { recursive: true });
 
+      logger.debug("install.phase1.download.started", {
+         environmentName: this.environmentName,
+         packageName,
+         stagingPath,
+      });
+      const downloadStartedAt = performance.now();
       try {
          await downloader(stagingPath);
       } catch (err) {
@@ -770,8 +858,17 @@ export class Environment {
             .catch(() => {});
          throw err;
       }
+      logger.debug("install.phase1.download.completed", {
+         environmentName: this.environmentName,
+         packageName,
+         durationMs: performance.now() - downloadStartedAt,
+      });
 
       return this.withPackageLock(packageName, async () => {
+         logger.debug("install.phase2.swap.started", {
+            environmentName: this.environmentName,
+            packageName,
+         });
          const canonicalPath = safeJoinUnderRoot(
             this.environmentPath,
             packageName,
@@ -790,6 +887,11 @@ export class Environment {
                recursive: true,
             });
             await fs.promises.rename(canonicalPath, retiredPath);
+            logger.debug("install.phase2.retired_old", {
+               environmentName: this.environmentName,
+               packageName,
+               retiredPath,
+            });
          }
 
          let newPackage: Package;
@@ -803,6 +905,11 @@ export class Environment {
                canonicalPath,
                () => this.malloyConfig.malloyConfig,
             );
+            logger.debug("install.phase2.committed", {
+               environmentName: this.environmentName,
+               packageName,
+               canonicalPath,
+            });
          } catch (err) {
             // Rollback: clobber whatever (partial) content sits at canonical
             // — Package.create's own failure-cleanup may have already rm'd
@@ -814,9 +921,11 @@ export class Environment {
             await fs.promises
                .rm(canonicalPath, { recursive: true, force: true })
                .catch(() => {});
+            let restored = false;
             if (retiredPath) {
                try {
                   await fs.promises.rename(retiredPath, canonicalPath);
+                  restored = true;
                } catch (restoreErr) {
                   logger.error(
                      "Failed to restore retired package after install rollback",
@@ -832,6 +941,12 @@ export class Environment {
                .rm(stagingPath, { recursive: true, force: true })
                .catch(() => {});
             this.deletePackageStatus(packageName);
+            logger.debug("install.phase2.rollback", {
+               environmentName: this.environmentName,
+               packageName,
+               restored,
+               errorName: err instanceof Error ? err.name : "Unknown",
+            });
             throw err;
          }
 
@@ -847,6 +962,11 @@ export class Environment {
          if (retiredPath) {
             const pathToClean = retiredPath;
             setImmediate(() => {
+               logger.debug("install.phase3.retired_cleanup", {
+                  environmentName: this.environmentName,
+                  packageName,
+                  retiredPath: pathToClean,
+               });
                void fs.promises
                   .rm(pathToClean, { recursive: true, force: true })
                   .catch((err) => {

package/src/service/environment_admission.spec.ts

@@ -4,8 +4,12 @@ import * as os from "os";
 import * as path from "path";
 
 import { ServiceUnavailableError } from "../errors";
+import {
+   startMetricsHarness,
+   type MetricsHarness,
+} from "../test_helpers/metrics_harness";
 import { buildEnvironmentMalloyConfig } from "./connection";
-import { Environment } from "./environment";
+import { Environment, resetAdmissionTelemetryForTesting } from "./environment";
 import type { PackageMemoryGovernor } from "./package_memory_governor";
 
 /**
@@ -178,3 +182,163 @@ describe("Environment admission gate (memory governor choke point)", () => {
       expect(caught).not.toBeInstanceOf(ServiceUnavailableError);
    });
 });
+
+describe("Environment.assertCanAdmitQuery (query-path back-pressure)", () => {
+   let envDir: string;
+
+   beforeEach(() => {
+      envDir = fs.mkdtempSync(
+         path.join(os.tmpdir(), "publisher-env-query-admission-"),
+      );
+   });
+
+   afterEach(() => {
+      fs.rmSync(envDir, { recursive: true, force: true });
+   });
+
+   it("is a no-op when no governor is attached", () => {
+      const env = makeEnvironment(envDir);
+      // No governor — must not throw. Equivalent of an OSS / non-Docker
+      // deployment that never opted into PUBLISHER_MAX_MEMORY_BYTES.
+      expect(() => env.assertCanAdmitQuery()).not.toThrow();
+   });
+
+   it("is a no-op when the governor is happy (under high-water mark)", () => {
+      const env = makeEnvironment(envDir);
+      const governor = new StubGovernor();
+      env.setMemoryGovernor(governor as unknown as PackageMemoryGovernor);
+      governor.backpressured = false;
+
+      expect(() => env.assertCanAdmitQuery()).not.toThrow();
+   });
+
+   it("throws ServiceUnavailableError (→503) when back-pressured", () => {
+      const env = makeEnvironment(envDir);
+      const governor = new StubGovernor();
+      env.setMemoryGovernor(governor as unknown as PackageMemoryGovernor);
+      governor.backpressured = true;
+
+      expect(() => env.assertCanAdmitQuery()).toThrow(ServiceUnavailableError);
+   });
+
+   it("error message names the environment so operators can pinpoint the hot pod's load", () => {
+      const env = makeEnvironment(envDir);
+      const governor = new StubGovernor();
+      env.setMemoryGovernor(governor as unknown as PackageMemoryGovernor);
+      governor.backpressured = true;
+
+      let caught: unknown;
+      try {
+         env.assertCanAdmitQuery();
+      } catch (err) {
+         caught = err;
+      }
+      expect(caught).toBeInstanceOf(ServiceUnavailableError);
+      expect((caught as Error).message).toContain('environment "test-env"');
+      expect((caught as Error).message).toContain("memory pressure");
+   });
+
+   it("clearing back-pressure immediately re-admits queries (matches governor hysteresis)", () => {
+      const env = makeEnvironment(envDir);
+      const governor = new StubGovernor();
+      env.setMemoryGovernor(governor as unknown as PackageMemoryGovernor);
+
+      governor.backpressured = true;
+      expect(() => env.assertCanAdmitQuery()).toThrow(ServiceUnavailableError);
+
+      governor.backpressured = false;
+      expect(() => env.assertCanAdmitQuery()).not.toThrow();
+   });
+
+   it("detaching the governor reverts to legacy admit-everything", () => {
+      const env = makeEnvironment(envDir);
+      const governor = new StubGovernor();
+      env.setMemoryGovernor(governor as unknown as PackageMemoryGovernor);
+      governor.backpressured = true;
+
+      env.setMemoryGovernor(null);
+      // Even with the stub still claiming back-pressure, a detached
+      // governor leaves nothing to consult. Mirrors the package-admission
+      // detach behavior.
+      expect(() => env.assertCanAdmitQuery()).not.toThrow();
+   });
+});
+
+describe("Environment admission telemetry", () => {
+   let envDir: string;
+   let harness: MetricsHarness;
+
+   beforeEach(async () => {
+      envDir = fs.mkdtempSync(
+         path.join(os.tmpdir(), "publisher-env-admission-telemetry-"),
+      );
+      harness = await startMetricsHarness();
+      resetAdmissionTelemetryForTesting();
+   });
+
+   afterEach(async () => {
+      fs.rmSync(envDir, { recursive: true, force: true });
+      resetAdmissionTelemetryForTesting();
+      await harness.shutdown();
+   });
+
+   it("publisher_query_admission_rejections_total ticks per query rejection, labeled by environment", async () => {
+      const env = makeEnvironment(envDir);
+      const governor = new StubGovernor();
+      env.setMemoryGovernor(governor as unknown as PackageMemoryGovernor);
+      governor.backpressured = true;
+
+      // Drive three rejections so the counter is unambiguous (a
+      // single-tick assertion can pass against a leaked counter
+      // from a different test; three is harder to fake).
+      for (let i = 0; i < 3; i++) {
+         expect(() => env.assertCanAdmitQuery()).toThrow(
+            ServiceUnavailableError,
+         );
+      }
+      expect(
+         await harness.collectCounter(
+            "publisher_query_admission_rejections_total",
+            { environment: "test-env" },
+         ),
+      ).toBe(3);
+   });
+
+   it("counter stays at zero when the governor is happy (no spurious ticks)", async () => {
+      const env = makeEnvironment(envDir);
+      const governor = new StubGovernor();
+      env.setMemoryGovernor(governor as unknown as PackageMemoryGovernor);
+      // Not back-pressured.
+      env.assertCanAdmitQuery();
+      env.assertCanAdmitQuery();
+      // No rejection should have been recorded; verifies that the
+      // counter doesn't tick on the happy-path admission either.
+      expect(
+         await harness.collectCounter(
+            "publisher_query_admission_rejections_total",
+         ),
+      ).toBe(0);
+   });
+
+   it("publisher_package_admission_rejections_total ticks per package-load rejection", async () => {
+      // Ensure the package directory exists so the 404 doesn't
+      // short-circuit ahead of the back-pressure gate.
+      const pkgName = "real-pkg";
+      fs.mkdirSync(path.join(envDir, pkgName));
+
+      const env = makeEnvironment(envDir);
+      const governor = new StubGovernor();
+      env.setMemoryGovernor(governor as unknown as PackageMemoryGovernor);
+      governor.backpressured = true;
+
+      await expect(env.addPackage(pkgName)).rejects.toBeInstanceOf(
+         ServiceUnavailableError,
+      );
+      expect(
+         await harness.collectCounter(
+            "publisher_package_admission_rejections_total",
+            { environment: "test-env", reason: "add a new package" },
+         ),
+      ).toBe(1);
+   });
+});

package/src/service/environment_store.spec.ts

@@ -5,6 +5,7 @@ import * as sinon from "sinon";
 import { components } from "../api";
 import { isPublisherConfigFrozen } from "../config";
 import { TEMP_DIR_PATH } from "../constants";
+import { BadRequestError } from "../errors";
 import { Environment } from "./environment";
 import { EnvironmentStore } from "./environment_store";
 
@@ -1020,3 +1021,105 @@ describe("Project Service Error Recovery", () => {
       );
    });
 });
+
+const TRAVERSAL_NAMES: ReadonlyArray<readonly [string, string]> = [
+   ["leading traversal", "../etc"],
+   ["embedded traversal", "foo/../../bar"],
+   ["slash in name", "foo/bar"],
+   ["backslash in name", "foo\\bar"],
+   ["leading dot", ".staging"],
+   ["bare dot-dot", ".."],
+   ["bare dot", "."],
+   ["empty", ""],
+   ["NUL byte", "foo\0bar"],
+   ["oversized", "a".repeat(256)],
+   ["absolute", "/etc/passwd"],
+] as const;
+
+describe("EnvironmentStore path-injection guards", () => {
+   let environmentStore: EnvironmentStore;
+
+   beforeEach(async () => {
+      if (existsSync(serverRootPath)) {
+         rmSync(serverRootPath, { recursive: true, force: true });
+      }
+      mkdirSync(serverRootPath);
+      mock(isPublisherConfigFrozen).mockReturnValue(false);
+      mock.module("../config", () => ({
+         isPublisherConfigFrozen: () => false,
+      }));
+      environmentStore = new EnvironmentStore(serverRootPath);
+      await environmentStore.finishedInitialization;
+   });
+
+   afterEach(() => {
+      if (existsSync(serverRootPath)) {
+         rmSync(serverRootPath, { recursive: true, force: true });
+      }
+      mkdirSync(serverRootPath);
+   });
+
+   describe("addEnvironment", () => {
+      it.each(TRAVERSAL_NAMES)(
+         "rejects %s as environment.name (%p)",
+         async (_label, name) => {
+            await expect(
+               environmentStore.addEnvironment({ name } as never, true),
+            ).rejects.toBeInstanceOf(BadRequestError);
+         },
+      );
+
+      it.each(TRAVERSAL_NAMES)(
+         "rejects %s as packages[].name (%p)",
+         async (_label, packageName) => {
+            await expect(
+               environmentStore.addEnvironment(
+                  {
+                     name: "ok-env",
+                     packages: [
+                        {
+                           name: packageName,
+                           location: "https://github.com/example/repo",
+                        },
+                     ],
+                  } as never,
+                  true,
+               ),
+            ).rejects.toBeInstanceOf(BadRequestError);
+         },
+      );
+   });
+
+   describe("updateEnvironment", () => {
+      it.each(TRAVERSAL_NAMES)(
+         "rejects %s as environment.name (%p)",
+         async (_label, name) => {
+            await expect(
+               environmentStore.updateEnvironment({ name } as never),
+            ).rejects.toBeInstanceOf(BadRequestError);
+         },
+      );
+   });
+
+   describe("deleteEnvironment", () => {
+      it.each(TRAVERSAL_NAMES)(
+         "rejects %s as environmentName (%p)",
+         async (_label, name) => {
+            await expect(
+               environmentStore.deleteEnvironment(name),
+            ).rejects.toBeInstanceOf(BadRequestError);
+         },
+      );
+   });
+
+   describe("getEnvironment", () => {
+      it.each(TRAVERSAL_NAMES)(
+         "rejects %s as environmentName (%p)",
+         async (_label, name) => {
+            await expect(
+               environmentStore.getEnvironment(name),
+            ).rejects.toBeInstanceOf(BadRequestError);
+         },
+      );
+   });
+});