@malloy-publisher/server 0.0.227 → 0.0.228

package/src/service/model.ts

@@ -1147,6 +1147,7 @@ export class Model {
             `Model compilation failed: ${this.compilationError.message}`,
          );
       }
+
       let runnable: QueryMaterializer;
       if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
          throw new BadRequestError("Model has no queryable entities.");
@@ -1309,16 +1310,23 @@ export class Model {
       // (for the row limit) and the run so a stale persist source falls back per
       // its declared policy — and prep/run agree on the same substitution.
       const buildManifest = this.resolveFreshBuildManifest();
-      const rowLimit = resolveModelQueryRowLimit(
-         (await runnable.getPreparedResult({ givens, buildManifest }))
-            .resultExplore.limit,
-         { defaultLimit: getDefaultQueryRowLimit(), maxRows },
-      );
-      const endTime = performance.now();
-      const executionTime = endTime - startTime;
 
+      // Prepare INSIDE the run try/catch: a bad-given / value-type throw at
+      // prepare time (getPreparedResult binds the givens) gets the same
+      // MalloyError→rethrow / else→400 handling as run, instead of escaping as
+      // a 500. `executionTime` is still captured after prepare and before run,
+      // preserving the pre-existing timing recorded by the success histogram.
+      let rowLimit = 0;
+      let executionTime = 0;
       let queryResults;
       try {
+         rowLimit = resolveModelQueryRowLimit(
+            (await runnable.getPreparedResult({ givens, buildManifest }))
+               .resultExplore.limit,
+            { defaultLimit: getDefaultQueryRowLimit(), maxRows },
+         );
+         executionTime = performance.now() - startTime;
+
          queryResults = await runnable.run({
             rowLimit,
             givens,
@@ -1337,6 +1345,31 @@ export class Model {
             "malloy.model.query.status": "error",
          });
 
+         // Bad client-supplied givens (unknown name, wrong-typed value, an
+         // operator-finalized override, ...) all surface as a Malloy
+         // `runtime-given-*` error. Malloy is the single validator; the publisher
+         // just maps its rejection to a clean 400. Duck-type on `.code`
+         // (MalloyCompileError extends Error, not MalloyError, and isn't
+         // root-exported). The `runtime-given-` prefix is a pinned coupling to
+         // Malloy's error codes (@malloydata/malloy given_binding.ts / runtime.ts);
+         // if they're renamed upstream, update it here (and in environment.ts) —
+         // otherwise these fall through to the generic 400 below with a worse
+         // message, and the /compile path silently omits `sql`.
+         const givenCode = (error as { code?: string })?.code;
+         if (
+            typeof givenCode === "string" &&
+            givenCode.startsWith("runtime-given-")
+         ) {
+            logger.debug("Rejected client-supplied given", {
+               environmentName: this.packageName,
+               modelPath: this.modelPath,
+               error: error instanceof Error ? error.message : String(error),
+            });
+            throw new BadRequestError(
+               error instanceof Error ? error.message : String(error),
+            );
+         }
+
          // Re-throw Malloy errors as-is (they will be handled by error handler)
          if (error instanceof MalloyError) {
             throw error;
@@ -1607,6 +1640,24 @@ export class Model {
             if (error instanceof FilterValidationError) {
                throw new BadRequestError(error.message);
             }
+            // Bad client-supplied givens (unknown name, wrong-typed value,
+            // finalized override, ...) surface as a Malloy `runtime-given-*`
+            // error; see getQueryResults. Malloy validates, the publisher maps
+            // to 400. Duck-type on `.code` (not a MalloyError, not root-exported).
+            const givenCode = (error as { code?: string })?.code;
+            if (
+               typeof givenCode === "string" &&
+               givenCode.startsWith("runtime-given-")
+            ) {
+               logger.debug("Rejected client-supplied given", {
+                  environmentName: this.packageName,
+                  modelPath: this.modelPath,
+                  error: error instanceof Error ? error.message : String(error),
+               });
+               throw new BadRequestError(
+                  error instanceof Error ? error.message : String(error),
+               );
+            }
             if (error instanceof MalloyError) {
                throw error;
             }

package/src/service/package.ts

@@ -237,6 +237,17 @@ export class Package {
       packageName: string,
       packagePath: string,
       environmentMalloyConfig: PackageConnectionInput,
+      /**
+       * Delete `packagePath` if the load fails. Opt-in, and only correct for a
+       * caller that created the directory itself (an install staged into place),
+       * where the half-built tree is Publisher's to clean up and `installPackage`
+       * rolls the previous one back. Every other caller loads a directory that
+       * already existed: a reload of a package that is currently serving, or a
+       * user directory registered via addPackage. Deleting those on a transient
+       * compile error destroys the source and takes the package offline, so the
+       * default is to leave the directory alone.
+       */
+      cleanupDirectoryOnFailure: boolean = false,
    ): Promise<Package> {
       assertSafeEnvironmentPath(packagePath);
       const startTime = performance.now();
@@ -278,23 +289,30 @@ export class Package {
             malloy_package_name: packageName,
             status: packageLoadFailureStatus(error),
          });
-         // Clean up the package directory on failure, but NOT when packagePath
-         // is an in-place mount symlink (watch mode). Removing it would unmount
-         // the package, so a transient compile error from a half-typed model
-         // saved mid-edit would brick the package until a restart. The symlink
-         // points at the user's live source, which is left untouched; the next
-         // save recompiles against it.
+         // Clean up the package directory only when the caller opted in (an
+         // install that staged this tree), and never when packagePath is an
+         // in-place mount symlink (watch mode). Removing it would unmount the
+         // package, so a transient compile error from a half-typed model saved
+         // mid-edit would brick the package until a restart. The symlink points
+         // at the user's live source, which is left untouched; the next save
+         // recompiles against it.
          try {
-            const stat = await fs.lstat(packagePath).catch(() => null);
-            if (stat?.isSymbolicLink()) {
+            if (!cleanupDirectoryOnFailure) {
                logger.info(
-                  `Skipping cleanup of symlinked package path on failure: ${packagePath}`,
+                  `Preserving existing package directory after failed load: ${packagePath}`,
                );
             } else {
-               await fs.rm(packagePath, { recursive: true, force: true });
-               logger.info(
-                  `Cleaned up failed package directory: ${packagePath}`,
-               );
+               const stat = await fs.lstat(packagePath).catch(() => null);
+               if (stat?.isSymbolicLink()) {
+                  logger.info(
+                     `Skipping cleanup of symlinked package path on failure: ${packagePath}`,
+                  );
+               } else {
+                  await fs.rm(packagePath, { recursive: true, force: true });
+                  logger.info(
+                     `Cleaned up failed package directory: ${packagePath}`,
+                  );
+               }
             }
          } catch (cleanupError) {
             logger.warn(`Failed to clean up package directory ${packagePath}`, {

package/src/service/package_reload_safety.spec.ts

@@ -0,0 +1,193 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import * as fs from "fs/promises";
+import * as os from "os";
+import * as path from "path";
+
+import { Environment, PackageStatus } from "./environment";
+
+/**
+ * Regression tests for RELOAD data loss.
+ *
+ * `Package.create`'s failure cleanup was written for a fresh install, where the
+ * half-built tree is Publisher's to remove. A reload runs the same code against
+ * a directory that already exists and is already serving, so a model that
+ * failed to compile deleted the user's package directory and evicted the
+ * last-good compiled model, taking the package offline until it was
+ * re-provisioned. That is reachable from the reload endpoint and the
+ * malloy_reloadPackage MCP tool by any caller that saves a broken model and
+ * reloads, which is the ordinary authoring mistake.
+ *
+ * Directory cleanup is now opt-in and only `installPackage` asks for it, so
+ * these two guard the reload path. Run against a real `Environment` and a real
+ * `Package.create` over temp dirs, the same way package_rollback.spec.ts does.
+ */
+describe("failed reload does not destroy a serving package", () => {
+   let rootDir: string;
+   let envPath: string;
+
+   const GOOD_MODEL = `source: ones is duckdb.sql("SELECT 1 as x")\n`;
+   const BROKEN_MODEL = `source: broken is @@@ not valid malloy !!!\n`;
+
+   async function writePackageDir(dir: string, model: string): Promise<void> {
+      await fs.mkdir(dir, { recursive: true });
+      await fs.writeFile(
+         path.join(dir, "publisher.json"),
+         JSON.stringify({ name: "pkg", description: "reload fixture" }),
+      );
+      await fs.writeFile(path.join(dir, "model.malloy"), model);
+   }
+
+   async function copyDir(src: string, dst: string): Promise<void> {
+      await fs.mkdir(dst, { recursive: true });
+      await fs.cp(src, dst, { recursive: true });
+   }
+
+   beforeEach(async () => {
+      rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-reload-"));
+      envPath = path.join(rootDir, "env");
+      await fs.mkdir(envPath, { recursive: true });
+   });
+
+   afterEach(async () => {
+      await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
+   });
+
+   it("keeps the package directory on disk when the reloaded model does not compile", async () => {
+      const env = await Environment.create("testEnv", envPath, []);
+      const pkgDir = path.join(envPath, "pkg");
+      await writePackageDir(pkgDir, GOOD_MODEL);
+      await env.addPackage("pkg");
+
+      // The ordinary authoring mistake: save something that does not compile,
+      // then reload.
+      await fs.writeFile(path.join(pkgDir, "model.malloy"), BROKEN_MODEL);
+      await expect(env.getPackage("pkg", true)).rejects.toThrow();
+
+      // The directory and the file the user was editing both survive.
+      const modelText = await fs.readFile(
+         path.join(pkgDir, "model.malloy"),
+         "utf-8",
+      );
+      expect(modelText).toBe(BROKEN_MODEL);
+   });
+
+   it("keeps serving the last good model after a failed reload", async () => {
+      const env = await Environment.create("testEnv", envPath, []);
+      const pkgDir = path.join(envPath, "pkg");
+      await writePackageDir(pkgDir, GOOD_MODEL);
+      await env.addPackage("pkg");
+
+      const servingBefore = await env.getPackage("pkg", false);
+
+      await fs.writeFile(path.join(pkgDir, "model.malloy"), BROKEN_MODEL);
+      await expect(env.getPackage("pkg", true)).rejects.toThrow();
+
+      // The package is still loaded and answerable: a failed reload reports the
+      // compile error, it does not take the package down. Assert the exact
+      // state, not just that something is there: a status stranded at LOADING
+      // would satisfy toBeDefined() while listPackages skips it, so the package
+      // would answer getPackage and be invisible to listings and discovery.
+      const stillServing = await env.getPackage("pkg", false);
+      expect(stillServing).toBe(servingBefore);
+      expect(env.getPackageStatus("pkg")?.status).toBe(PackageStatus.SERVING);
+      expect((await env.listPackages()).map((p) => p.name)).toContain("pkg");
+   });
+
+   it("keeps a rolled-back reinstall listed and serving", async () => {
+      // The other reload path. A package with an install location reloads via
+      // installPackage, whose rollback restores the previous tree but used to
+      // drop the status while `packages` kept the old package: it answered
+      // getPackage but vanished from listPackages and discovery until a
+      // restart. Both maps must agree that it is still serving.
+      const env = await Environment.create("testEnv", envPath, []);
+      const goodFixture = path.join(rootDir, "good");
+      const brokenFixture = path.join(rootDir, "broken");
+      await writePackageDir(goodFixture, GOOD_MODEL);
+      await writePackageDir(brokenFixture, BROKEN_MODEL);
+
+      await env.installPackage("pkg", (stagingPath) =>
+         copyDir(goodFixture, stagingPath),
+      );
+      const servingBefore = await env.getPackage("pkg", false);
+
+      await expect(
+         env.installPackage("pkg", (stagingPath) =>
+            copyDir(brokenFixture, stagingPath),
+         ),
+      ).rejects.toThrow();
+
+      expect(env.getPackageStatus("pkg")?.status).toBe(PackageStatus.SERVING);
+      expect((await env.listPackages()).map((p) => p.name)).toContain("pkg");
+      expect(await env.getPackage("pkg", false)).toBe(servingBefore);
+      // The on-disk half of the same claim: the rejected tree is gone and the
+      // user's is back. Without this, nothing pins "your files are left alone".
+      expect(
+         await fs.readFile(path.join(envPath, "pkg", "model.malloy"), "utf-8"),
+      ).toBe(GOOD_MODEL);
+   });
+
+   it("does not advertise a package as serving when the rollback could not restore it", async () => {
+      // Keeping the last-good package served is only honest when the rollback
+      // actually put its tree back. If nothing was restored, the cached package
+      // no longer matches disk, and claiming SERVING would advertise a package
+      // over a canonical path that is missing or still holds rejected content.
+      const env = await Environment.create("testEnv", envPath, []);
+      const pkgDir = path.join(envPath, "pkg");
+      await writePackageDir(pkgDir, GOOD_MODEL);
+      await env.addPackage("pkg");
+      expect(env.getPackageStatus("pkg")?.status).toBe(PackageStatus.SERVING);
+
+      // Record retirements rather than scheduling the real drain, which is a
+      // 30s timer this test has no reason to wait on.
+      const retiredLabels: string[] = [];
+      (
+         env as unknown as {
+            retireConnectionGeneration: (label: string) => void;
+         }
+      ).retireConnectionGeneration = (label: string) => {
+         retiredLabels.push(label);
+      };
+
+      // The tree disappears from under Publisher, so the reinstall has nothing
+      // to retire and the rollback has nothing to restore.
+      await fs.rm(pkgDir, { recursive: true, force: true });
+
+      const brokenFixture = path.join(rootDir, "broken-nofallback");
+      await writePackageDir(brokenFixture, BROKEN_MODEL);
+      await expect(
+         env.installPackage("pkg", (stagingPath) =>
+            copyDir(brokenFixture, stagingPath),
+         ),
+      ).rejects.toThrow();
+
+      // Eviction is the honest outcome here, and both maps must agree on it.
+      expect(env.getPackageStatus("pkg")).toBeUndefined();
+      expect((await env.listPackages()).map((p) => p.name)).not.toContain(
+         "pkg",
+      );
+      // Assert the packages map too, not just the status: asserting only the
+      // status leaves the half that actually strands a package untested.
+      await expect(env.getPackage("pkg", false)).rejects.toThrow();
+
+      // Evicting must retire the old package's connections. Once it leaves
+      // this.packages, closeAllConnections cannot reach its MalloyConfig, so
+      // skipping the retire orphans its native handles for the process's life.
+      // The drain is timer-based, so assert it was scheduled, not that it ran.
+      expect(retiredLabels).toContain("package pkg");
+   });
+
+   it("recovers on the next reload once the model compiles again", async () => {
+      const env = await Environment.create("testEnv", envPath, []);
+      const pkgDir = path.join(envPath, "pkg");
+      await writePackageDir(pkgDir, GOOD_MODEL);
+      await env.addPackage("pkg");
+
+      await fs.writeFile(path.join(pkgDir, "model.malloy"), BROKEN_MODEL);
+      await expect(env.getPackage("pkg", true)).rejects.toThrow();
+
+      // Because nothing was deleted, fixing the model and reloading is enough.
+      await fs.writeFile(path.join(pkgDir, "model.malloy"), GOOD_MODEL);
+      const reloaded = await env.getPackage("pkg", true);
+      expect(reloaded).toBeDefined();
+   });
+});

package/tests/fixtures/query-givens/data/orders.csv

@@ -0,0 +1,7 @@
+id,region,status,amount
+1,US,active,100
+2,US,active,200
+3,US,cancelled,50
+4,EU,active,300
+5,EU,cancelled,400
+6,EU,active,500

package/tests/fixtures/query-givens/model.malloy

@@ -0,0 +1,34 @@
+##! experimental.givens
+
+given: target_region :: string is 'US'
+given: min_amount :: number is 0
+given: role :: string is 'guest'
+
+#(filter) dimension=status type=equal
+source: orders is duckdb.table('data/orders.csv') extend {
+   primary_key: id
+
+   measure:
+      order_count is count()
+
+   view: by_given_region is {
+      where: region = $target_region and amount >= $min_amount
+      aggregate: order_count
+   }
+}
+
+// Gated variant for authorize-interaction coverage: only $role = 'admin' may
+// read it. Shares the givens above so we can prove the name-vs-value-vs-authorize
+// status matrix on a gated source.
+#(authorize) "$role = 'admin'"
+source: gated_orders is duckdb.table('data/orders.csv') extend {
+   primary_key: id
+
+   measure:
+      order_count is count()
+
+   view: by_given_region is {
+      where: region = $target_region and amount >= $min_amount
+      aggregate: order_count
+   }
+}

package/tests/fixtures/query-givens/publisher.json

@@ -0,0 +1,5 @@
+{
+  "name": "query-givens",
+  "version": "1.0.0",
+  "description": "Fixture for /query givens forwarding (server + SDK)"
+}

package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts

@@ -23,8 +23,8 @@ describe.serial("MCP Tool Handlers (E2E Integration)", () => {
    let env: McpE2ETestEnvironment | null = null;
    let mcpClient: Client;
 
-   const ENVIRONMENT_NAME = "malloy-samples";
-   const PACKAGE_NAME = "faa";
+   const ENVIRONMENT_NAME = "examples";
+   const PACKAGE_NAME = "storefront";
 
    beforeAll(async () => {
       // Setup the E2E environment (starts server, connects client)
@@ -49,8 +49,8 @@ describe.serial("MCP Tool Handlers (E2E Integration)", () => {
                arguments: {
                   environmentName: ENVIRONMENT_NAME,
                   packageName: PACKAGE_NAME,
-                  modelPath: "flights.malloy",
-                  query: "run: flights->{ aggregate: c is count() }",
+                  modelPath: "storefront.malloy",
+                  query: "run: order_items->{ aggregate: c is count() }",
                },
             });
 
@@ -97,9 +97,9 @@ describe.serial("MCP Tool Handlers (E2E Integration)", () => {
             const params = {
                environmentName: ENVIRONMENT_NAME,
                packageName: PACKAGE_NAME,
-               modelPath: "flights.malloy",
-               sourceName: "flights", // Added sourceName
-               queryName: "top_carriers",
+               modelPath: "storefront.malloy",
+               sourceName: "order_items", // Added sourceName
+               queryName: "top_products",
             };
 
             // Expect RESOLUTION with success
@@ -142,8 +142,8 @@ describe.serial("MCP Tool Handlers (E2E Integration)", () => {
             const params = {
                environmentName: ENVIRONMENT_NAME,
                packageName: PACKAGE_NAME,
-               modelPath: "flights.malloy",
-               query: "run: flights->{BAD SYNTAX aggregate: flight_count is count()}",
+               modelPath: "storefront.malloy",
+               query: "run: order_items->{BAD SYNTAX aggregate: order_count is count()}",
             };
 
             // Application Error (Malloy Compilation): Expect RESOLUTION with isError: true
@@ -181,9 +181,9 @@ describe.serial("MCP Tool Handlers (E2E Integration)", () => {
          const params = {
             environmentName: ENVIRONMENT_NAME,
             packageName: PACKAGE_NAME,
-            modelPath: "flights.malloy",
-            query: "run: flights->{aggregate: c is count()}",
-            queryName: "top_carriers",
+            modelPath: "storefront.malloy",
+            query: "run: order_items->{aggregate: c is count()}",
+            queryName: "top_products",
          };
 
          // Expect RESOLUTION because the error is thrown *inside* the handler
@@ -209,7 +209,7 @@ describe.serial("MCP Tool Handlers (E2E Integration)", () => {
          const params = {
             environmentName: ENVIRONMENT_NAME,
             packageName: PACKAGE_NAME,
-            modelPath: "flights.malloy",
+            modelPath: "storefront.malloy",
             // Missing query AND queryName
          };
 
@@ -237,7 +237,7 @@ describe.serial("MCP Tool Handlers (E2E Integration)", () => {
             // Missing modelPath
             environmentName: ENVIRONMENT_NAME,
             packageName: PACKAGE_NAME,
-            query: "run: flights->{aggregate: flight_count is count()}",
+            query: "run: order_items->{aggregate: c is count()}",
          };
 
          // Protocol Error (Caught by Zod/MCP): Expect REJECTION
@@ -259,8 +259,8 @@ describe.serial("MCP Tool Handlers (E2E Integration)", () => {
          const params = {
             environmentName: ENVIRONMENT_NAME,
             packageName: "nonexistent_package", // Use a package that doesn't exist
-            modelPath: "flights.malloy",
-            query: "run: flights->{aggregate: c is count()}",
+            modelPath: "storefront.malloy",
+            query: "run: order_items->{aggregate: c is count()}",
          };
 
          // Application Error (Service Layer): Expect RESOLUTION with isError: true
@@ -304,7 +304,7 @@ describe.serial("MCP Tool Handlers (E2E Integration)", () => {
             environmentName: ENVIRONMENT_NAME,
             packageName: PACKAGE_NAME,
             modelPath: "nonexistent_model.malloy", // Use a model that doesn't exist
-            query: "run: flights->{aggregate: c is count()}",
+            query: "run: order_items->{aggregate: c is count()}",
          };
 
          // Application Error (Service Layer): Expect RESOLUTION with isError: true
@@ -364,8 +364,8 @@ describe.serial("MCP Tool Handlers (E2E Integration)", () => {
                arguments: {
                   environmentName: ENVIRONMENT_NAME,
                   packageName: PACKAGE_NAME,
-                  modelPath: "flights.malloy",
-                  query: "run: flights->{aggregate: c is count()}",
+                  modelPath: "storefront.malloy",
+                  query: "run: order_items->{aggregate: c is count()}",
                },
             }),
          ).rejects.toThrow();
@@ -377,8 +377,8 @@ describe.serial("MCP Tool Handlers (E2E Integration)", () => {
          const params = {
             environmentName: ENVIRONMENT_NAME,
             packageName: PACKAGE_NAME,
-            modelPath: "flights.malloy",
-            queryName: "top_carriers", // Nested view, but sourceName is missing
+            modelPath: "storefront.malloy",
+            queryName: "top_products", // Nested view, but sourceName is missing
          };
 
          // Expect RESOLUTION with error because it's invalid usage processed by the handler
@@ -405,7 +405,7 @@ describe.serial("MCP Tool Handlers (E2E Integration)", () => {
          const errorPayload = JSON.parse(errorJsonText);
          // Expect error about the query/view itself not found
          expect(errorPayload.error).toMatch(
-            /Query 'top_carriers' not found|Reference to undefined object 'top_carriers'/i,
+            /Query 'top_products' not found|Reference to undefined object 'top_products'/i,
          );
          expect(Array.isArray(errorPayload.suggestions)).toBe(true);
       });