@malloy-publisher/server 0.0.199 → 0.0.201

package/src/service/environment_store.ts

@@ -27,6 +27,11 @@ import {
 } from "../errors";
 import { getOperationalState, markNotReady, markReady } from "../health";
 import { formatDuration, logger } from "../logger";
+import {
+   assertSafeEnvironmentPath,
+   assertSafePackageName,
+   safeJoinUnderRoot,
+} from "../path_safety";
 import { Connection } from "../storage/DatabaseInterface";
 import { StorageConfig, StorageManager } from "../storage/StorageManager";
 import { Environment, PackageStatus } from "./environment";
@@ -756,6 +761,7 @@ export class EnvironmentStore {
       reload: boolean = false,
    ): Promise<Environment> {
       await this.finishedInitialization;
+      assertSafePackageName(environmentName);
 
       // Check if environment is already loaded first
       const environment = this.environments.get(environmentName);
@@ -816,9 +822,10 @@ export class EnvironmentStore {
       if (!skipInitialization && this.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
       }
+      assertSafePackageName(environment.name);
       const environmentName = environment.name;
-      if (!environmentName) {
-         throw new Error("Environment name is required");
+      for (const _package of environment.packages || []) {
+         assertSafePackageName(_package.name);
       }
       // Check if environment already exists and update it instead of creating a new one
       const existingEnvironment = this.environments.get(environmentName);
@@ -884,6 +891,7 @@ export class EnvironmentStore {
    }
 
    public async unzipEnvironment(absoluteEnvironmentPath: string) {
+      assertSafeEnvironmentPath(absoluteEnvironmentPath);
       const startedAt = Date.now();
       logger.info(
          `Detected zip file at "${absoluteEnvironmentPath}". Unzipping...`,
@@ -930,9 +938,10 @@ export class EnvironmentStore {
          throw new FrozenConfigError();
       }
       validateEnvironmentAzureUrls(environment);
+      assertSafePackageName(environment.name);
       const environmentName = environment.name;
-      if (!environmentName) {
-         throw new Error("Environment name is required");
+      for (const _package of environment.packages || []) {
+         assertSafePackageName(_package.name);
       }
       const existingEnvironment = this.environments.get(environmentName);
       if (!existingEnvironment) {
@@ -953,6 +962,7 @@ export class EnvironmentStore {
       if (this.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
       }
+      assertSafePackageName(environmentName);
       const environment = this.environments.get(environmentName);
       if (!environment) {
          return;
@@ -1027,15 +1037,17 @@ export class EnvironmentStore {
    }
 
    private async scaffoldEnvironment(environment: ApiEnvironment) {
+      assertSafePackageName(environment.name);
       const environmentName = environment.name;
-      if (!environmentName) {
-         throw new Error("Environment name is required");
-      }
-      const absoluteEnvironmentPath = `${this.serverRootPath}/${PUBLISHER_DATA_DIR}/${environmentName}`;
+      const absoluteEnvironmentPath = safeJoinUnderRoot(
+         this.serverRootPath,
+         PUBLISHER_DATA_DIR,
+         environmentName,
+      );
       await fs.promises.mkdir(absoluteEnvironmentPath, { recursive: true });
       if (environment.readme) {
          await fs.promises.writeFile(
-            path.join(absoluteEnvironmentPath, "README.md"),
+            safeJoinUnderRoot(absoluteEnvironmentPath, "README.md"),
             environment.readme,
          );
       }
@@ -1071,7 +1083,12 @@ export class EnvironmentStore {
       environmentName: string,
       packages: ApiEnvironment["packages"],
    ) {
-      const absoluteTargetPath = `${this.serverRootPath}/${PUBLISHER_DATA_DIR}/${environmentName}`;
+      assertSafePackageName(environmentName);
+      const absoluteTargetPath = safeJoinUnderRoot(
+         this.serverRootPath,
+         PUBLISHER_DATA_DIR,
+         environmentName,
+      );
 
       await fs.promises.mkdir(absoluteTargetPath, { recursive: true });
 
@@ -1126,7 +1143,10 @@ export class EnvironmentStore {
             .update(groupedLocation)
             .digest("hex")
             .substring(0, 16); // Use first 16 chars for shorter paths
-         const tempDownloadPath = `${absoluteTargetPath}/.temp_${locationHash}`;
+         const tempDownloadPath = safeJoinUnderRoot(
+            absoluteTargetPath,
+            `.temp_${locationHash}`,
+         );
          await fs.promises.mkdir(tempDownloadPath, { recursive: true });
          logger.info(`Created temporary directory: ${tempDownloadPath}`);
          try {
@@ -1140,7 +1160,11 @@ export class EnvironmentStore {
             // Extract each package from the downloaded content
             for (const _package of packagesForLocation) {
                const packageDir = _package.name;
-               const absolutePackagePath = `${absoluteTargetPath}/${packageDir}`;
+               assertSafePackageName(packageDir);
+               const absolutePackagePath = safeJoinUnderRoot(
+                  absoluteTargetPath,
+                  packageDir,
+               );
                // For GitHub URLs, extract the subdirectory path from the original location
                let sourcePath: string;
                if (this.isGitHubURL(_package.location)) {
@@ -1151,7 +1175,7 @@ export class EnvironmentStore {
                      const subPathMatch =
                         _package.location.match(/\/tree\/[^/]+\/(.+)$/);
                      if (subPathMatch) {
-                        sourcePath = path.join(
+                        sourcePath = safeJoinUnderRoot(
                            tempDownloadPath,
                            subPathMatch[1],
                         );
@@ -1168,7 +1192,10 @@ export class EnvironmentStore {
                   if (this.isLocalPath(_package.location)) {
                      sourcePath = _package.location;
                   } else {
-                     sourcePath = path.join(tempDownloadPath, groupedLocation);
+                     sourcePath = safeJoinUnderRoot(
+                        tempDownloadPath,
+                        groupedLocation,
+                     );
                   }
                }
 
@@ -1347,6 +1374,10 @@ export class EnvironmentStore {
       environmentName: string,
       packageName: string,
    ) {
+      // `environmentPath` is the operator-supplied mount source and may
+      // legitimately be a relative path that resolves outside the
+      // server root; only the target is asserted.
+      assertSafeEnvironmentPath(absoluteTargetPath);
       if (environmentPath.endsWith(".zip")) {
          environmentPath = await this.unzipEnvironment(environmentPath);
       }
@@ -1374,6 +1405,7 @@ export class EnvironmentStore {
       absoluteDirPath: string,
       isCompressedFile: boolean,
    ) {
+      assertSafeEnvironmentPath(absoluteDirPath);
       const trimmedPath = gcsPath.slice(5);
       const [bucketName, ...prefixParts] = trimmedPath.split("/");
       const prefix = prefixParts.join("/");
@@ -1396,10 +1428,15 @@ export class EnvironmentStore {
       }
       await Promise.all(
          files.map(async (file) => {
-            const relativeFilePath = file.name.replace(prefix, "");
+            // Strip leading `/` left over from prefix removal when the
+            // GCS prefix lacked a trailing slash — otherwise
+            // `safeJoinUnderRoot` treats it as absolute and rejects.
+            const relativeFilePath = file.name
+               .replace(prefix, "")
+               .replace(/^\/+/, "");
             const absoluteFilePath = isCompressedFile
                ? absoluteDirPath
-               : path.join(absoluteDirPath, relativeFilePath);
+               : safeJoinUnderRoot(absoluteDirPath, relativeFilePath);
             if (file.name.endsWith("/")) {
                return;
             }
@@ -1424,6 +1461,7 @@ export class EnvironmentStore {
       absoluteDirPath: string,
       isCompressedFile: boolean = false,
    ) {
+      assertSafeEnvironmentPath(absoluteDirPath);
       const trimmedPath = s3Path.slice(5);
       const [bucketName, ...prefixParts] = trimmedPath.split("/");
       const prefix = prefixParts.join("/");
@@ -1477,11 +1515,16 @@ export class EnvironmentStore {
             if (!key) {
                return;
             }
-            const relativeFilePath = key.replace(prefix, "");
+            // Strip leading `/` left over from prefix removal when the
+            // S3 prefix lacked a trailing slash — otherwise
+            // `safeJoinUnderRoot` treats it as absolute and rejects.
+            const relativeFilePath = key
+               .replace(prefix, "")
+               .replace(/^\/+/, "");
             if (!relativeFilePath || relativeFilePath.endsWith("/")) {
                return;
             }
-            const absoluteFilePath = path.join(
+            const absoluteFilePath = safeJoinUnderRoot(
                absoluteDirPath,
                relativeFilePath,
             );
@@ -1532,6 +1575,7 @@ export class EnvironmentStore {
    }
 
    async downloadGitHubDirectory(githubUrl: string, absoluteDirPath: string) {
+      assertSafeEnvironmentPath(absoluteDirPath);
       // First we'll clone the repo without the additional path
       // E.g. we're removing `/tree/main/imdb` from https://github.com/credibledata/malloy-samples/tree/main/imdb
       const githubInfo = this.parseGitHubUrl(githubUrl);
@@ -1539,7 +1583,11 @@ export class EnvironmentStore {
          throw new Error(`Invalid GitHub URL: ${githubUrl}`);
       }
       const { owner, repoName, packagePath } = githubInfo;
-      const cleanPackagePath = packagePath?.replace("/tree/main", "") || "";
+      // `packagePath` is captured with a leading `/`; strip it so the
+      // value is a relative segment usable with `safeJoinUnderRoot`.
+      const cleanPackagePath = (
+         packagePath?.replace("/tree/main", "") || ""
+      ).replace(/^\/+/, "");
 
       // We'll make sure whatever was in absoluteDirPath is removed,
       // so we have a nice a clean directory where we can clone the repo
@@ -1579,7 +1627,10 @@ export class EnvironmentStore {
 
       // Remove all contents of absoluteDirPath (/var/publisher/asd123)
       // except for the cleanPackagePath directory (/var/publisher/asd123/imdb)
-      const packageFullPath = path.join(absoluteDirPath, cleanPackagePath);
+      const packageFullPath = safeJoinUnderRoot(
+         absoluteDirPath,
+         cleanPackagePath,
+      );
 
       // Check if the cleanPackagePath (/var/publisher/asd123/imdb) exists
       const packageExists = await fs.promises
@@ -1598,7 +1649,7 @@ export class EnvironmentStore {
       for (const entry of dirContents) {
          // Don't remove the cleanPackagePath directory itself (/var/publisher/asd123/imdb)
          if (entry !== cleanPackagePath.replace(/^\/+/, "").split("/")[0]) {
-            await fs.promises.rm(path.join(absoluteDirPath, entry), {
+            await fs.promises.rm(safeJoinUnderRoot(absoluteDirPath, entry), {
                recursive: true,
                force: true,
             });
@@ -1609,8 +1660,8 @@ export class EnvironmentStore {
       const packageContents = await fs.promises.readdir(packageFullPath);
       for (const entry of packageContents) {
          await fs.promises.rename(
-            path.join(packageFullPath, entry),
-            path.join(absoluteDirPath, entry),
+            safeJoinUnderRoot(packageFullPath, entry),
+            safeJoinUnderRoot(absoluteDirPath, entry),
          );
       }
 

package/src/service/filter_integration.spec.ts

@@ -485,6 +485,75 @@ describe("filter integration", () => {
       });
    });
 
+   // -----------------------------------------------------------------------
+   // Mixed givens + #(filter) composition on the query-results path
+   //
+   // PR 5/6 of the givens migration deprecates `#(filter)` but keeps both
+   // injection paths working independently and in combination. These tests
+   // assert that `model.getQueryResults` (the path behind POST /…/query)
+   // composes a `given:` substitution with a `filterParams` WHERE-injection
+   // — same model, same call, both effects apply.
+   // -----------------------------------------------------------------------
+   describe("givens + filterParams composition", () => {
+      it("composes a given override with filterParams on getQueryResults", async () => {
+         await writeFile(
+            "orders_givens_filter.malloy",
+            MODEL_WITH_GIVENS_AND_FILTER,
+         );
+         const model = await Model.create(
+            "test-pkg",
+            TEST_PKG_DIR,
+            "orders_givens_filter.malloy",
+            getConnections(),
+         );
+
+         // given restricts to APAC; filterParam restricts to active.
+         // APAC + active: only (5,'APAC','active',300) → order_count=1, total_amount=300.
+         const { compactResult } = await model.getQueryResults(
+            "orders",
+            "by_given_region",
+            undefined,
+            { status: "active" },
+            undefined,
+            { target_region: "APAC" },
+         );
+
+         const r = asRows(compactResult);
+         expect(r.length).toBe(1);
+         expect(Number(r[0].order_count)).toBe(1);
+         expect(Number(r[0].total_amount)).toBe(300);
+      });
+
+      it("falls back to the given default when no override is supplied", async () => {
+         await writeFile(
+            "orders_givens_filter.malloy",
+            MODEL_WITH_GIVENS_AND_FILTER,
+         );
+         const model = await Model.create(
+            "test-pkg",
+            TEST_PKG_DIR,
+            "orders_givens_filter.malloy",
+            getConnections(),
+         );
+
+         // Default given target_region='US'; filterParam restricts to active.
+         // US + active: (1,'US','active',100) and (2,'US','active',200) → order_count=2, total_amount=300.
+         const { compactResult } = await model.getQueryResults(
+            "orders",
+            "by_given_region",
+            undefined,
+            { status: "active" },
+            undefined,
+            undefined,
+         );
+
+         const r = asRows(compactResult);
+         expect(r.length).toBe(1);
+         expect(Number(r[0].order_count)).toBe(2);
+         expect(Number(r[0].total_amount)).toBe(300);
+      });
+   });
+
    // -----------------------------------------------------------------------
    // Required filter enforcement
    // -----------------------------------------------------------------------

package/src/service/model.spec.ts

@@ -1,9 +1,13 @@
-import { MalloyError, Runtime } from "@malloydata/malloy";
-import { describe, expect, it } from "bun:test";
+import { API, MalloyError, Runtime } from "@malloydata/malloy";
+import { afterEach, describe, expect, it } from "bun:test";
 import fs from "fs/promises";
 import sinon from "sinon";
 
-import { BadRequestError, ModelNotFoundError } from "../errors";
+import {
+   BadRequestError,
+   ModelNotFoundError,
+   PayloadTooLargeError,
+} from "../errors";
 import { Model, ModelType } from "./model";
 
 describe("service/model", () => {
@@ -287,6 +291,192 @@ describe("service/model", () => {
 
             sinon.restore();
          });
+
+         /**
+          * The row/byte caps live in `model_limits.ts` (unit-tested in
+          * `model_limits.spec.ts`); these tests just confirm the wiring —
+          * that `Model.getQueryResults` calls the helpers with the right
+          * values and that an overflow propagates as `PayloadTooLargeError`
+          * (HTTP 413), not the generic `BadRequestError` (HTTP 400).
+          */
+         describe("response caps", () => {
+            const originalRowsEnv = process.env.PUBLISHER_MAX_QUERY_ROWS;
+            const originalBytesEnv = process.env.PUBLISHER_MAX_RESPONSE_BYTES;
+            const originalDefaultEnv =
+               process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT;
+
+            afterEach(() => {
+               sinon.restore();
+               for (const [name, original] of [
+                  ["PUBLISHER_MAX_QUERY_ROWS", originalRowsEnv],
+                  ["PUBLISHER_MAX_RESPONSE_BYTES", originalBytesEnv],
+                  ["PUBLISHER_DEFAULT_QUERY_ROW_LIMIT", originalDefaultEnv],
+               ] as const) {
+                  if (original === undefined) {
+                     delete process.env[name];
+                  } else {
+                     process.env[name] = original;
+                  }
+               }
+            });
+
+            /**
+             * Build a Model whose `runnable.run` resolves to a fake Result
+             * with the given totalRows; stub `API.util.wrapResult` so we
+             * don't need to construct a real Malloy schema/queryResult.
+             */
+            function buildModelWithFakeRun(opts: {
+               userLimit?: number;
+               totalRows: number;
+               wrappedJson: object;
+            }): { model: Model; runStub: sinon.SinonStub } {
+               const preparedResultStub = sinon
+                  .stub()
+                  .resolves({ resultExplore: { limit: opts.userLimit ?? 0 } });
+               const fakeResult = {
+                  _queryResult: { data: { rawData: [] } },
+                  totalRows: opts.totalRows,
+                  data: { value: [] },
+                  connectionName: "fake",
+               };
+               const runStub = sinon.stub().resolves(fakeResult);
+               sinon
+                  .stub(API.util, "wrapResult")
+                  .returns(
+                     opts.wrappedJson as unknown as ReturnType<
+                        typeof API.util.wrapResult
+                     >,
+                  );
+               const modelMaterializer = {
+                  loadQuery: sinon.stub().returns({
+                     getPreparedResult: preparedResultStub,
+                     run: runStub,
+                  }),
+               };
+               const model = new Model(
+                  packageName,
+                  mockModelPath,
+                  {},
+                  "model",
+                  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                  modelMaterializer as any,
+                  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                  { contents: {}, exports: [], queryList: [] } as any,
+                  undefined,
+                  undefined,
+                  undefined,
+                  undefined,
+                  undefined,
+               );
+               return { model, runStub };
+            }
+
+            it("clamps user LIMIT to maxRows + 1 when the user requested more than the cap", async () => {
+               process.env.PUBLISHER_MAX_QUERY_ROWS = "100";
+               const { model, runStub } = buildModelWithFakeRun({
+                  userLimit: 1_000_000,
+                  totalRows: 10,
+                  wrappedJson: { rows: [] },
+               });
+
+               await model.getQueryResults(
+                  undefined,
+                  undefined,
+                  "run: orders -> summary",
+               );
+
+               expect(runStub.firstCall.args[0].rowLimit).toBe(101);
+            });
+
+            it("passes user LIMIT through when below maxRows", async () => {
+               process.env.PUBLISHER_MAX_QUERY_ROWS = "100";
+               const { model, runStub } = buildModelWithFakeRun({
+                  userLimit: 50,
+                  totalRows: 10,
+                  wrappedJson: { rows: [] },
+               });
+
+               await model.getQueryResults(
+                  undefined,
+                  undefined,
+                  "run: orders -> summary",
+               );
+
+               expect(runStub.firstCall.args[0].rowLimit).toBe(50);
+            });
+
+            it("falls back to PUBLISHER_DEFAULT_QUERY_ROW_LIMIT when the user query has no LIMIT", async () => {
+               process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT = "42";
+               delete process.env.PUBLISHER_MAX_QUERY_ROWS;
+               const { model, runStub } = buildModelWithFakeRun({
+                  userLimit: 0,
+                  totalRows: 10,
+                  wrappedJson: { rows: [] },
+               });
+
+               await model.getQueryResults(
+                  undefined,
+                  undefined,
+                  "run: orders -> summary",
+               );
+
+               expect(runStub.firstCall.args[0].rowLimit).toBe(42);
+            });
+
+            it("throws PayloadTooLargeError (not BadRequestError) when totalRows exceeds the cap", async () => {
+               process.env.PUBLISHER_MAX_QUERY_ROWS = "100";
+               const { model } = buildModelWithFakeRun({
+                  userLimit: 1000,
+                  totalRows: 101,
+                  wrappedJson: { rows: [] },
+               });
+
+               await expect(
+                  model.getQueryResults(
+                     undefined,
+                     undefined,
+                     "run: orders -> summary",
+                  ),
+               ).rejects.toBeInstanceOf(PayloadTooLargeError);
+            });
+
+            it("throws PayloadTooLargeError when the wrapped response exceeds the byte cap", async () => {
+               process.env.PUBLISHER_MAX_QUERY_ROWS = "1000";
+               process.env.PUBLISHER_MAX_RESPONSE_BYTES = "100";
+               const huge = "x".repeat(500);
+               const { model } = buildModelWithFakeRun({
+                  userLimit: 10,
+                  totalRows: 10,
+                  wrappedJson: { rows: [{ s: huge }] },
+               });
+
+               await expect(
+                  model.getQueryResults(
+                     undefined,
+                     undefined,
+                     "run: orders -> summary",
+                  ),
+               ).rejects.toBeInstanceOf(PayloadTooLargeError);
+            });
+
+            it("does not throw when both counts are within their caps", async () => {
+               process.env.PUBLISHER_MAX_QUERY_ROWS = "1000";
+               process.env.PUBLISHER_MAX_RESPONSE_BYTES = "10000";
+               const { model } = buildModelWithFakeRun({
+                  userLimit: 10,
+                  totalRows: 10,
+                  wrappedJson: { rows: [{ a: 1 }] },
+               });
+
+               await expect(
+                  model.getQueryResults(
+                     undefined,
+                     undefined,
+                     "run: orders -> summary",
+                  ),
+               ).resolves.toBeDefined();
+            });
+         });
       });
 
       describe("executeNotebookCell", () => {