@malloy-publisher/server 0.0.198 → 0.0.200

package/src/service/environment_store.spec.ts

@@ -5,6 +5,7 @@ import * as sinon from "sinon";
 import { components } from "../api";
 import { isPublisherConfigFrozen } from "../config";
 import { TEMP_DIR_PATH } from "../constants";
+import { BadRequestError } from "../errors";
 import { Environment } from "./environment";
 import { EnvironmentStore } from "./environment_store";
 
@@ -1020,3 +1021,105 @@ describe("Project Service Error Recovery", () => {
       );
    });
 });
+
+const TRAVERSAL_NAMES: ReadonlyArray<readonly [string, string]> = [
+   ["leading traversal", "../etc"],
+   ["embedded traversal", "foo/../../bar"],
+   ["slash in name", "foo/bar"],
+   ["backslash in name", "foo\\bar"],
+   ["leading dot", ".staging"],
+   ["bare dot-dot", ".."],
+   ["bare dot", "."],
+   ["empty", ""],
+   ["NUL byte", "foo\0bar"],
+   ["oversized", "a".repeat(256)],
+   ["absolute", "/etc/passwd"],
+] as const;
+
+describe("EnvironmentStore path-injection guards", () => {
+   let environmentStore: EnvironmentStore;
+
+   beforeEach(async () => {
+      if (existsSync(serverRootPath)) {
+         rmSync(serverRootPath, { recursive: true, force: true });
+      }
+      mkdirSync(serverRootPath);
+      mock(isPublisherConfigFrozen).mockReturnValue(false);
+      mock.module("../config", () => ({
+         isPublisherConfigFrozen: () => false,
+      }));
+      environmentStore = new EnvironmentStore(serverRootPath);
+      await environmentStore.finishedInitialization;
+   });
+
+   afterEach(() => {
+      if (existsSync(serverRootPath)) {
+         rmSync(serverRootPath, { recursive: true, force: true });
+      }
+      mkdirSync(serverRootPath);
+   });
+
+   describe("addEnvironment", () => {
+      it.each(TRAVERSAL_NAMES)(
+         "rejects %s as environment.name (%p)",
+         async (_label, name) => {
+            await expect(
+               environmentStore.addEnvironment({ name } as never, true),
+            ).rejects.toBeInstanceOf(BadRequestError);
+         },
+      );
+
+      it.each(TRAVERSAL_NAMES)(
+         "rejects %s as packages[].name (%p)",
+         async (_label, packageName) => {
+            await expect(
+               environmentStore.addEnvironment(
+                  {
+                     name: "ok-env",
+                     packages: [
+                        {
+                           name: packageName,
+                           location: "https://github.com/example/repo",
+                        },
+                     ],
+                  } as never,
+                  true,
+               ),
+            ).rejects.toBeInstanceOf(BadRequestError);
+         },
+      );
+   });
+
+   describe("updateEnvironment", () => {
+      it.each(TRAVERSAL_NAMES)(
+         "rejects %s as environment.name (%p)",
+         async (_label, name) => {
+            await expect(
+               environmentStore.updateEnvironment({ name } as never),
+            ).rejects.toBeInstanceOf(BadRequestError);
+         },
+      );
+   });
+
+   describe("deleteEnvironment", () => {
+      it.each(TRAVERSAL_NAMES)(
+         "rejects %s as environmentName (%p)",
+         async (_label, name) => {
+            await expect(
+               environmentStore.deleteEnvironment(name),
+            ).rejects.toBeInstanceOf(BadRequestError);
+         },
+      );
+   });
+
+   describe("getEnvironment", () => {
+      it.each(TRAVERSAL_NAMES)(
+         "rejects %s as environmentName (%p)",
+         async (_label, name) => {
+            await expect(
+               environmentStore.getEnvironment(name),
+            ).rejects.toBeInstanceOf(BadRequestError);
+         },
+      );
+   });
+});

package/src/service/environment_store.ts

@@ -1,8 +1,8 @@
 import { GetObjectCommand, S3 } from "@aws-sdk/client-s3";
 import { Storage } from "@google-cloud/storage";
-import AdmZip from "adm-zip";
 import { Mutex } from "async-mutex";
 import crypto from "crypto";
+import extract from "extract-zip";
 import * as fs from "fs";
 import * as path from "path";
 import simpleGit from "simple-git";
@@ -27,6 +27,11 @@ import {
 } from "../errors";
 import { getOperationalState, markNotReady, markReady } from "../health";
 import { formatDuration, logger } from "../logger";
+import {
+   assertSafeEnvironmentPath,
+   assertSafePackageName,
+   safeJoinUnderRoot,
+} from "../path_safety";
 import { Connection } from "../storage/DatabaseInterface";
 import { StorageConfig, StorageManager } from "../storage/StorageManager";
 import { Environment, PackageStatus } from "./environment";
@@ -756,6 +761,7 @@ export class EnvironmentStore {
       reload: boolean = false,
    ): Promise<Environment> {
       await this.finishedInitialization;
+      assertSafePackageName(environmentName);
 
       // Check if environment is already loaded first
       const environment = this.environments.get(environmentName);
@@ -816,9 +822,10 @@ export class EnvironmentStore {
       if (!skipInitialization && this.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
       }
+      assertSafePackageName(environment.name);
       const environmentName = environment.name;
-      if (!environmentName) {
-         throw new Error("Environment name is required");
+      for (const _package of environment.packages || []) {
+         assertSafePackageName(_package.name);
       }
       // Check if environment already exists and update it instead of creating a new one
       const existingEnvironment = this.environments.get(environmentName);
@@ -884,6 +891,8 @@ export class EnvironmentStore {
    }
 
    public async unzipEnvironment(absoluteEnvironmentPath: string) {
+      assertSafeEnvironmentPath(absoluteEnvironmentPath);
+      const startedAt = Date.now();
       logger.info(
          `Detected zip file at "${absoluteEnvironmentPath}". Unzipping...`,
       );
@@ -897,8 +906,28 @@ export class EnvironmentStore {
       });
       await fs.promises.mkdir(unzippedEnvironmentPath, { recursive: true });
 
-      const zip = new AdmZip(absoluteEnvironmentPath);
-      zip.extractAllTo(unzippedEnvironmentPath, true);
+      // Stream-extract via yauzl (wrapped by extract-zip). Each entry's
+      // inflate and write are dispatched to the libuv thread pool, so the
+      // main event loop stays responsive even for very large archives.
+      // The previous adm-zip path used fs.readFileSync + zlib.inflateRawSync
+      // on the main thread, which parked the loop long enough on multi-
+      // hundred-MB packages to fail Kubernetes liveness probes mid-extract.
+      let entryCount = 0;
+      let totalUncompressedBytes = 0;
+      await extract(absoluteEnvironmentPath, {
+         dir: path.resolve(unzippedEnvironmentPath),
+         onEntry: (entry) => {
+            entryCount += 1;
+            totalUncompressedBytes += entry.uncompressedSize ?? 0;
+         },
+      });
+
+      const mib = (totalUncompressedBytes / (1024 * 1024)).toFixed(1);
+      logger.info(
+         `Unzipped "${absoluteEnvironmentPath}" -> "${unzippedEnvironmentPath}" ` +
+            `(${entryCount} entries, ${mib} MiB uncompressed) in ` +
+            `${formatDuration(Date.now() - startedAt)}`,
+      );
 
       return unzippedEnvironmentPath;
    }
@@ -909,9 +938,10 @@ export class EnvironmentStore {
          throw new FrozenConfigError();
       }
       validateEnvironmentAzureUrls(environment);
+      assertSafePackageName(environment.name);
       const environmentName = environment.name;
-      if (!environmentName) {
-         throw new Error("Environment name is required");
+      for (const _package of environment.packages || []) {
+         assertSafePackageName(_package.name);
       }
       const existingEnvironment = this.environments.get(environmentName);
       if (!existingEnvironment) {
@@ -932,6 +962,7 @@ export class EnvironmentStore {
       if (this.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
       }
+      assertSafePackageName(environmentName);
       const environment = this.environments.get(environmentName);
       if (!environment) {
          return;
@@ -1006,15 +1037,17 @@ export class EnvironmentStore {
    }
 
    private async scaffoldEnvironment(environment: ApiEnvironment) {
+      assertSafePackageName(environment.name);
       const environmentName = environment.name;
-      if (!environmentName) {
-         throw new Error("Environment name is required");
-      }
-      const absoluteEnvironmentPath = `${this.serverRootPath}/${PUBLISHER_DATA_DIR}/${environmentName}`;
+      const absoluteEnvironmentPath = safeJoinUnderRoot(
+         this.serverRootPath,
+         PUBLISHER_DATA_DIR,
+         environmentName,
+      );
       await fs.promises.mkdir(absoluteEnvironmentPath, { recursive: true });
       if (environment.readme) {
          await fs.promises.writeFile(
-            path.join(absoluteEnvironmentPath, "README.md"),
+            safeJoinUnderRoot(absoluteEnvironmentPath, "README.md"),
             environment.readme,
          );
       }
@@ -1050,7 +1083,12 @@ export class EnvironmentStore {
       environmentName: string,
       packages: ApiEnvironment["packages"],
    ) {
-      const absoluteTargetPath = `${this.serverRootPath}/${PUBLISHER_DATA_DIR}/${environmentName}`;
+      assertSafePackageName(environmentName);
+      const absoluteTargetPath = safeJoinUnderRoot(
+         this.serverRootPath,
+         PUBLISHER_DATA_DIR,
+         environmentName,
+      );
 
       await fs.promises.mkdir(absoluteTargetPath, { recursive: true });
 
@@ -1105,7 +1143,10 @@ export class EnvironmentStore {
             .update(groupedLocation)
             .digest("hex")
             .substring(0, 16); // Use first 16 chars for shorter paths
-         const tempDownloadPath = `${absoluteTargetPath}/.temp_${locationHash}`;
+         const tempDownloadPath = safeJoinUnderRoot(
+            absoluteTargetPath,
+            `.temp_${locationHash}`,
+         );
          await fs.promises.mkdir(tempDownloadPath, { recursive: true });
          logger.info(`Created temporary directory: ${tempDownloadPath}`);
          try {
@@ -1119,7 +1160,11 @@ export class EnvironmentStore {
             // Extract each package from the downloaded content
             for (const _package of packagesForLocation) {
                const packageDir = _package.name;
-               const absolutePackagePath = `${absoluteTargetPath}/${packageDir}`;
+               assertSafePackageName(packageDir);
+               const absolutePackagePath = safeJoinUnderRoot(
+                  absoluteTargetPath,
+                  packageDir,
+               );
                // For GitHub URLs, extract the subdirectory path from the original location
                let sourcePath: string;
                if (this.isGitHubURL(_package.location)) {
@@ -1130,7 +1175,7 @@ export class EnvironmentStore {
                      const subPathMatch =
                         _package.location.match(/\/tree\/[^/]+\/(.+)$/);
                      if (subPathMatch) {
-                        sourcePath = path.join(
+                        sourcePath = safeJoinUnderRoot(
                            tempDownloadPath,
                            subPathMatch[1],
                         );
@@ -1147,7 +1192,10 @@ export class EnvironmentStore {
                   if (this.isLocalPath(_package.location)) {
                      sourcePath = _package.location;
                   } else {
-                     sourcePath = path.join(tempDownloadPath, groupedLocation);
+                     sourcePath = safeJoinUnderRoot(
+                        tempDownloadPath,
+                        groupedLocation,
+                     );
                   }
                }
 
@@ -1326,6 +1374,10 @@ export class EnvironmentStore {
       environmentName: string,
       packageName: string,
    ) {
+      // `environmentPath` is the operator-supplied mount source and may
+      // legitimately be a relative path that resolves outside the
+      // server root; only the target is asserted.
+      assertSafeEnvironmentPath(absoluteTargetPath);
       if (environmentPath.endsWith(".zip")) {
          environmentPath = await this.unzipEnvironment(environmentPath);
       }
@@ -1353,6 +1405,7 @@ export class EnvironmentStore {
       absoluteDirPath: string,
       isCompressedFile: boolean,
    ) {
+      assertSafeEnvironmentPath(absoluteDirPath);
       const trimmedPath = gcsPath.slice(5);
       const [bucketName, ...prefixParts] = trimmedPath.split("/");
       const prefix = prefixParts.join("/");
@@ -1375,10 +1428,15 @@ export class EnvironmentStore {
       }
       await Promise.all(
          files.map(async (file) => {
-            const relativeFilePath = file.name.replace(prefix, "");
+            // Strip leading `/` left over from prefix removal when the
+            // GCS prefix lacked a trailing slash — otherwise
+            // `safeJoinUnderRoot` treats it as absolute and rejects.
+            const relativeFilePath = file.name
+               .replace(prefix, "")
+               .replace(/^\/+/, "");
             const absoluteFilePath = isCompressedFile
                ? absoluteDirPath
-               : path.join(absoluteDirPath, relativeFilePath);
+               : safeJoinUnderRoot(absoluteDirPath, relativeFilePath);
             if (file.name.endsWith("/")) {
                return;
             }
@@ -1403,6 +1461,7 @@ export class EnvironmentStore {
       absoluteDirPath: string,
       isCompressedFile: boolean = false,
    ) {
+      assertSafeEnvironmentPath(absoluteDirPath);
       const trimmedPath = s3Path.slice(5);
       const [bucketName, ...prefixParts] = trimmedPath.split("/");
       const prefix = prefixParts.join("/");
@@ -1456,11 +1515,16 @@ export class EnvironmentStore {
             if (!key) {
                return;
             }
-            const relativeFilePath = key.replace(prefix, "");
+            // Strip leading `/` left over from prefix removal when the
+            // S3 prefix lacked a trailing slash — otherwise
+            // `safeJoinUnderRoot` treats it as absolute and rejects.
+            const relativeFilePath = key
+               .replace(prefix, "")
+               .replace(/^\/+/, "");
             if (!relativeFilePath || relativeFilePath.endsWith("/")) {
                return;
             }
-            const absoluteFilePath = path.join(
+            const absoluteFilePath = safeJoinUnderRoot(
                absoluteDirPath,
                relativeFilePath,
             );
@@ -1511,6 +1575,7 @@ export class EnvironmentStore {
    }
 
    async downloadGitHubDirectory(githubUrl: string, absoluteDirPath: string) {
+      assertSafeEnvironmentPath(absoluteDirPath);
       // First we'll clone the repo without the additional path
       // E.g. we're removing `/tree/main/imdb` from https://github.com/credibledata/malloy-samples/tree/main/imdb
       const githubInfo = this.parseGitHubUrl(githubUrl);
@@ -1518,7 +1583,11 @@ export class EnvironmentStore {
          throw new Error(`Invalid GitHub URL: ${githubUrl}`);
       }
       const { owner, repoName, packagePath } = githubInfo;
-      const cleanPackagePath = packagePath?.replace("/tree/main", "") || "";
+      // `packagePath` is captured with a leading `/`; strip it so the
+      // value is a relative segment usable with `safeJoinUnderRoot`.
+      const cleanPackagePath = (
+         packagePath?.replace("/tree/main", "") || ""
+      ).replace(/^\/+/, "");
 
       // We'll make sure whatever was in absoluteDirPath is removed,
       // so we have a nice a clean directory where we can clone the repo
@@ -1558,7 +1627,10 @@ export class EnvironmentStore {
 
       // Remove all contents of absoluteDirPath (/var/publisher/asd123)
       // except for the cleanPackagePath directory (/var/publisher/asd123/imdb)
-      const packageFullPath = path.join(absoluteDirPath, cleanPackagePath);
+      const packageFullPath = safeJoinUnderRoot(
+         absoluteDirPath,
+         cleanPackagePath,
+      );
 
       // Check if the cleanPackagePath (/var/publisher/asd123/imdb) exists
       const packageExists = await fs.promises
@@ -1577,7 +1649,7 @@ export class EnvironmentStore {
       for (const entry of dirContents) {
          // Don't remove the cleanPackagePath directory itself (/var/publisher/asd123/imdb)
          if (entry !== cleanPackagePath.replace(/^\/+/, "").split("/")[0]) {
-            await fs.promises.rm(path.join(absoluteDirPath, entry), {
+            await fs.promises.rm(safeJoinUnderRoot(absoluteDirPath, entry), {
                recursive: true,
                force: true,
             });
@@ -1588,8 +1660,8 @@ export class EnvironmentStore {
       const packageContents = await fs.promises.readdir(packageFullPath);
       for (const entry of packageContents) {
          await fs.promises.rename(
-            path.join(packageFullPath, entry),
-            path.join(absoluteDirPath, entry),
+            safeJoinUnderRoot(packageFullPath, entry),
+            safeJoinUnderRoot(absoluteDirPath, entry),
          );
       }
 

package/src/service/filter_integration.spec.ts

@@ -133,6 +133,55 @@ import "child_orders.malloy"
 run: child_orders -> summary
 `;
 
+// Model with a given: declaration — view filters rows by the given value
+const MODEL_WITH_GIVENS = `##! experimental.givens
+
+given: target_region :: string is 'US'
+
+source: orders is duckdb.table('orders') extend {
+   primary_key: order_id
+
+   measure:
+      order_count is count()
+      total_amount is sum(amount)
+
+   view: by_given_region is {
+      where: region = $target_region
+      aggregate: order_count, total_amount
+   }
+}
+`;
+
+// Model with both a #(filter) annotation and a given: declaration to verify composition
+const MODEL_WITH_GIVENS_AND_FILTER = `##! experimental.givens
+
+given: target_region :: string is 'US'
+
+#(filter) dimension=status type=equal
+source: orders is duckdb.table('orders') extend {
+   primary_key: order_id
+
+   measure:
+      order_count is count()
+      total_amount is sum(amount)
+
+   view: by_given_region is {
+      where: region = $target_region
+      aggregate: order_count, total_amount
+   }
+}
+`;
+
+const NOTEBOOK_GIVENS = `>>>markdown
+# Givens Test
+
+>>>malloy
+import "orders_givens.malloy"
+
+>>>malloy
+run: orders -> by_given_region
+`;
+
 beforeAll(async () => {
    await fs.mkdir(TEST_DB_DIR, { recursive: true });
    await fs.mkdir(TEST_PKG_DIR, { recursive: true });
@@ -657,6 +706,67 @@ describe("filter integration", () => {
          expect(markdownCell.type).toBe("markdown");
          expect(markdownCell.text).toContain("Test Notebook");
       });
+
+      it("applies givens to notebook cell execution", async () => {
+         await writeFile("orders_givens.malloy", MODEL_WITH_GIVENS);
+         await writeFile("givens_notebook.malloynb", NOTEBOOK_GIVENS);
+         const model = await Model.create(
+            "test-pkg",
+            TEST_PKG_DIR,
+            "givens_notebook.malloynb",
+            getConnections(),
+         );
+
+         // Cell 2: run: orders -> by_given_region with target_region overridden to 'EU'
+         // EU rows: (3,'EU','active',150) and (4,'EU','cancelled',75) → order_count=2, total_amount=225
+         const codeCell = await model.executeNotebookCell(
+            2,
+            undefined,
+            undefined,
+            { target_region: "EU" },
+         );
+         expect(codeCell.result).toBeDefined();
+
+         const notebookRows = parseNotebookResult(codeCell.result!);
+         expect(notebookRows.length).toBe(1);
+         expect(Number(notebookRows[0].order_count)).toBe(2);
+         expect(Number(notebookRows[0].total_amount)).toBe(225);
+      });
+
+      it("composes givens and filterParams in notebook cell execution", async () => {
+         await writeFile(
+            "orders_givens_filter.malloy",
+            MODEL_WITH_GIVENS_AND_FILTER,
+         );
+         await writeFile(
+            "givens_filter_notebook.malloynb",
+            NOTEBOOK_GIVENS.replace(
+               "orders_givens.malloy",
+               "orders_givens_filter.malloy",
+            ),
+         );
+         const model = await Model.create(
+            "test-pkg",
+            TEST_PKG_DIR,
+            "givens_filter_notebook.malloynb",
+            getConnections(),
+         );
+
+         // given restricts to APAC; filterParam restricts to active
+         // APAC + active: only (5,'APAC','active',300) → order_count=1, total_amount=300
+         const codeCell = await model.executeNotebookCell(
+            2,
+            { status: "active" },
+            undefined,
+            { target_region: "APAC" },
+         );
+         expect(codeCell.result).toBeDefined();
+
+         const notebookRows = parseNotebookResult(codeCell.result!);
+         expect(notebookRows.length).toBe(1);
+         expect(Number(notebookRows[0].order_count)).toBe(1);
+         expect(Number(notebookRows[0].total_amount)).toBe(300);
+      });
    });
 
    // -----------------------------------------------------------------------

package/src/service/given.ts

@@ -0,0 +1,80 @@
+/**
+ * Shared utilities for surfacing Malloy `Given` declarations on
+ * compiled models.
+ *
+ * The Malloy SDK's `Given` class is declared in
+ * `@malloydata/malloy/dist/api/foundation/core.d.ts` but is not
+ * re-exported from the package root, so we duck-type against the
+ * surface we actually use and don't pull in the private type.
+ *
+ * Lives here so both the main-thread `Model` constructor and the
+ * package-load worker can use the same conversion. The worker
+ * imports this file directly (it's pure TypeScript with no native
+ * deps, so it's safe to bundle into the worker entry).
+ */
+
+/**
+ * Duck-typed shape of a Malloy SDK `Given` instance (the value type
+ * of `Model.givens`).
+ */
+export interface MalloyGiven {
+   readonly name: string;
+   readonly type: { type: string; filterType?: string };
+   getTaglines(prefix?: RegExp): string[];
+}
+
+/**
+ * Wire/API shape of a given. Structurally identical to the
+ * `components["schemas"]["Given"]` shape from the OpenAPI spec —
+ * callers can cast freely.
+ */
+export interface MalloyGivenApi {
+   name: string;
+   type: string;
+   annotations?: string[];
+}
+
+/**
+ * Convert a Malloy SDK `Given` to the wire/API shape.
+ *
+ * Two fields are deliberately not surfaced:
+ *
+ * - `location` — Malloy's `DocumentLocation.url` is an absolute
+ *   `file://` path on the publisher's filesystem. Surfacing it
+ *   would leak the OS user, install directory, and internal
+ *   layout. Existing `Filter` introspection does not expose
+ *   location either; matching that floor. A future PR can add a
+ *   sanitised package-relative path if a client needs it.
+ *
+ * - `default` / `defaultText` — Malloy's API only exposes the
+ *   parsed `ConstantExpr` AST, not a rendered source string.
+ *   Rendering it here would duplicate the Malloy printer. Add
+ *   when Malloy surfaces a stringified accessor.
+ *
+ * `annotations` is restricted to `#(...)` declaration annotations
+ * (the caller-facing kind, e.g. `#(doc)`). `getTaglines()` with no
+ * prefix would also return `##` doc-comment lines and the
+ * model-level `##!` pragma, which aren't part of the given's
+ * surface contract.
+ *
+ * Type rendering: `GivenTypeDef` is typed as `AtomicTypeDef |
+ * FilterExpressionParamTypeDef`, but Malloy's grammar only emits
+ * the scalar parameter types (`string` | `number` | `boolean` |
+ * `date` | `timestamp` | `timestamptz` | `filter expression` |
+ * `error`) for given declarations today. If the grammar expands
+ * to allow array or record givens, the bare `type.type`
+ * discriminator (`'array'`, `'record'`) will land in the wire
+ * response with no element info — revisit when that happens.
+ */
+export function malloyGivenToApi(given: MalloyGiven): MalloyGivenApi {
+   const type = given.type;
+   const renderedType =
+      type.type === "filter expression"
+         ? `filter<${type.filterType}>`
+         : type.type;
+   return {
+      name: given.name,
+      type: renderedType,
+      annotations: given.getTaglines(/^#\(/),
+   };
+}