@malloy-publisher/server 0.0.205 → 0.0.206

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.205",
+   "version": "0.0.206",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"

package/src/constants.ts

@@ -5,6 +5,18 @@ export const PUBLISHER_CONFIG_NAME = "publisher.config.json";
 export const PACKAGE_MANIFEST_NAME = "publisher.json";
 export const MODEL_FILE_SUFFIX = ".malloy";
 export const NOTEBOOK_FILE_SUFFIX = ".malloynb";
+
+/**
+ * Normalize a package-relative model path so author-written `explores`
+ * entries compare equal to the paths produced by `listPackageFiles`
+ * (forward slashes, no leading "./"). Shared by every input channel — the
+ * worker's on-disk parse and the API publish/update path — so a manifest
+ * behaves the same whether it's loaded from disk or written via the API.
+ */
+export function normalizeModelPath(p: string): string {
+   return p.replace(/\\/g, "/").replace(/^\.\//, "");
+}
+
 /**
  * Default row cap applied to Malloy model queries (the `runnable.run`
  * path used by `getQueryResults` and notebook cell execution) when

package/src/controller/materialization.controller.ts

@@ -1,4 +1,5 @@
 import { BadRequestError } from "../errors";
+import { BuildInstruction } from "../storage/DatabaseInterface";
 import { MaterializationService } from "../service/materialization_service";
 
 export class MaterializationController {
@@ -9,46 +10,113 @@ export class MaterializationController {
       packageName: string,
       body: Record<string, unknown>,
    ) {
-      const options = this.validateCreateBody(body);
       return this.materializationService.createMaterialization(
          environmentName,
          packageName,
-         options,
+         this.validateCreateBody(body),
       );
    }
 
    private validateCreateBody(body: Record<string, unknown>): {
       forceRefresh?: boolean;
-      autoLoadManifest?: boolean;
+      sourceNames?: string[];
+      pauseBetweenPhases?: boolean;
    } {
-      const result: { forceRefresh?: boolean; autoLoadManifest?: boolean } = {};
+      const result: {
+         forceRefresh?: boolean;
+         sourceNames?: string[];
+         pauseBetweenPhases?: boolean;
+      } = {};
+      if (body.pauseBetweenPhases !== undefined) {
+         if (typeof body.pauseBetweenPhases !== "boolean") {
+            throw new BadRequestError("pauseBetweenPhases must be a boolean");
+         }
+         result.pauseBetweenPhases = body.pauseBetweenPhases;
+      }
       if (body.forceRefresh !== undefined) {
          if (typeof body.forceRefresh !== "boolean") {
             throw new BadRequestError("forceRefresh must be a boolean");
          }
          result.forceRefresh = body.forceRefresh;
       }
-      if (body.autoLoadManifest !== undefined) {
-         if (typeof body.autoLoadManifest !== "boolean") {
-            throw new BadRequestError("autoLoadManifest must be a boolean");
+      if (body.sourceNames !== undefined) {
+         if (
+            !Array.isArray(body.sourceNames) ||
+            body.sourceNames.some((n) => typeof n !== "string")
+         ) {
+            throw new BadRequestError(
+               "sourceNames must be an array of strings",
+            );
          }
-         result.autoLoadManifest = body.autoLoadManifest;
+         result.sourceNames = body.sourceNames as string[];
       }
       return result;
    }
 
-   async startMaterialization(
+   async buildMaterialization(
       environmentName: string,
       packageName: string,
       materializationId: string,
+      body: Record<string, unknown>,
    ) {
-      return this.materializationService.startMaterialization(
+      return this.materializationService.buildMaterialization(
          environmentName,
          packageName,
          materializationId,
+         this.validateBuildBody(body),
       );
    }
 
+   private validateBuildBody(
+      body: Record<string, unknown>,
+   ): BuildInstruction[] {
+      const sources = body.sources;
+      if (!Array.isArray(sources) || sources.length === 0) {
+         throw new BadRequestError(
+            "build requires a non-empty 'sources' array of BuildInstruction",
+         );
+      }
+      return sources.map((raw) => this.validateInstruction(raw));
+   }
+
+   private validateInstruction(raw: unknown): BuildInstruction {
+      if (typeof raw !== "object" || raw === null) {
+         throw new BadRequestError("Each build instruction must be an object");
+      }
+      const instruction = raw as Record<string, unknown>;
+      const required = [
+         "buildId",
+         "materializedTableId",
+         "physicalTableName",
+         "realization",
+      ] as const;
+      for (const field of required) {
+         if (typeof instruction[field] !== "string") {
+            throw new BadRequestError(
+               `Build instruction is missing required string field '${field}'`,
+            );
+         }
+      }
+      if (
+         instruction.realization !== "COPY" &&
+         instruction.realization !== "SNAPSHOT"
+      ) {
+         throw new BadRequestError(
+            "Build instruction 'realization' must be COPY or SNAPSHOT",
+         );
+      }
+      return {
+         buildId: instruction.buildId as string,
+         sourceID:
+            typeof instruction.sourceID === "string"
+               ? instruction.sourceID
+               : undefined,
+         materializedTableId: instruction.materializedTableId as string,
+         physicalTableName: instruction.physicalTableName as string,
+         realization: instruction.realization,
+      };
+   }
+
    async stopMaterialization(
       environmentName: string,
       packageName: string,
@@ -89,37 +157,13 @@ export class MaterializationController {
       environmentName: string,
       packageName: string,
       materializationId: string,
+      options: { dropTables?: boolean } = {},
    ) {
       return this.materializationService.deleteMaterialization(
          environmentName,
          packageName,
          materializationId,
-      );
-   }
-
-   async teardownPackage(
-      environmentName: string,
-      packageName: string,
-      body: Record<string, unknown>,
-   ) {
-      const options = this.validateTeardownBody(body);
-      return this.materializationService.teardownPackage(
-         environmentName,
-         packageName,
          options,
       );
    }
-
-   private validateTeardownBody(body: Record<string, unknown>): {
-      dryRun?: boolean;
-   } {
-      const options: { dryRun?: boolean } = {};
-      if (body.dryRun !== undefined) {
-         if (typeof body.dryRun !== "boolean") {
-            throw new BadRequestError("dryRun must be a boolean");
-         }
-         options.dryRun = body.dryRun;
-      }
-      return options;
-   }
 }

package/src/controller/package.controller.spec.ts

@@ -0,0 +1,179 @@
+import { afterEach, describe, expect, it } from "bun:test";
+import sinon from "sinon";
+
+import { BadRequestError } from "../errors";
+import type { EnvironmentStore } from "../service/environment_store";
+import { PackageController } from "./package.controller";
+
+describe("PackageController.addPackage explores validation", () => {
+   afterEach(() => {
+      sinon.restore();
+   });
+
+   it("no-location: rejects invalid explores and rolls back via unloadPackage (NOT deletePackage)", async () => {
+      // The no-location path registers a PRE-EXISTING user directory, so a bad
+      // manifest must unload it from memory — never deletePackage, which would
+      // delete the user's files.
+      const invalidMsg =
+         "Invalid explores entry 'missing.malloy' in publisher.json: file not found";
+      const mockPackage = {
+         formatInvalidExplores: () => invalidMsg,
+      };
+      const unloadPackage = sinon.stub().resolves(undefined);
+      const deletePackage = sinon.stub().resolves(undefined);
+      const addPackage = sinon.stub().resolves(mockPackage);
+      const environment = { addPackage, unloadPackage, deletePackage };
+      const getEnvironment = sinon.stub().resolves(environment);
+      const addPackageToDatabase = sinon.stub().resolves(undefined);
+      const environmentStore = {
+         publisherConfigIsFrozen: false,
+         getEnvironment,
+         addPackageToDatabase,
+      } as unknown as EnvironmentStore;
+
+      const controller = new PackageController(environmentStore);
+
+      await expect(
+         controller.addPackage("env", {
+            name: "pkg",
+            description: "test",
+            explores: ["missing.malloy"],
+         }),
+      ).rejects.toBeInstanceOf(BadRequestError);
+
+      expect(unloadPackage.calledOnceWith("pkg")).toBe(true);
+      expect(deletePackage.called).toBe(false);
+      expect(addPackageToDatabase.called).toBe(false);
+   });
+
+   it("location: validation runs inside installPackage's rollback window, not as a controller delete", async () => {
+      // For the location path the tree was freshly downloaded, so validation is
+      // delegated to installPackage (which rolls the swap back on failure). The
+      // controller passes a validator and does NOT call delete/unload itself.
+      const invalidMsg =
+         "Invalid explores entry 'missing.malloy' in publisher.json: file not found";
+      const mockPackage = { formatInvalidExplores: () => invalidMsg };
+      // installPackage mimics the real contract: invoke the validator and, if it
+      // returns a message, throw BadRequestError (after its internal rollback).
+      const installPackage = sinon
+         .stub()
+         .callsFake(
+            async (
+               _name: string,
+               _downloader: unknown,
+               validate?: (pkg: unknown) => string | undefined,
+            ) => {
+               const msg = validate?.(mockPackage);
+               if (msg) throw new BadRequestError(msg);
+               return mockPackage;
+            },
+         );
+      const unloadPackage = sinon.stub().resolves(undefined);
+      const deletePackage = sinon.stub().resolves(undefined);
+      const environment = { installPackage, unloadPackage, deletePackage };
+      const getEnvironment = sinon.stub().resolves(environment);
+      const addPackageToDatabase = sinon.stub().resolves(undefined);
+      const environmentStore = {
+         publisherConfigIsFrozen: false,
+         getEnvironment,
+         addPackageToDatabase,
+      } as unknown as EnvironmentStore;
+
+      const controller = new PackageController(environmentStore);
+
+      await expect(
+         controller.addPackage("env", {
+            name: "pkg",
+            description: "test",
+            location: "gs://bucket/pkg.zip",
+            explores: ["missing.malloy"],
+         }),
+      ).rejects.toBeInstanceOf(BadRequestError);
+
+      expect(installPackage.calledOnce).toBe(true);
+      expect(typeof installPackage.firstCall.args[2]).toBe("function");
+      expect(deletePackage.called).toBe(false);
+      expect(unloadPackage.called).toBe(false);
+      expect(addPackageToDatabase.called).toBe(false);
+   });
+
+   it("persists when explores are valid (no-location)", async () => {
+      const mockPackage = {
+         formatInvalidExplores: () => "",
+      };
+      const addPackage = sinon.stub().resolves(mockPackage);
+      const getEnvironment = sinon.stub().resolves({ addPackage });
+      const addPackageToDatabase = sinon.stub().resolves(undefined);
+      const environmentStore = {
+         publisherConfigIsFrozen: false,
+         getEnvironment,
+         addPackageToDatabase,
+      } as unknown as EnvironmentStore;
+
+      const controller = new PackageController(environmentStore);
+
+      await controller.addPackage("env", {
+         name: "pkg",
+         description: "test",
+         explores: ["index.malloy"],
+      });
+
+      expect(addPackageToDatabase.calledOnceWith("env", "pkg")).toBe(true);
+   });
+});
+
+describe("PackageController.updatePackage explores validation", () => {
+   afterEach(() => {
+      sinon.restore();
+   });
+
+   it("location update: validates the EFFECTIVE explores (body override) before the swap commits", async () => {
+      // body.location triggers a reinstall (atomic swap). The effective explores
+      // — the body override here — must be validated inside installPackage so a
+      // bad update rolls back to the previous tree instead of swapping in the
+      // rejected one and 400-ing after the fact.
+      const invalidMsg =
+         "Invalid explores entry 'nope.malloy' in publisher.json: file not found";
+      // The mock package validates whatever override it's handed.
+      const mockPackage = {
+         formatInvalidExplores: (override?: string[]) =>
+            override?.includes("nope.malloy") ? invalidMsg : "",
+      };
+      const installPackage = sinon
+         .stub()
+         .callsFake(
+            async (
+               _name: string,
+               _downloader: unknown,
+               validate?: (pkg: unknown) => string | undefined,
+            ) => {
+               const msg = validate?.(mockPackage);
+               if (msg) throw new BadRequestError(msg);
+               return mockPackage;
+            },
+         );
+      const updatePackage = sinon.stub().resolves(mockPackage);
+      const environment = { installPackage, updatePackage };
+      const getEnvironment = sinon.stub().resolves(environment);
+      const addPackageToDatabase = sinon.stub().resolves(undefined);
+      const environmentStore = {
+         publisherConfigIsFrozen: false,
+         getEnvironment,
+         addPackageToDatabase,
+      } as unknown as EnvironmentStore;
+
+      const controller = new PackageController(environmentStore);
+
+      await expect(
+         controller.updatePackage("env", "pkg", {
+            name: "pkg",
+            location: "gs://bucket/pkg.zip",
+            explores: ["nope.malloy"],
+         }),
+      ).rejects.toBeInstanceOf(BadRequestError);
+
+      // The rejected swap never reached the metadata-apply / persist steps.
+      expect(updatePackage.called).toBe(false);
+      expect(addPackageToDatabase.called).toBe(false);
+   });
+});

package/src/controller/package.controller.ts

@@ -1,21 +1,15 @@
 import { components } from "../api";
+import { normalizeModelPath } from "../constants";
 import { BadRequestError, FrozenConfigError } from "../errors";
-import { logger } from "../logger";
 import { EnvironmentStore } from "../service/environment_store";
-import { ManifestService } from "../service/manifest_service";
 
 type ApiPackage = components["schemas"]["Package"];
 
 export class PackageController {
    private environmentStore: EnvironmentStore;
-   private manifestService: ManifestService;
 
-   constructor(
-      environmentStore: EnvironmentStore,
-      manifestService: ManifestService,
-   ) {
+   constructor(environmentStore: EnvironmentStore) {
       this.environmentStore = environmentStore;
-      this.manifestService = manifestService;
    }
 
    public async listPackages(environmentName: string): Promise<ApiPackage[]> {
@@ -70,11 +64,7 @@ export class PackageController {
       return _package.getPackageMetadata();
    }
 
-   async addPackage(
-      environmentName: string,
-      body: ApiPackage,
-      options?: { autoLoadManifest?: boolean },
-   ) {
+   async addPackage(environmentName: string, body: ApiPackage) {
       if (this.environmentStore.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
       }
@@ -86,73 +76,61 @@ export class PackageController {
          environmentName,
          false,
       );
+      // Strict at publish: the author is in the loop here, so reject a bad
+      // explores with an actionable 400 instead of silently serving a hidden
+      // surface. (At startup/reload we fail safe and only warn — see
+      // Package.loadViaWorker.) The rollback differs by path so a rejected
+      // publish never destroys user content:
+      //   - location: the tree was just downloaded into a fresh canonical, so
+      //     validation runs inside installPackage's swap window and a failure
+      //     wipes that download (the existing rollback) — nothing pre-existing
+      //     to lose.
+      //   - no-location: addPackage registered a *pre-existing* user directory,
+      //     so we validate after the fact and `unloadPackage` (evict from
+      //     memory, keep the files) rather than delete it.
       let result;
       if (body.location) {
          const bodyLocation = body.location;
-         result = await environment.installPackage(packageName, (stagingPath) =>
-            this.downloadInto(
-               environmentName,
-               packageName,
-               bodyLocation,
-               stagingPath,
-            ),
+         result = await environment.installPackage(
+            packageName,
+            (stagingPath) =>
+               this.downloadInto(
+                  environmentName,
+                  packageName,
+                  bodyLocation,
+                  stagingPath,
+               ),
+            (pkg) => pkg.formatInvalidExplores(),
          );
       } else {
          result = await environment.addPackage(packageName);
       }
+
+      // `addPackage`/`installPackage` are typed `Package | undefined`; a missing
+      // result here is a should-never-happen internal fault. Fail loudly rather
+      // than letting optional chaining silently skip the validation below.
+      if (!result) {
+         throw new Error(`Failed to create package ${packageName}`);
+      }
+
+      if (!body.location) {
+         const invalidMsg = result.formatInvalidExplores();
+         if (invalidMsg) {
+            await environment.unloadPackage(packageName).catch(() => {
+               /* best-effort; the package is not persisted below */
+            });
+            throw new BadRequestError(invalidMsg);
+         }
+      }
+
       await this.environmentStore.addPackageToDatabase(
          environmentName,
          packageName,
       );
 
-      if (options?.autoLoadManifest === true) {
-         await this.tryLoadExistingManifest(environmentName, packageName);
-      }
-
       return result;
    }
 
-   /**
-    * If there are already manifest entries for this package (e.g. from a
-    * previous materialization run), reload all models with the manifest so
-    * persist references resolve to the materialized tables immediately.
-    */
-   private async tryLoadExistingManifest(
-      environmentName: string,
-      packageName: string,
-   ): Promise<void> {
-      try {
-         const repository =
-            this.environmentStore.storageManager.getRepository();
-         const dbEnvironment =
-            await repository.getEnvironmentByName(environmentName);
-         if (!dbEnvironment) return;
-
-         const manifest = await this.manifestService.getManifest(
-            dbEnvironment.id,
-            packageName,
-         );
-         if (Object.keys(manifest.entries).length === 0) return;
-
-         await this.manifestService.reloadManifest(
-            dbEnvironment.id,
-            packageName,
-            environmentName,
-         );
-         logger.info("Auto-loaded existing manifest for added package", {
-            environmentName,
-            packageName,
-            entryCount: Object.keys(manifest.entries).length,
-         });
-      } catch (error) {
-         logger.warn("Failed to auto-load manifest for package", {
-            environmentName,
-            packageName,
-            error,
-         });
-      }
-   }
-
    public async deletePackage(environmentName: string, packageName: string) {
       if (this.environmentStore.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
@@ -184,15 +162,24 @@ export class PackageController {
       );
       if (body.location) {
          // Re-install: stream the new content into a staging dir (no lock)
-         // and atomically swap it in (under the lock).
+         // and atomically swap it in (under the lock). Validate the effective
+         // explores (the body override, else the new tree's own manifest)
+         // INSIDE the swap window, so a rejected update rolls back to the
+         // previous tree instead of swapping the bad one in and 400-ing after.
          const bodyLocation = body.location;
-         await environment.installPackage(packageName, (stagingPath) =>
-            this.downloadInto(
-               environmentName,
-               packageName,
-               bodyLocation,
-               stagingPath,
-            ),
+         await environment.installPackage(
+            packageName,
+            (stagingPath) =>
+               this.downloadInto(
+                  environmentName,
+                  packageName,
+                  bodyLocation,
+                  stagingPath,
+               ),
+            (pkg) =>
+               pkg.formatInvalidExplores(
+                  body.explores?.map(normalizeModelPath),
+               ),
          );
       }
       // Apply metadata changes (publisher.json) under the same per-package

package/src/dto/package.dto.ts

@@ -1,4 +1,10 @@
-import { IsString, IsNotEmpty } from "class-validator";
+import {
+   IsString,
+   IsNotEmpty,
+   IsOptional,
+   IsArray,
+   IsIn,
+} from "class-validator";
 import { ApiPackage } from "../service/package";
 
 export class PackageDto implements ApiPackage {
@@ -9,4 +15,13 @@ export class PackageDto implements ApiPackage {
    @IsString()
    @IsNotEmpty()
    description: string;
+
+   @IsOptional()
+   @IsArray()
+   @IsString({ each: true })
+   explores?: string[];
+
+   @IsOptional()
+   @IsIn(["declared", "all"])
+   queryableSources?: "declared" | "all";
 }

package/src/errors.spec.ts

@@ -6,6 +6,7 @@ import {
    ConnectionError,
    internalErrorToHttpError,
    ModelCompilationError,
+   NotQueryableError,
    PayloadTooLargeError,
    QueryTimeoutError,
    ServiceUnavailableError,
@@ -42,6 +43,17 @@ describe("internalErrorToHttpError", () => {
       });
    });
 
+   it("maps NotQueryableError to 404 (explore boundary)", () => {
+      const { status, json } = internalErrorToHttpError(
+         new NotQueryableError('No queryable source "hidden".'),
+      );
+      expect(status).toBe(404);
+      expect(json).toEqual({
+         code: 404,
+         message: 'No queryable source "hidden".',
+      });
+   });
+
    it("maps ModelCompilationError to 424", () => {
       const { status, json } = internalErrorToHttpError(
          new ModelCompilationError({ message: "compile failed" }),

package/src/errors.ts

@@ -14,6 +14,8 @@ export function internalErrorToHttpError(error: Error) {
       return httpError(404, error.message);
    } else if (error instanceof ModelNotFoundError) {
       return httpError(404, error.message);
+   } else if (error instanceof NotQueryableError) {
+      return httpError(404, error.message);
    } else if (error instanceof MalloyError) {
       return httpError(400, error.message);
    } else if (error instanceof ConnectionNotFoundError) {
@@ -128,6 +130,22 @@ export class AccessDeniedError extends Error {
    }
 }
 
+/**
+ * A query targeted a source/model that is not part of the package's queryable
+ * surface under `queryableSources: "declared"` (a non-`explores` model file, or
+ * a source not in a model's `export {}` closure). Mapped to HTTP **404**, not
+ * 403, and with a deliberately generic message: unlike `#(authorize)` (which is
+ * identity-scoped and answers "who"), the explore boundary is identity-free and
+ * answers "what is queryable" — so a hidden target should be indistinguishable
+ * from a non-existent one (no enumeration / existence oracle).
+ */
+export class NotQueryableError extends Error {
+   constructor(message: string) {
+      super(message);
+      this.name = "NotQueryableError";
+   }
+}
+
 export class MaterializationNotFoundError extends Error {
    constructor(message: string) {
       super(message);