@malloy-publisher/server 0.0.85 → 0.0.87

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.85",
+   "version": "0.0.87",
    "main": "dist/server.js",
    "bin": {
       "malloy-publisher": "dist/server.js"

package/publisher.config.json

@@ -1,4 +1,5 @@
 {
+   "frozenConfig": false,
    "projects": {
       "malloy-samples": "./malloy-samples"
    }

package/src/controller/package.controller.ts

@@ -1,4 +1,5 @@
 import { components } from "../api";
+import { BadRequestError, FrozenConfigError } from "../errors";
 import { ProjectStore } from "../service/project_store";
 
 type ApiPackage = components["schemas"]["Package"];
@@ -24,4 +25,35 @@ export class PackageController {
       const p = await project.getPackage(packageName, reload);
       return p.getPackageMetadata();
    }
+
+   async addPackage(projectName: string, body: ApiPackage) {
+      if (this.projectStore.publisherConfigIsFrozen) {
+         throw new FrozenConfigError();
+      }
+      if (!body.name) {
+         throw new BadRequestError("Package name is required");
+      }
+      const project = await this.projectStore.getProject(projectName, false);
+      return project.addPackage(body.name);
+   }
+
+   public async deletePackage(projectName: string, packageName: string) {
+      if (this.projectStore.publisherConfigIsFrozen) {
+         throw new FrozenConfigError();
+      }
+      const project = await this.projectStore.getProject(projectName, false);
+      return project.deletePackage(packageName);
+   }
+
+   public async updatePackage(
+      projectName: string,
+      packageName: string,
+      body: ApiPackage,
+   ) {
+      if (this.projectStore.publisherConfigIsFrozen) {
+         throw new FrozenConfigError();
+      }
+      const project = await this.projectStore.getProject(projectName, false);
+      return project.updatePackage(packageName, body);
+   }
 }

package/src/errors.ts

@@ -3,10 +3,16 @@ import { MalloyError } from "@malloydata/malloy";
 export function internalErrorToHttpError(error: Error) {
    if (error instanceof BadRequestError) {
       return httpError(400, error.message);
+   } else if (error instanceof FrozenConfigError) {
+      return httpError(403, error.message);
+   } else if (error instanceof ProjectNotFoundError) {
+      return httpError(404, error.message);
    } else if (error instanceof PackageNotFoundError) {
       return httpError(404, error.message);
    } else if (error instanceof ModelNotFoundError) {
       return httpError(404, error.message);
+   } else if (error instanceof MalloyError) {
+      return httpError(400, error.message);
    } else if (error instanceof ConnectionNotFoundError) {
       return httpError(404, error.message);
    } else if (error instanceof ModelCompilationError) {
@@ -75,3 +81,11 @@ export class ModelCompilationError extends Error {
       super(error.message);
    }
 }
+
+export class FrozenConfigError extends Error {
+   constructor(
+      message = "Publisher config can't be updated when publisher.config.json has `frozenConfig: true`",
+   ) {
+      super(message);
+   }
+}

package/src/logger.ts

@@ -2,12 +2,19 @@ import { AxiosError } from "axios";
 import { RequestHandler } from "express";
 import winston from "winston";
 
+const isTelemetryEnabled = Boolean(process.env.OTEL_EXPORTER_OTLP_ENDPOINT);
+
 export const logger = winston.createLogger({
-   format: winston.format.combine(
-      winston.format.uncolorize(),
-      winston.format.timestamp(),
-      winston.format.json(),
-   ),
+   format: isTelemetryEnabled
+      ? winston.format.combine(
+           winston.format.uncolorize(),
+           winston.format.timestamp(),
+           winston.format.json(),
+        )
+      : winston.format.combine(
+           winston.format.colorize(),
+           winston.format.simple(),
+        ),
    transports: [new winston.transports.Console()],
 });
 

package/src/mcp/tools/discovery_tools.ts

@@ -1,7 +1,7 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
 import { ProjectStore } from "../../service/project_store";
 import { buildMalloyUri } from "../handler_utils";
-import { z } from "zod";
 
 const listPackagesShape = {
    // projectName is required; other fields mirror SDK expectations
@@ -78,7 +78,7 @@ export function registerTools(
             allProjects.map(async (project) => {
                const name = project.name;
                const projectInstance = await project.project;
-               const metadata = await projectInstance.getProjectMetadata();
+               const metadata = await projectInstance.reloadProjectMetadata();
                const readme = metadata.readme;
                return {
                   name,

package/src/server.ts

@@ -92,7 +92,6 @@ const scheduleController = new ScheduleController(projectStore);
 
 const mcpApp = express();
 
-initProjects();
 mcpApp.use(MCP_ENDPOINT, express.json());
 mcpApp.use(MCP_ENDPOINT, cors());
 
@@ -192,7 +191,7 @@ if (!isDevelopment) {
          target: "http://localhost:5173",
          changeOrigin: true,
          ws: true,
-         pathFilter: (path) => !path.startsWith("/api"),
+         pathFilter: (path) => !path.startsWith("/api/"),
       }),
    );
 }
@@ -217,13 +216,45 @@ app.get(`${API_PREFIX}/projects`, async (_req, res) => {
    }
 });
 
+app.post(`${API_PREFIX}/projects`, async (req, res) => {
+   try {
+      res.status(200).json(await projectStore.addProject(req.body));
+   } catch (error) {
+      logger.error(error);
+      const { json, status } = internalErrorToHttpError(error as Error);
+      res.status(status).json(json);
+   }
+});
+
 app.get(`${API_PREFIX}/projects/:projectName`, async (req, res) => {
    try {
       const project = await projectStore.getProject(
          req.params.projectName,
          req.query.reload === "true",
       );
-      res.status(200).json(await project.getProjectMetadata());
+      res.status(200).json(project.metadata);
+   } catch (error) {
+      logger.error(error);
+      const { json, status } = internalErrorToHttpError(error as Error);
+      res.status(status).json(json);
+   }
+});
+
+app.patch(`${API_PREFIX}/projects/:projectName`, async (req, res) => {
+   try {
+      res.status(200).json(await projectStore.updateProject(req.body));
+   } catch (error) {
+      logger.error(error);
+      const { json, status } = internalErrorToHttpError(error as Error);
+      res.status(status).json(json);
+   }
+});
+
+app.delete(`${API_PREFIX}/projects/:projectName`, async (req, res) => {
+   try {
+      res.status(200).json(
+         await projectStore.deleteProject(req.params.projectName),
+      );
    } catch (error) {
       logger.error(error);
       const { json, status } = internalErrorToHttpError(error as Error);
@@ -412,6 +443,18 @@ app.get(`${API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
    }
 });
 
+app.post(`${API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
+   try {
+      res.status(200).json(
+         await packageController.addPackage(req.params.projectName, req.body),
+      );
+   } catch (error) {
+      logger.error(error);
+      const { json, status } = internalErrorToHttpError(error as Error);
+      res.status(status).json(json);
+   }
+});
+
 app.get(
    `${API_PREFIX}/projects/:projectName/packages/:packageName`,
    async (req, res) => {
@@ -436,6 +479,43 @@ app.get(
    },
 );
 
+app.patch(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName`,
+   async (req, res) => {
+      try {
+         res.status(200).json(
+            await packageController.updatePackage(
+               req.params.projectName,
+               req.params.packageName,
+               req.body,
+            ),
+         );
+      } catch (error) {
+         logger.error(error);
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.delete(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName`,
+   async (req, res) => {
+      try {
+         res.status(200).json(
+            await packageController.deletePackage(
+               req.params.projectName,
+               req.params.packageName,
+            ),
+         );
+      } catch (error) {
+         logger.error(error);
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
 app.get(
    `${API_PREFIX}/projects/:projectName/packages/:packageName/models`,
    async (req, res) => {
@@ -611,11 +691,18 @@ if (!isDevelopment) {
    app.get("*", (_req, res) => res.sendFile(path.resolve(ROOT, "index.html")));
 }
 
-app.use((err: Error, _req: express.Request, res: express.Response) => {
-   logger.error("Unhandled error:", err);
-   const { json, status } = internalErrorToHttpError(err);
-   res.status(status).json(json);
-});
+app.use(
+   (
+      err: Error,
+      _req: express.Request,
+      res: express.Response,
+      _next: express.NextFunction,
+   ) => {
+      logger.error("Unhandled error:", err);
+      const { json, status } = internalErrorToHttpError(err);
+      res.status(status).json(json);
+   },
+);
 
 const mainServer = http.createServer(app);
 mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, () => {
@@ -635,14 +722,3 @@ const mcpHttpServer = mcpApp.listen(MCP_PORT, PUBLISHER_HOST, () => {
 });
 
 export { app, mainServer as httpServer, mcpApp, mcpHttpServer };
-
-// Warm up the packages
-function initProjects() {
-   projectStore.listProjects().then((projects) => {
-      projects.forEach((project) => {
-         projectStore.getProject(project.name!, false).then((project) => {
-            project.listPackages();
-         });
-      });
-   });
-}

package/src/service/db_utils.ts

@@ -93,12 +93,9 @@ export async function getSchemasForConnection(
       }
       try {
          const bigquery = getBigqueryConnection(connection);
-         // Set the projectId if it's provided in the bigqueryConnection
-         const [datasets] = await bigquery.getDatasets({
-            ...(connection.bigqueryConnection.defaultProjectId
-               ? { projectId: connection.bigqueryConnection.defaultProjectId }
-               : {}),
-         });
+         const projectId = connection.bigqueryConnection.defaultProjectId;
+         const options = projectId ? { projectId } : {};
+         const [datasets] = await bigquery.getDatasets(options);
          return datasets
             .filter((dataset) => dataset.id)
             .map((dataset) => {

package/src/service/package.ts

@@ -373,4 +373,16 @@ export class Package {
       const rowCount = result.data.value[0].row_count?.valueOf() as number;
       return { name: databasePath, rowCount, columns: schema };
    }
+
+   public setName(name: string) {
+      this.packageName = name;
+   }
+
+   public setProjectName(projectName: string) {
+      this.projectName = projectName;
+   }
+
+   public setPackageMetadata(packageMetadata: ApiPackage) {
+      this.packageMetadata = packageMetadata;
+   }
 }

package/src/service/project.ts

@@ -4,7 +4,11 @@ import * as fs from "fs/promises";
 import * as path from "path";
 import { components } from "../api";
 import { API_PREFIX, README_NAME } from "../constants";
-import { ConnectionNotFoundError, ProjectNotFoundError } from "../errors";
+import {
+   ConnectionNotFoundError,
+   PackageNotFoundError,
+   ProjectNotFoundError,
+} from "../errors";
 import { createConnections, InternalConnection } from "./connection";
 import { ApiConnection } from "./model";
 import { Package } from "./package";
@@ -19,6 +23,7 @@ export class Project {
    private internalConnections: InternalConnection[];
    private projectPath: string;
    private projectName: string;
+   public metadata: ApiProject;
 
    constructor(
       projectName: string,
@@ -33,6 +38,35 @@ export class Project {
       // InternalConnections have full connection details for doing schema inspection
       this.internalConnections = internalConnections;
       this.apiConnections = apiConnections;
+      this.metadata = {
+         resource: `${API_PREFIX}/projects/${this.projectName}`,
+         name: this.projectName,
+      };
+      void this.reloadProjectMetadata();
+   }
+
+   public async update(payload: ApiProject) {
+      if (payload.name) {
+         this.projectName = payload.name;
+         this.packages.forEach((_package) => {
+            _package.setProjectName(this.projectName);
+         });
+         this.metadata.name = this.projectName;
+      }
+      if (payload.resource) {
+         this.projectPath = payload.resource.replace(
+            `${API_PREFIX}/projects/`,
+            "",
+         );
+         if (!(await fs.exists(this.projectPath))) {
+            throw new ProjectNotFoundError(
+               `Project path "${this.projectPath}" not found`,
+            );
+         }
+         this.metadata.resource = payload.resource;
+      }
+      this.metadata.readme = payload.readme;
+      return this;
    }
 
    static async create(
@@ -65,7 +99,7 @@ export class Project {
       );
    }
 
-   public async getProjectMetadata(): Promise<ApiProject> {
+   public async reloadProjectMetadata(): Promise<ApiProject> {
       let readme = "";
       try {
          readme = (
@@ -74,11 +108,12 @@ export class Project {
       } catch {
          // Readme not found, so we'll just return an empty string
       }
-      return {
+      this.metadata = {
          resource: `${API_PREFIX}/projects/${this.projectName}`,
          name: this.projectName,
          readme: readme,
       };
+      return this.metadata;
    }
 
    public listApiConnections(): ApiConnection[] {
@@ -187,4 +222,51 @@ export class Project {
          }
       });
    }
+
+   public async addPackage(packageName: string) {
+      const packagePath = path.join(this.projectPath, packageName);
+      if (
+         !(await fs.exists(packagePath)) ||
+         !(await fs.stat(packagePath)).isDirectory()
+      ) {
+         throw new PackageNotFoundError(`Package ${packageName} not found`);
+      }
+      this.packages.set(
+         packageName,
+         await Package.create(
+            this.projectName,
+            packageName,
+            packagePath,
+            this.malloyConnections,
+         ),
+      );
+      return this.packages.get(packageName);
+   }
+
+   public async updatePackage(
+      packageName: string,
+      body: { resource?: string; name?: string; description?: string },
+   ) {
+      const _package = this.packages.get(packageName);
+      if (!_package) {
+         throw new PackageNotFoundError(`Package ${packageName} not found`);
+      }
+      if (body.name) {
+         _package.setName(body.name);
+      }
+      _package.setPackageMetadata({
+         name: body.name,
+         description: body.description,
+         resource: body.resource,
+      });
+      return _package.getPackageMetadata();
+   }
+
+   public async deletePackage(packageName: string) {
+      const _package = this.packages.get(packageName);
+      if (!_package) {
+         throw new PackageNotFoundError(`Package ${packageName} not found`);
+      }
+      this.packages.delete(packageName);
+   }
 }

package/src/service/project_store.spec.ts

@@ -0,0 +1,173 @@
+import { describe, expect, it, mock, spyOn } from "bun:test";
+import * as fs from "fs/promises";
+import path from "path";
+import { FrozenConfigError, ProjectNotFoundError } from "../errors";
+import { isPublisherConfigFrozen } from "../utils";
+import { ProjectStore } from "./project_store";
+
+describe("ProjectStore", () => {
+   const serverRoot = path.resolve(
+      process.cwd(),
+      process.env.SERVER_ROOT || ".",
+   );
+
+   async function waitForInitialization(projectStore: ProjectStore) {
+      const maxRetries = 100;
+      let retries = 0;
+      do {
+         await new Promise((resolve) => setTimeout(resolve, 10));
+         retries++;
+      } while (projectStore.listProjects().length === 0 && retries < maxRetries);
+      if (projectStore.listProjects().length === 0) {
+         throw new Error("Timed out initializing ProjectStore");
+      }
+   }
+
+   it("should load all projects from publisher.config.json within a second on initialization", async () => {
+      mock(isPublisherConfigFrozen).mockReturnValue(true);
+      const projectStore = new ProjectStore(serverRoot);
+      await waitForInitialization(projectStore);
+      expect(projectStore.listProjects()).toEqual([
+         {
+            name: "malloy-samples",
+            readme: expect.stringContaining("# Malloy Analysis Examples"),
+            resource: "/api/v0/projects/malloy-samples",
+         },
+      ]);
+   });
+
+   it("should list projects from memory by default", async () => {
+      mock(isPublisherConfigFrozen).mockReturnValue(true);
+      const projectStore = new ProjectStore(serverRoot);
+      // Mock fs.readFile & fs.readdir to track calls
+      const readFileSpy = spyOn(fs, "readFile");
+      const readdirSpy = spyOn(fs, "readdir");
+      // Call listProjects, which should use memory and not call fs.readFile
+      projectStore.listProjects();
+
+      expect(readFileSpy).not.toHaveBeenCalled();
+      expect(readdirSpy).not.toHaveBeenCalled();
+   });
+
+   it("should list projects from disk if reload is true", async () => {
+      // Mock fs.readFile to track calls
+      const fs = await import("fs/promises");
+      const readFileSpy = spyOn(fs, "readFile");
+      mock(isPublisherConfigFrozen).mockReturnValue(true);
+      const projectStore = new ProjectStore(serverRoot);
+
+      // Call getProject with reload=true
+      await projectStore.getProject("malloy-samples", true);
+
+      expect(readFileSpy).toHaveBeenCalled();
+   });
+
+   it("should allow modifying the in-memory hashmap if config is not frozen", async () => {
+      mock.module("../utils", () => ({
+         isPublisherConfigFrozen: () => false,
+      }));
+      const projectStore = new ProjectStore(serverRoot);
+      await waitForInitialization(projectStore);
+      await projectStore.updateProject({
+         name: "malloy-samples",
+         readme: "Updated README",
+      });
+      expect(projectStore.listProjects()).toEqual([
+         {
+            name: "malloy-samples",
+            readme: "Updated README",
+            resource: "/api/v0/projects/malloy-samples",
+         },
+      ]);
+      await projectStore.deleteProject("malloy-samples");
+      expect(projectStore.listProjects()).toEqual([]);
+      await projectStore.addProject({
+         name: "malloy-samples",
+      });
+
+      expect(
+         await projectStore.getProject("malloy-samples", false),
+      ).toHaveProperty("metadata", {
+         name: "malloy-samples",
+         resource: "/api/v0/projects/malloy-samples",
+      });
+
+      // After a while, it'll resolve the async promise where the readme gets loaded in memory
+      await waitForInitialization(projectStore);
+      expect(projectStore.listProjects()).toEqual([
+         {
+            name: "malloy-samples",
+            readme: expect.stringContaining("# Malloy Analysis Examples"),
+            resource: "/api/v0/projects/malloy-samples",
+         },
+      ]);
+   });
+
+   it("should not allow modifying the in-memory hashmap if config is frozen", async () => {
+      mock.module("../utils", () => ({
+         isPublisherConfigFrozen: () => true,
+      }));
+      const projectStore = new ProjectStore(serverRoot);
+      // Initialization should succeed
+      await waitForInitialization(projectStore);
+      expect(projectStore.listProjects()).toEqual([
+         {
+            name: "malloy-samples",
+            readme: expect.stringContaining("# Malloy Analysis Examples"),
+            resource: "/api/v0/projects/malloy-samples",
+         },
+      ]);
+      // Adding a project should fail
+      expect(
+         projectStore.addProject({
+            name: "malloy-samples",
+         }),
+      ).rejects.toThrow(FrozenConfigError);
+
+      // Updating a project should fail
+      expect(
+         projectStore.updateProject({
+            name: "malloy-samples",
+            readme: "Updated README",
+         }),
+      ).rejects.toThrow(FrozenConfigError);
+
+      // Deleting a project should fail
+      expect(projectStore.deleteProject("malloy-samples")).rejects.toThrow(
+         FrozenConfigError,
+      );
+
+      // Failed methods should not modify the in-memory hashmap
+      expect(projectStore.listProjects()).toEqual([
+         {
+            name: "malloy-samples",
+            readme: expect.stringContaining("# Malloy Analysis Examples"),
+            resource: "/api/v0/projects/malloy-samples",
+         },
+      ]);
+   });
+
+   it("should always try to reload a project if it's not in the hashmap", async () => {
+      mock.module("../utils", () => ({
+         isPublisherConfigFrozen: () => false,
+      }));
+      const projectStore = new ProjectStore(serverRoot);
+      await waitForInitialization(projectStore);
+      await projectStore.deleteProject("malloy-samples");
+      expect(projectStore.listProjects()).toEqual([]);
+      const readFileSpy = spyOn(fs, "readFile");
+      await projectStore.getProject("malloy-samples", true);
+      expect(readFileSpy).toHaveBeenCalled();
+   });
+
+   it("should throw a NotFound error when reloading a project that is not in disk", async () => {
+      mock.module("../utils", () => ({
+         isPublisherConfigFrozen: () => false,
+      }));
+      const projectStore = new ProjectStore(serverRoot);
+      await waitForInitialization(projectStore);
+      expect(projectStore.getProject("this-one-does-not-exist", true)).rejects.toThrow(
+         ProjectNotFoundError,
+      );
+   });
+});