@malloy-publisher/server 0.0.87 → 0.0.89

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.87",
+   "version": "0.0.89",
    "main": "dist/server.js",
    "bin": {
       "malloy-publisher": "dist/server.js"
@@ -10,17 +10,19 @@
       "access": "public"
    },
    "scripts": {
-      "test": "bun test ./**/*.spec.ts",
-      "build": "bun generate-api-types && NODE_ENV=production && bun build:app && bun build src/server.ts --outdir dist --target node --format cjs --external='@malloydata/db-duckdb' --external='@malloydata/malloy' --external='@malloydata/malloy-sql' --external='@malloydata/render' --external='@malloydata/db-bigquery' --external='@malloydata/db-mysql' --external='@malloydata/db-postgres' --external='@malloydata/db-snowflake' --external='@malloydata/db-trino' && NODE_ENV=production bun build src/instrumentation.ts --outdir dist --target node --format cjs && bun run build:app && echo '#!/usr/bin/env node' | cat - dist/server.js > temp && mv temp dist/server.js && chmod +x dist/server.js",
+      "test": "bun test --timeout 100000",
+      "build": "bun generate-api-types && bun build:app && NODE_ENV=production bun run build.ts",
+      "build:server-only": "bun generate-api-types && NODE_ENV=production bun run build.ts",
       "start": "NODE_ENV=production node ./dist/server.js",
       "start:dev": "NODE_ENV=development bun --watch src/server.ts",
       "start:instrumented": "node --require ./dist/instrumentation.js ./dist/server.js",
       "lint": "bunx eslint \"./src/**/*.ts\"",
       "format": "bunx prettier --write '**/*.{ts,tsx}'",
-      "build:app": "cd ../app && bun run build:server && rm -rf ../server/dist/app && mkdir -p ../server/dist && cp -r dist ../server/dist/app",
+      "build:app": "cd ../app && NODE_ENV=production bunx vite build",
       "generate-api-types": "bunx openapi-typescript ../../api-doc.yaml --output src/api.ts"
    },
    "dependencies": {
+      "@google-cloud/storage": "^7.16.0",
       "@malloydata/db-bigquery": "^0.0.295",
       "@malloydata/db-duckdb": "^0.0.295",
       "@malloydata/db-mysql": "^0.0.295",
@@ -36,8 +38,11 @@
       "@opentelemetry/sdk-metrics": "^2.0.0",
       "@opentelemetry/sdk-node": "^0.200.0",
       "@opentelemetry/sdk-trace-node": "^2.0.0",
+      "adm-zip": "^0.5.16",
       "async-mutex": "^0.5.0",
+      "aws-sdk": "^2.1692.0",
       "body-parser": "^1.20.2",
+      "chokidar": "^4.0.3",
       "class-transformer": "^0.5.1",
       "class-validator": "^0.14.1",
       "cors": "^2.8.5",
@@ -48,6 +53,7 @@
       "morgan": "^1.10.0",
       "node-cron": "^3.0.3",
       "recursive-readdir": "^2.2.3",
+      "simple-git": "^3.28.0",
       "uuid": "^11.0.3"
    },
    "devDependencies": {
@@ -56,6 +62,7 @@
       "@eslint/eslintrc": "^3.3.1",
       "@eslint/js": "^9.23.0",
       "@faker-js/faker": "^9.4.0",
+      "@types/adm-zip": "^0.5.7",
       "@types/cors": "^2.8.12",
       "@types/express": "^4.17.14",
       "@types/morgan": "^1.9.9",
@@ -65,7 +72,6 @@
       "@types/supertest": "^6.0.2",
       "@typescript-eslint/eslint-plugin": "^8.6.0",
       "@typescript-eslint/parser": "^8.6.0",
-      "aws-sdk": "^2.1692.0",
       "eslint-config-prettier": "^9.1.0",
       "eslint-plugin-prettier": "^5.2.1",
       "mock-aws-s3": "^4.0.2",

package/publisher.config.json

@@ -1,6 +1,6 @@
 {
    "frozenConfig": false,
    "projects": {
-      "malloy-samples": "./malloy-samples"
+      "malloy-samples": "https://github.com/ms2data/malloy-samples"
    }
 }

package/src/config.ts

@@ -0,0 +1,20 @@
+import fs from "fs";
+import path from "path";
+import { PUBLISHER_CONFIG_NAME } from "./constants";
+
+type FilesystemPath = `./${string}` | `../${string}` | `/${string}`;
+type GcsPath = `gs://${string}`;
+export type PublisherConfig = {
+   frozenConfig: boolean;
+   projects: Record<string, FilesystemPath | GcsPath>;
+};
+
+export const getPublisherConfig = (serverRoot: string): PublisherConfig => {
+   const publisherConfigPath = path.join(serverRoot, PUBLISHER_CONFIG_NAME);
+   return JSON.parse(fs.readFileSync(publisherConfigPath, "utf8"));
+};
+
+export const isPublisherConfigFrozen = (serverRoot: string) => {
+   const publisherConfig = getPublisherConfig(serverRoot);
+   return Boolean(publisherConfig.frozenConfig);
+};

package/src/constants.ts

@@ -1,2 +1,16 @@
+import fs from "fs";
 export const API_PREFIX = "/api/v0";
 export const README_NAME = "README.md";
+export const PUBLISHER_CONFIG_NAME = "publisher.config.json";
+export const PACKAGE_MANIFEST_NAME = "publisher.json";
+export const CONNECTIONS_MANIFEST_NAME = "publisher.connections.json";
+export const MODEL_FILE_SUFFIX = ".malloy";
+export const NOTEBOOK_FILE_SUFFIX = ".malloynb";
+export const ROW_LIMIT = 1000;
+
+export let publisherPath = "/etc/publisher";
+try {
+   fs.accessSync(publisherPath, fs.constants.W_OK);
+} catch (e) {
+   publisherPath = "/tmp/publisher";
+}

package/src/controller/connection.controller.ts

@@ -1,6 +1,9 @@
-import { RunSQLOptions, TestableConnection } from "@malloydata/malloy";
+import {
+   RunSQLOptions,
+   TableSourceDef,
+   TestableConnection,
+} from "@malloydata/malloy";
 import { Connection, PersistSQLResults } from "@malloydata/malloy/connection";
-import { TableSourceDef } from "../../../../../malloy/packages/malloy/dist";
 import { components } from "../api";
 import { ConnectionError } from "../errors";
 import { logger } from "../logger";
@@ -11,6 +14,7 @@ import {
 import { ProjectStore } from "../service/project_store";
 
 type ApiConnection = components["schemas"]["Connection"];
+type ApiConnectionStatus = components["schemas"]["ConnectionStatus"];
 type ApiSqlSource = components["schemas"]["SqlSource"];
 type ApiTableSource = components["schemas"]["TableSource"];
 type ApiQueryData = components["schemas"]["QueryData"];
@@ -57,15 +61,28 @@ export class ConnectionController {
       return getTablesForSchema(connection, schemaName);
    }
 
-   public async testConnection(projectName: string, connectionName: string) {
+   public async testConnection(
+      projectName: string,
+      connectionName: string,
+   ): Promise<ApiConnectionStatus> {
       const project = await this.projectStore.getProject(projectName, false);
       const connection = project.getMalloyConnection(
          connectionName,
       ) as Connection;
       try {
          await (connection as TestableConnection).test();
+         return {
+            status: "ok",
+            errorMessage: "",
+         };
       } catch (error) {
-         throw new ConnectionError((error as Error).message);
+         if (error instanceof ConnectionError) {
+            return {
+               status: "failed",
+               errorMessage: error.message,
+            };
+         }
+         throw error;
       }
    }
 

package/src/controller/package.controller.ts

@@ -1,4 +1,5 @@
 import { components } from "../api";
+import { publisherPath } from "../constants";
 import { BadRequestError, FrozenConfigError } from "../errors";
 import { ProjectStore } from "../service/project_store";
 
@@ -22,8 +23,12 @@ export class PackageController {
       reload: boolean,
    ): Promise<ApiPackage> {
       const project = await this.projectStore.getProject(projectName, false);
-      const p = await project.getPackage(packageName, reload);
-      return p.getPackageMetadata();
+      const _package = await project.getPackage(packageName, reload);
+      const packageLocation = _package.getPackageMetadata().location;
+      if (reload && packageLocation) {
+         await this.downloadPackage(projectName, packageName, packageLocation);
+      }
+      return _package.getPackageMetadata();
    }
 
    async addPackage(projectName: string, body: ApiPackage) {
@@ -34,6 +39,9 @@ export class PackageController {
          throw new BadRequestError("Package name is required");
       }
       const project = await this.projectStore.getProject(projectName, false);
+      if (body.location) {
+         await this.downloadPackage(projectName, body.name, body.location);
+      }
       return project.addPackage(body.name);
    }
 
@@ -54,6 +62,48 @@ export class PackageController {
          throw new FrozenConfigError();
       }
       const project = await this.projectStore.getProject(projectName, false);
+      if (body.location) {
+         await this.downloadPackage(projectName, packageName, body.location);
+      }
       return project.updatePackage(packageName, body);
    }
+
+   private async downloadPackage(
+      projectName: string,
+      packageName: string,
+      packageLocation: string,
+   ) {
+      const absoluteTargetPath = `${publisherPath}/${projectName}/${packageName}`;
+      if (
+         packageLocation.startsWith("https://") ||
+         packageLocation.startsWith("git@")
+      ) {
+         await this.projectStore.downloadGitHubDirectory(
+            packageLocation,
+            absoluteTargetPath,
+         );
+      } else if (packageLocation.startsWith("gs://")) {
+         await this.projectStore.downloadGcsDirectory(
+            packageLocation,
+            projectName,
+            absoluteTargetPath,
+         );
+      } else if (packageLocation.startsWith("s3://")) {
+         await this.projectStore.downloadS3Directory(
+            packageLocation,
+            projectName,
+            absoluteTargetPath,
+         );
+      } else if (packageLocation.startsWith("/")) {
+         if (packageLocation.endsWith(".zip")) {
+            packageLocation =
+               await this.projectStore.unzipProject(packageLocation);
+         }
+         await this.projectStore.mountLocalDirectory(
+            packageLocation,
+            absoluteTargetPath,
+            projectName,
+         );
+      }
+   }
 }

package/src/controller/schedule.controller.ts

@@ -14,8 +14,8 @@ export class ScheduleController {
       projectName: string,
       packageName: string,
    ): Promise<ApiSchedule[]> {
-      const project = await this.projectStore.getProject(projectName);
-      const p = await project.getPackage(packageName);
-      return p.listSchedules();
+      const project = await this.projectStore?.getProject?.(projectName);
+      const p = await project?.getPackage?.(packageName);
+      return p?.listSchedules?.();
    }
 }

package/src/controller/watch-mode.controller.ts

@@ -0,0 +1,83 @@
+import chokidar, { FSWatcher } from "chokidar";
+import { RequestHandler } from "express";
+import path from "path";
+import { components } from "../api";
+import { logger } from "../logger";
+import { ProjectStore } from "../service/project_store";
+
+type StartWatchReq = components["schemas"]["StartWatchRequest"];
+type WatchStatusRes = components["schemas"]["WatchStatus"];
+type Handler<Req = {}, Res = void> = RequestHandler<{}, Res, Req>;
+
+export class WatchModeController {
+   watchingPath: string | null;
+   watchingProjectName: string | null;
+   watcher: FSWatcher;
+
+   constructor(private projectStore: ProjectStore) {
+      this.watchingPath = null;
+      this.watchingProjectName = null;
+   }
+
+   public getWatchStatus: Handler<{}, WatchStatusRes> = async (_req, res) => {
+      return res.json({
+         enabled: !!this.watchingPath,
+         watchingPath: this.watchingPath,
+         projectName: this.watchingProjectName ?? undefined,
+      });
+   };
+
+   public startWatching: Handler<StartWatchReq> = async (req, res) => {
+      const projectManifest = await ProjectStore.reloadProjectManifest(
+         this.projectStore.serverRootPath,
+      );
+      this.watchingProjectName = req.body.projectName;
+      this.watchingPath = path.join(
+         this.projectStore.serverRootPath,
+         projectManifest.projects[req.body.projectName],
+      );
+      this.watcher = chokidar.watch(this.watchingPath, {
+         ignored: (path, stats) =>
+            !!stats?.isFile() &&
+            !path.endsWith(".malloy") &&
+            !path.endsWith(".md"),
+         ignoreInitial: true,
+      });
+      const reloadProject = async () => {
+         // Overwrite the project with it's existing metadata to trigger a re-read
+         const project = await this.projectStore.getProject(
+            req.body.projectName,
+            true,
+         );
+         await this.projectStore.addProject(project.metadata);
+         logger.info(`Reloaded ${req.body.projectName}`);
+      };
+
+      this.watcher.on("add", async (path) => {
+         logger.info(
+            `Detected new file ${path}, reloading ${req.body.projectName}`,
+         );
+         await reloadProject();
+      });
+      this.watcher.on("unlink", async (path) => {
+         logger.info(
+            `Detected deletion of ${path}, reloading ${req.body.projectName}`,
+         );
+         await reloadProject();
+      });
+      this.watcher.on("change", async (path) => {
+         logger.info(
+            `Detected change on ${path}, reloading ${req.body.projectName}`,
+         );
+         await reloadProject();
+      });
+      res.json();
+   };
+
+   public stopWatchMode: Handler = async (_req, res) => {
+      this.watcher.close();
+      this.watchingPath = null;
+      this.watchingProjectName = null;
+      res.json();
+   };
+}

package/src/errors.ts

@@ -1,4 +1,5 @@
 import { MalloyError } from "@malloydata/malloy";
+import { PUBLISHER_CONFIG_NAME } from "./constants";
 
 export function internalErrorToHttpError(error: Error) {
    if (error instanceof BadRequestError) {
@@ -84,7 +85,7 @@ export class ModelCompilationError extends Error {
 
 export class FrozenConfigError extends Error {
    constructor(
-      message = "Publisher config can't be updated when publisher.config.json has `frozenConfig: true`",
+      message = `Publisher config can't be updated when ${PUBLISHER_CONFIG_NAME} has { "frozenConfig": true }`,
    ) {
       super(message);
    }

package/src/logger.ts

@@ -20,11 +20,20 @@ export const logger = winston.createLogger({
 
 export const loggerMiddleware: RequestHandler = (req, res, next) => {
    const startTime = performance.now();
+   const resJson = res.json;
+   res.json = (body: unknown) => {
+      res.locals.body = body;
+      return resJson.call(res, body);
+   };
    res.on("finish", () => {
       const endTime = performance.now();
       logger.info(`${req.method} ${req.url}`, {
          statusCode: res.statusCode,
          duration: endTime - startTime,
+         payload: req.body,
+         response: res.locals.body,
+         params: req.params,
+         query: req.query,
       });
    });
    next();

package/src/server.ts

@@ -12,6 +12,7 @@ import { ModelController } from "./controller/model.controller";
 import { PackageController } from "./controller/package.controller";
 import { QueryController } from "./controller/query.controller";
 import { ScheduleController } from "./controller/schedule.controller";
+import { WatchModeController } from "./controller/watch-mode.controller";
 import { internalErrorToHttpError, NotImplementedError } from "./errors";
 import { logger, loggerMiddleware } from "./logger";
 import { initializeMcpServer } from "./mcp/server";
@@ -62,7 +63,7 @@ function parseArgs() {
 parseArgs();
 
 const PUBLISHER_PORT = Number(process.env.PUBLISHER_PORT || 4000);
-const PUBLISHER_HOST = process.env.PUBLISHER_HOST || "localhost";
+const PUBLISHER_HOST = process.env.PUBLISHER_HOST || "0.0.0.0";
 const MCP_PORT = Number(process.env.MCP_PORT || 4040);
 const MCP_ENDPOINT = "/mcp";
 // Find the app directory - handle NPX vs local execution
@@ -83,6 +84,7 @@ app.use(loggerMiddleware);
 app.use(cors());
 
 const projectStore = new ProjectStore(SERVER_ROOT);
+const watchModeController = new WatchModeController(projectStore);
 const connectionController = new ConnectionController(projectStore);
 const modelController = new ModelController(projectStore);
 const packageController = new PackageController(projectStore);
@@ -90,7 +92,7 @@ const databaseController = new DatabaseController(projectStore);
 const queryController = new QueryController(projectStore);
 const scheduleController = new ScheduleController(projectStore);
 
-const mcpApp = express();
+export const mcpApp = express();
 
 mcpApp.use(MCP_ENDPOINT, express.json());
 mcpApp.use(MCP_ENDPOINT, cors());
@@ -206,6 +208,17 @@ const setVersionIdError = (res: express.Response) => {
 app.use(cors());
 app.use(bodyParser.json());
 
+app.get(`${API_PREFIX}/status`, async (_req, res) => {
+   res.status(200).json({
+      timestamp: Date.now(),
+      projects: await projectStore.listProjects(),
+   });
+});
+
+app.get(`${API_PREFIX}/watch-mode/status`, watchModeController.getWatchStatus);
+app.post(`${API_PREFIX}/watch-mode/start`, watchModeController.startWatching);
+app.post(`${API_PREFIX}/watch-mode/stop`, watchModeController.stopWatchMode);
+
 app.get(`${API_PREFIX}/projects`, async (_req, res) => {
    try {
       res.status(200).json(await projectStore.listProjects());
@@ -218,7 +231,9 @@ app.get(`${API_PREFIX}/projects`, async (_req, res) => {
 
 app.post(`${API_PREFIX}/projects`, async (req, res) => {
    try {
-      res.status(200).json(await projectStore.addProject(req.body));
+      logger.info("Adding project", { body: req.body });
+      const project = await projectStore.addProject(req.body);
+      res.status(200).json(await project.serialize());
    } catch (error) {
       logger.error(error);
       const { json, status } = internalErrorToHttpError(error as Error);
@@ -232,7 +247,7 @@ app.get(`${API_PREFIX}/projects/:projectName`, async (req, res) => {
          req.params.projectName,
          req.query.reload === "true",
       );
-      res.status(200).json(project.metadata);
+      res.status(200).json(await project.serialize());
    } catch (error) {
       logger.error(error);
       const { json, status } = internalErrorToHttpError(error as Error);
@@ -242,7 +257,8 @@ app.get(`${API_PREFIX}/projects/:projectName`, async (req, res) => {
 
 app.patch(`${API_PREFIX}/projects/:projectName`, async (req, res) => {
    try {
-      res.status(200).json(await projectStore.updateProject(req.body));
+      const project = await projectStore.updateProject(req.body);
+      res.status(200).json(await project.serialize());
    } catch (error) {
       logger.error(error);
       const { json, status } = internalErrorToHttpError(error as Error);
@@ -252,9 +268,8 @@ app.patch(`${API_PREFIX}/projects/:projectName`, async (req, res) => {
 
 app.delete(`${API_PREFIX}/projects/:projectName`, async (req, res) => {
    try {
-      res.status(200).json(
-         await projectStore.deleteProject(req.params.projectName),
-      );
+      const project = await projectStore.deleteProject(req.params.projectName);
+      res.status(200).json(await project.serialize());
    } catch (error) {
       logger.error(error);
       const { json, status } = internalErrorToHttpError(error as Error);
@@ -296,12 +311,11 @@ app.get(
    `${API_PREFIX}/projects/:projectName/connections/:connectionName/test`,
    async (req, res) => {
       try {
-         res.status(200).json(
-            await connectionController.testConnection(
-               req.params.projectName,
-               req.params.connectionName,
-            ),
+         const connectionStatus = await connectionController.testConnection(
+            req.params.projectName,
+            req.params.connectionName,
          );
+         res.status(200).json(connectionStatus);
       } catch (error) {
          logger.error(error);
          const { json, status } = internalErrorToHttpError(error as Error);
@@ -445,9 +459,11 @@ app.get(`${API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
 
 app.post(`${API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
    try {
-      res.status(200).json(
-         await packageController.addPackage(req.params.projectName, req.body),
+      const _package = await packageController.addPackage(
+         req.params.projectName,
+         req.body,
       );
+      res.status(200).json(_package?.getPackageMetadata());
    } catch (error) {
       logger.error(error);
       const { json, status } = internalErrorToHttpError(error as Error);
@@ -705,6 +721,7 @@ app.use(
 );
 
 const mainServer = http.createServer(app);
+
 mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, () => {
    const address = mainServer.address() as AddressInfo;
    logger.info(
@@ -716,9 +733,6 @@ mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, () => {
       );
    }
 });
-
-const mcpHttpServer = mcpApp.listen(MCP_PORT, PUBLISHER_HOST, () => {
+mcpApp.listen(MCP_PORT, PUBLISHER_HOST, () => {
    logger.info(`MCP server listening at http://${PUBLISHER_HOST}:${MCP_PORT}`);
 });
-
-export { app, mainServer as httpServer, mcpApp, mcpHttpServer };