@malloy-publisher/server 0.0.86 → 0.0.88

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.86",
+   "version": "0.0.88",
    "main": "dist/server.js",
    "bin": {
       "malloy-publisher": "dist/server.js"
@@ -11,16 +11,17 @@
    },
    "scripts": {
       "test": "bun test ./**/*.spec.ts",
-      "build": "bun generate-api-types && NODE_ENV=production && bun build:app && bun build src/server.ts --outdir dist --target node --format cjs --external='@malloydata/db-duckdb' --external='@malloydata/malloy' --external='@malloydata/malloy-sql' --external='@malloydata/render' --external='@malloydata/db-bigquery' --external='@malloydata/db-mysql' --external='@malloydata/db-postgres' --external='@malloydata/db-snowflake' --external='@malloydata/db-trino' && NODE_ENV=production bun build src/instrumentation.ts --outdir dist --target node --format cjs && bun run build:app && echo '#!/usr/bin/env node' | cat - dist/server.js > temp && mv temp dist/server.js && chmod +x dist/server.js",
+      "build": "bun generate-api-types && bun build:app && NODE_ENV=production bun run build.ts",
       "start": "NODE_ENV=production node ./dist/server.js",
       "start:dev": "NODE_ENV=development bun --watch src/server.ts",
       "start:instrumented": "node --require ./dist/instrumentation.js ./dist/server.js",
       "lint": "bunx eslint \"./src/**/*.ts\"",
       "format": "bunx prettier --write '**/*.{ts,tsx}'",
-      "build:app": "cd ../app && bun run build:server && rm -rf ../server/dist/app && mkdir -p ../server/dist && cp -r dist ../server/dist/app",
+      "build:app": "cd ../app && NODE_ENV=production bunx vite build",
       "generate-api-types": "bunx openapi-typescript ../../api-doc.yaml --output src/api.ts"
    },
    "dependencies": {
+      "@google-cloud/storage": "^7.16.0",
       "@malloydata/db-bigquery": "^0.0.295",
       "@malloydata/db-duckdb": "^0.0.295",
       "@malloydata/db-mysql": "^0.0.295",
@@ -37,7 +38,9 @@
       "@opentelemetry/sdk-node": "^0.200.0",
       "@opentelemetry/sdk-trace-node": "^2.0.0",
       "async-mutex": "^0.5.0",
+      "aws-sdk": "^2.1692.0",
       "body-parser": "^1.20.2",
+      "chokidar": "^4.0.3",
       "class-transformer": "^0.5.1",
       "class-validator": "^0.14.1",
       "cors": "^2.8.5",
@@ -48,6 +51,7 @@
       "morgan": "^1.10.0",
       "node-cron": "^3.0.3",
       "recursive-readdir": "^2.2.3",
+      "simple-git": "^3.28.0",
       "uuid": "^11.0.3"
    },
    "devDependencies": {
@@ -65,7 +69,6 @@
       "@types/supertest": "^6.0.2",
       "@typescript-eslint/eslint-plugin": "^8.6.0",
       "@typescript-eslint/parser": "^8.6.0",
-      "aws-sdk": "^2.1692.0",
       "eslint-config-prettier": "^9.1.0",
       "eslint-plugin-prettier": "^5.2.1",
       "mock-aws-s3": "^4.0.2",

package/publisher.config.json

@@ -1,6 +1,6 @@
 {
    "frozenConfig": false,
    "projects": {
-      "malloy-samples": "./malloy-samples"
+      "malloy-samples": "https://github.com/ms2data/malloy-samples"
    }
 }

package/src/config.ts

@@ -0,0 +1,20 @@
+import fs from "fs";
+import path from "path";
+import { PUBLISHER_CONFIG_NAME } from "./constants";
+
+type FilesystemPath = `./${string}` | `../${string}` | `/${string}`;
+type GcsPath = `gs://${string}`;
+export type PublisherConfig = {
+   frozenConfig: boolean;
+   projects: Record<string, FilesystemPath | GcsPath>;
+};
+
+export const getPublisherConfig = (serverRoot: string): PublisherConfig => {
+   const publisherConfigPath = path.join(serverRoot, PUBLISHER_CONFIG_NAME);
+   return JSON.parse(fs.readFileSync(publisherConfigPath, "utf8"));
+};
+
+export const isPublisherConfigFrozen = (serverRoot: string) => {
+   const publisherConfig = getPublisherConfig(serverRoot);
+   return Boolean(publisherConfig.frozenConfig);
+};

package/src/constants.ts

@@ -1,2 +1,8 @@
 export const API_PREFIX = "/api/v0";
 export const README_NAME = "README.md";
+export const PUBLISHER_CONFIG_NAME = "publisher.config.json";
+export const PACKAGE_MANIFEST_NAME = "publisher.json";
+export const CONNECTIONS_MANIFEST_NAME = "publisher.connections.json";
+export const MODEL_FILE_SUFFIX = ".malloy";
+export const NOTEBOOK_FILE_SUFFIX = ".malloynb";
+export const ROW_LIMIT = 1000;

package/src/controller/package.controller.ts

@@ -22,8 +22,12 @@ export class PackageController {
       reload: boolean,
    ): Promise<ApiPackage> {
       const project = await this.projectStore.getProject(projectName, false);
-      const p = await project.getPackage(packageName, reload);
-      return p.getPackageMetadata();
+      const _package = await project.getPackage(packageName, reload);
+      const packageLocation = _package.getPackageMetadata().location;
+      if (reload && packageLocation) {
+         await this.downloadPackage(projectName, packageName, packageLocation);
+      }
+      return _package.getPackageMetadata();
    }
 
    async addPackage(projectName: string, body: ApiPackage) {
@@ -34,6 +38,9 @@ export class PackageController {
          throw new BadRequestError("Package name is required");
       }
       const project = await this.projectStore.getProject(projectName, false);
+      if (body.location) {
+         await this.downloadPackage(projectName, body.name, body.location);
+      }
       return project.addPackage(body.name);
    }
 
@@ -54,6 +61,38 @@ export class PackageController {
          throw new FrozenConfigError();
       }
       const project = await this.projectStore.getProject(projectName, false);
+      if (body.location) {
+         await this.downloadPackage(projectName, packageName, body.location);
+      }
       return project.updatePackage(packageName, body);
    }
+
+   private async downloadPackage(
+      projectName: string,
+      packageName: string,
+      packageLocation: string,
+   ) {
+      const absoluteTargetPath = `/etc/publisher/${projectName}/${packageName}`;
+      if (
+         packageLocation.startsWith("https://") ||
+         packageLocation.startsWith("git@")
+      ) {
+         await this.projectStore.downloadGitHubDirectory(
+            packageLocation,
+            absoluteTargetPath,
+         );
+      } else if (packageLocation.startsWith("gs://")) {
+         await this.projectStore.downloadGcsDirectory(
+            packageLocation,
+            projectName,
+            absoluteTargetPath,
+         );
+      } else if (packageLocation.startsWith("s3://")) {
+         await this.projectStore.downloadS3Directory(
+            packageLocation,
+            projectName,
+            absoluteTargetPath,
+         );
+      }
+   }
 }

package/src/controller/watch-mode.controller.ts

@@ -0,0 +1,83 @@
+import chokidar, { FSWatcher } from "chokidar";
+import { RequestHandler } from "express";
+import path from "path";
+import { components } from "../api";
+import { logger } from "../logger";
+import { ProjectStore } from "../service/project_store";
+
+type StartWatchReq = components["schemas"]["StartWatchRequest"];
+type WatchStatusRes = components["schemas"]["WatchStatus"];
+type Handler<Req = {}, Res = void> = RequestHandler<{}, Res, Req>;
+
+export class WatchModeController {
+   watchingPath: string | null;
+   watchingProjectName: string | null;
+   watcher: FSWatcher;
+
+   constructor(private projectStore: ProjectStore) {
+      this.watchingPath = null;
+      this.watchingProjectName = null;
+   }
+
+   public getWatchStatus: Handler<{}, WatchStatusRes> = async (_req, res) => {
+      return res.json({
+         enabled: !!this.watchingPath,
+         watchingPath: this.watchingPath,
+         projectName: this.watchingProjectName ?? undefined,
+      });
+   };
+
+   public startWatching: Handler<StartWatchReq> = async (req, res) => {
+      const projectManifest = await ProjectStore.reloadProjectManifest(
+         this.projectStore.serverRootPath,
+      );
+      this.watchingProjectName = req.body.projectName;
+      this.watchingPath = path.join(
+         this.projectStore.serverRootPath,
+         projectManifest.projects[req.body.projectName],
+      );
+      this.watcher = chokidar.watch(this.watchingPath, {
+         ignored: (path, stats) =>
+            !!stats?.isFile() &&
+            !path.endsWith(".malloy") &&
+            !path.endsWith(".md"),
+         ignoreInitial: true,
+      });
+      const reloadProject = async () => {
+         // Overwrite the project with it's existing metadata to trigger a re-read
+         const project = await this.projectStore.getProject(
+            req.body.projectName,
+            true,
+         );
+         await this.projectStore.addProject(project.metadata);
+         logger.info(`Reloaded ${req.body.projectName}`);
+      };
+
+      this.watcher.on("add", async (path) => {
+         logger.info(
+            `Detected new file ${path}, reloading ${req.body.projectName}`,
+         );
+         await reloadProject();
+      });
+      this.watcher.on("unlink", async (path) => {
+         logger.info(
+            `Detected deletion of ${path}, reloading ${req.body.projectName}`,
+         );
+         await reloadProject();
+      });
+      this.watcher.on("change", async (path) => {
+         logger.info(
+            `Detected change on ${path}, reloading ${req.body.projectName}`,
+         );
+         await reloadProject();
+      });
+      res.json();
+   };
+
+   public stopWatchMode: Handler = async (_req, res) => {
+      this.watcher.close();
+      this.watchingPath = null;
+      this.watchingProjectName = null;
+      res.json();
+   };
+}

package/src/errors.ts

@@ -1,4 +1,5 @@
 import { MalloyError } from "@malloydata/malloy";
+import { PUBLISHER_CONFIG_NAME } from "./constants";
 
 export function internalErrorToHttpError(error: Error) {
    if (error instanceof BadRequestError) {
@@ -84,7 +85,7 @@ export class ModelCompilationError extends Error {
 
 export class FrozenConfigError extends Error {
    constructor(
-      message = "Publisher config can't be updated when publisher.config.json has `frozenConfig: true`",
+      message = `Publisher config can't be updated when ${PUBLISHER_CONFIG_NAME} has { "frozenConfig": true }`,
    ) {
       super(message);
    }

package/src/server.ts

@@ -12,6 +12,7 @@ import { ModelController } from "./controller/model.controller";
 import { PackageController } from "./controller/package.controller";
 import { QueryController } from "./controller/query.controller";
 import { ScheduleController } from "./controller/schedule.controller";
+import { WatchModeController } from "./controller/watch-mode.controller";
 import { internalErrorToHttpError, NotImplementedError } from "./errors";
 import { logger, loggerMiddleware } from "./logger";
 import { initializeMcpServer } from "./mcp/server";
@@ -62,7 +63,7 @@ function parseArgs() {
 parseArgs();
 
 const PUBLISHER_PORT = Number(process.env.PUBLISHER_PORT || 4000);
-const PUBLISHER_HOST = process.env.PUBLISHER_HOST || "localhost";
+const PUBLISHER_HOST = process.env.PUBLISHER_HOST || "0.0.0.0";
 const MCP_PORT = Number(process.env.MCP_PORT || 4040);
 const MCP_ENDPOINT = "/mcp";
 // Find the app directory - handle NPX vs local execution
@@ -83,6 +84,7 @@ app.use(loggerMiddleware);
 app.use(cors());
 
 const projectStore = new ProjectStore(SERVER_ROOT);
+const watchModeController = new WatchModeController(projectStore);
 const connectionController = new ConnectionController(projectStore);
 const modelController = new ModelController(projectStore);
 const packageController = new PackageController(projectStore);
@@ -206,6 +208,10 @@ const setVersionIdError = (res: express.Response) => {
 app.use(cors());
 app.use(bodyParser.json());
 
+app.get(`${API_PREFIX}/watch-mode/status`, watchModeController.getWatchStatus);
+app.post(`${API_PREFIX}/watch-mode/start`, watchModeController.startWatching);
+app.post(`${API_PREFIX}/watch-mode/stop`, watchModeController.stopWatchMode);
+
 app.get(`${API_PREFIX}/projects`, async (_req, res) => {
    try {
       res.status(200).json(await projectStore.listProjects());
@@ -705,20 +711,22 @@ app.use(
 );
 
 const mainServer = http.createServer(app);
-mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, () => {
-   const address = mainServer.address() as AddressInfo;
-   logger.info(
-      `Publisher server listening at http://${address.address}:${address.port}`,
-   );
-   if (isDevelopment) {
+projectStore.finishedInitialization.then(() => {
+   mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, () => {
+      const address = mainServer.address() as AddressInfo;
       logger.info(
-         "Running in development mode - proxying to React dev server at http://localhost:5173",
+         `Publisher server listening at http://${address.address}:${address.port}`,
       );
-   }
-});
-
-const mcpHttpServer = mcpApp.listen(MCP_PORT, PUBLISHER_HOST, () => {
-   logger.info(`MCP server listening at http://${PUBLISHER_HOST}:${MCP_PORT}`);
+      if (isDevelopment) {
+         logger.info(
+            "Running in development mode - proxying to React dev server at http://localhost:5173",
+         );
+      }
+   });
+   mcpApp.listen(MCP_PORT, PUBLISHER_HOST, () => {
+      logger.info(
+         `MCP server listening at http://${PUBLISHER_HOST}:${MCP_PORT}`,
+      );
+   });
 });
 
-export { app, mainServer as httpServer, mcpApp, mcpHttpServer };

package/src/service/connection.ts

@@ -1,15 +1,15 @@
-import { PostgresConnection } from "@malloydata/db-postgres";
 import { BigQueryConnection } from "@malloydata/db-bigquery";
+import { MySQLConnection } from "@malloydata/db-mysql";
+import { PostgresConnection } from "@malloydata/db-postgres";
 import { SnowflakeConnection } from "@malloydata/db-snowflake";
 import { TrinoConnection } from "@malloydata/db-trino";
-import { MySQLConnection } from "@malloydata/db-mysql";
-import { v4 as uuidv4 } from "uuid";
 import { Connection } from "@malloydata/malloy";
-import { components } from "../api";
-import path from "path";
-import fs from "fs/promises";
-import { CONNECTIONS_MANIFEST_NAME } from "../utils";
 import { BaseConnection } from "@malloydata/malloy/connection";
+import fs from "fs/promises";
+import path from "path";
+import { v4 as uuidv4 } from "uuid";
+import { components } from "../api";
+import { CONNECTIONS_MANIFEST_NAME } from "../constants";
 
 type ApiConnection = components["schemas"]["Connection"];
 type ApiConnectionAttributes = components["schemas"]["ConnectionAttributes"];

package/src/service/model.ts

@@ -25,6 +25,11 @@ import { metrics } from "@opentelemetry/api";
 import * as fs from "fs/promises";
 import * as path from "path";
 import { components } from "../api";
+import {
+   MODEL_FILE_SUFFIX,
+   NOTEBOOK_FILE_SUFFIX,
+   ROW_LIMIT,
+} from "../constants";
 import { HackyDataStylesAccumulator } from "../data_styles";
 import {
    BadRequestError,
@@ -32,12 +37,7 @@ import {
    ModelNotFoundError,
 } from "../errors";
 import { logger } from "../logger";
-import {
-   MODEL_FILE_SUFFIX,
-   NOTEBOOK_FILE_SUFFIX,
-   ROW_LIMIT,
-   URL_READER,
-} from "../utils";
+import { URL_READER } from "../utils";
 
 type ApiCompiledModel = components["schemas"]["CompiledModel"];
 type ApiNotebookCell = components["schemas"]["NotebookCell"];

package/src/service/package.ts

@@ -11,14 +11,14 @@ import {
 import { metrics } from "@opentelemetry/api";
 import recursive from "recursive-readdir";
 import { components } from "../api";
-import { API_PREFIX } from "../constants";
-import { PackageNotFoundError } from "../errors";
-import { logger } from "../logger";
 import {
+   API_PREFIX,
    MODEL_FILE_SUFFIX,
    NOTEBOOK_FILE_SUFFIX,
    PACKAGE_MANIFEST_NAME,
-} from "../utils";
+} from "../constants";
+import { PackageNotFoundError } from "../errors";
+import { logger } from "../logger";
 import { createConnections } from "./connection";
 import { Model } from "./model";
 import { Scheduler } from "./scheduler";

package/src/service/project.ts

@@ -41,6 +41,7 @@ export class Project {
       this.metadata = {
          resource: `${API_PREFIX}/projects/${this.projectName}`,
          name: this.projectName,
+         location: this.projectPath,
       };
       void this.reloadProjectMetadata();
    }
@@ -243,10 +244,7 @@ export class Project {
       return this.packages.get(packageName);
    }
 
-   public async updatePackage(
-      packageName: string,
-      body: { resource?: string; name?: string; description?: string },
-   ) {
+   public async updatePackage(packageName: string, body: ApiPackage) {
       const _package = this.packages.get(packageName);
       if (!_package) {
          throw new PackageNotFoundError(`Package ${packageName} not found`);
@@ -258,6 +256,7 @@ export class Project {
          name: body.name,
          description: body.description,
          resource: body.resource,
+         location: body.location,
       });
       return _package.getPackageMetadata();
    }
@@ -267,6 +266,9 @@ export class Project {
       if (!_package) {
          throw new PackageNotFoundError(`Package ${packageName} not found`);
       }
+      await fs.rm(path.join(this.projectPath, packageName), {
+         recursive: true,
+      });
       this.packages.delete(packageName);
    }
 }

package/src/service/project_store.spec.ts

@@ -1,8 +1,8 @@
 import { describe, expect, it, mock, spyOn } from "bun:test";
 import * as fs from "fs/promises";
 import path from "path";
+import { isPublisherConfigFrozen } from "../config";
 import { FrozenConfigError, ProjectNotFoundError } from "../errors";
-import { isPublisherConfigFrozen } from "../utils";
 import { ProjectStore } from "./project_store";
 
 describe("ProjectStore", () => {
@@ -17,8 +17,11 @@ describe("ProjectStore", () => {
       do {
          await new Promise((resolve) => setTimeout(resolve, 10));
          retries++;
-      } while (projectStore.listProjects().length === 0 && retries < maxRetries);
-      if (projectStore.listProjects().length === 0) {
+      } while (
+         (await projectStore.listProjects()).length === 0 &&
+         retries < maxRetries
+      );
+      if ((await projectStore.listProjects()).length === 0) {
          throw new Error("Timed out initializing ProjectStore");
       }
    }
@@ -27,7 +30,7 @@ describe("ProjectStore", () => {
       mock(isPublisherConfigFrozen).mockReturnValue(true);
       const projectStore = new ProjectStore(serverRoot);
       await waitForInitialization(projectStore);
-      expect(projectStore.listProjects()).toEqual([
+      expect(await projectStore.listProjects()).toEqual([
          {
             name: "malloy-samples",
             readme: expect.stringContaining("# Malloy Analysis Examples"),
@@ -39,11 +42,13 @@ describe("ProjectStore", () => {
    it("should list projects from memory by default", async () => {
       mock(isPublisherConfigFrozen).mockReturnValue(true);
       const projectStore = new ProjectStore(serverRoot);
+      await waitForInitialization(projectStore);
+
       // Mock fs.readFile & fs.readdir to track calls
       const readFileSpy = spyOn(fs, "readFile");
       const readdirSpy = spyOn(fs, "readdir");
       // Call listProjects, which should use memory and not call fs.readFile
-      projectStore.listProjects();
+      await projectStore.listProjects();
 
       expect(readFileSpy).not.toHaveBeenCalled();
       expect(readdirSpy).not.toHaveBeenCalled();
@@ -72,7 +77,7 @@ describe("ProjectStore", () => {
          name: "malloy-samples",
          readme: "Updated README",
       });
-      expect(projectStore.listProjects()).toEqual([
+      expect(await projectStore.listProjects()).toEqual([
          {
             name: "malloy-samples",
             readme: "Updated README",
@@ -80,7 +85,7 @@ describe("ProjectStore", () => {
          },
       ]);
       await projectStore.deleteProject("malloy-samples");
-      expect(projectStore.listProjects()).toEqual([]);
+      expect(await projectStore.listProjects()).toEqual([]);
       await projectStore.addProject({
          name: "malloy-samples",
       });
@@ -94,7 +99,7 @@ describe("ProjectStore", () => {
 
       // After a while, it'll resolve the async promise where the readme gets loaded in memory
       await waitForInitialization(projectStore);
-      expect(projectStore.listProjects()).toEqual([
+      expect(await projectStore.listProjects()).toEqual([
          {
             name: "malloy-samples",
             readme: expect.stringContaining("# Malloy Analysis Examples"),
@@ -110,7 +115,7 @@ describe("ProjectStore", () => {
       const projectStore = new ProjectStore(serverRoot);
       // Initialization should succeed
       await waitForInitialization(projectStore);
-      expect(projectStore.listProjects()).toEqual([
+      expect(await projectStore.listProjects()).toEqual([
          {
             name: "malloy-samples",
             readme: expect.stringContaining("# Malloy Analysis Examples"),
@@ -138,7 +143,7 @@ describe("ProjectStore", () => {
       );
 
       // Failed methods should not modify the in-memory hashmap
-      expect(projectStore.listProjects()).toEqual([
+      expect(await projectStore.listProjects()).toEqual([
          {
             name: "malloy-samples",
             readme: expect.stringContaining("# Malloy Analysis Examples"),
@@ -154,7 +159,7 @@ describe("ProjectStore", () => {
       const projectStore = new ProjectStore(serverRoot);
       await waitForInitialization(projectStore);
       await projectStore.deleteProject("malloy-samples");
-      expect(projectStore.listProjects()).toEqual([]);
+      expect(await projectStore.listProjects()).toEqual([]);
       const readFileSpy = spyOn(fs, "readFile");
       await projectStore.getProject("malloy-samples", true);
       expect(readFileSpy).toHaveBeenCalled();
@@ -166,8 +171,8 @@ describe("ProjectStore", () => {
       }));
       const projectStore = new ProjectStore(serverRoot);
       await waitForInitialization(projectStore);
-      expect(projectStore.getProject("this-one-does-not-exist", true)).rejects.toThrow(
-         ProjectNotFoundError,
-      );
+      expect(
+         projectStore.getProject("this-one-does-not-exist", true),
+      ).rejects.toThrow(ProjectNotFoundError);
    });
 });