@malloy-publisher/server 0.0.88 → 0.0.89

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.88",
+   "version": "0.0.89",
    "main": "dist/server.js",
    "bin": {
       "malloy-publisher": "dist/server.js"
@@ -10,8 +10,9 @@
       "access": "public"
    },
    "scripts": {
-      "test": "bun test ./**/*.spec.ts",
+      "test": "bun test --timeout 100000",
       "build": "bun generate-api-types && bun build:app && NODE_ENV=production bun run build.ts",
+      "build:server-only": "bun generate-api-types && NODE_ENV=production bun run build.ts",
       "start": "NODE_ENV=production node ./dist/server.js",
       "start:dev": "NODE_ENV=development bun --watch src/server.ts",
       "start:instrumented": "node --require ./dist/instrumentation.js ./dist/server.js",
@@ -37,6 +38,7 @@
       "@opentelemetry/sdk-metrics": "^2.0.0",
       "@opentelemetry/sdk-node": "^0.200.0",
       "@opentelemetry/sdk-trace-node": "^2.0.0",
+      "adm-zip": "^0.5.16",
       "async-mutex": "^0.5.0",
       "aws-sdk": "^2.1692.0",
       "body-parser": "^1.20.2",
@@ -60,6 +62,7 @@
       "@eslint/eslintrc": "^3.3.1",
       "@eslint/js": "^9.23.0",
       "@faker-js/faker": "^9.4.0",
+      "@types/adm-zip": "^0.5.7",
       "@types/cors": "^2.8.12",
       "@types/express": "^4.17.14",
       "@types/morgan": "^1.9.9",

package/src/constants.ts

@@ -1,3 +1,4 @@
+import fs from "fs";
 export const API_PREFIX = "/api/v0";
 export const README_NAME = "README.md";
 export const PUBLISHER_CONFIG_NAME = "publisher.config.json";
@@ -5,4 +6,11 @@ export const PACKAGE_MANIFEST_NAME = "publisher.json";
 export const CONNECTIONS_MANIFEST_NAME = "publisher.connections.json";
 export const MODEL_FILE_SUFFIX = ".malloy";
 export const NOTEBOOK_FILE_SUFFIX = ".malloynb";
-export const ROW_LIMIT = 1000;
+export const ROW_LIMIT = 1000;
+
+export let publisherPath = "/etc/publisher";
+try {
+   fs.accessSync(publisherPath, fs.constants.W_OK);
+} catch (e) {
+   publisherPath = "/tmp/publisher";
+}

package/src/controller/connection.controller.ts

@@ -1,6 +1,9 @@
-import { RunSQLOptions, TestableConnection } from "@malloydata/malloy";
+import {
+   RunSQLOptions,
+   TableSourceDef,
+   TestableConnection,
+} from "@malloydata/malloy";
 import { Connection, PersistSQLResults } from "@malloydata/malloy/connection";
-import { TableSourceDef } from "../../../../../malloy/packages/malloy/dist";
 import { components } from "../api";
 import { ConnectionError } from "../errors";
 import { logger } from "../logger";
@@ -11,6 +14,7 @@ import {
 import { ProjectStore } from "../service/project_store";
 
 type ApiConnection = components["schemas"]["Connection"];
+type ApiConnectionStatus = components["schemas"]["ConnectionStatus"];
 type ApiSqlSource = components["schemas"]["SqlSource"];
 type ApiTableSource = components["schemas"]["TableSource"];
 type ApiQueryData = components["schemas"]["QueryData"];
@@ -57,15 +61,28 @@ export class ConnectionController {
       return getTablesForSchema(connection, schemaName);
    }
 
-   public async testConnection(projectName: string, connectionName: string) {
+   public async testConnection(
+      projectName: string,
+      connectionName: string,
+   ): Promise<ApiConnectionStatus> {
       const project = await this.projectStore.getProject(projectName, false);
       const connection = project.getMalloyConnection(
          connectionName,
       ) as Connection;
       try {
          await (connection as TestableConnection).test();
+         return {
+            status: "ok",
+            errorMessage: "",
+         };
       } catch (error) {
-         throw new ConnectionError((error as Error).message);
+         if (error instanceof ConnectionError) {
+            return {
+               status: "failed",
+               errorMessage: error.message,
+            };
+         }
+         throw error;
       }
    }
 

package/src/controller/package.controller.ts

@@ -1,4 +1,5 @@
 import { components } from "../api";
+import { publisherPath } from "../constants";
 import { BadRequestError, FrozenConfigError } from "../errors";
 import { ProjectStore } from "../service/project_store";
 
@@ -72,7 +73,7 @@ export class PackageController {
       packageName: string,
       packageLocation: string,
    ) {
-      const absoluteTargetPath = `/etc/publisher/${projectName}/${packageName}`;
+      const absoluteTargetPath = `${publisherPath}/${projectName}/${packageName}`;
       if (
          packageLocation.startsWith("https://") ||
          packageLocation.startsWith("git@")
@@ -93,6 +94,16 @@ export class PackageController {
             projectName,
             absoluteTargetPath,
          );
+      } else if (packageLocation.startsWith("/")) {
+         if (packageLocation.endsWith(".zip")) {
+            packageLocation =
+               await this.projectStore.unzipProject(packageLocation);
+         }
+         await this.projectStore.mountLocalDirectory(
+            packageLocation,
+            absoluteTargetPath,
+            projectName,
+         );
       }
    }
 }

package/src/controller/schedule.controller.ts

@@ -14,8 +14,8 @@ export class ScheduleController {
       projectName: string,
       packageName: string,
    ): Promise<ApiSchedule[]> {
-      const project = await this.projectStore.getProject(projectName);
-      const p = await project.getPackage(packageName);
-      return p.listSchedules();
+      const project = await this.projectStore?.getProject?.(projectName);
+      const p = await project?.getPackage?.(packageName);
+      return p?.listSchedules?.();
    }
 }

package/src/logger.ts

@@ -20,11 +20,20 @@ export const logger = winston.createLogger({
 
 export const loggerMiddleware: RequestHandler = (req, res, next) => {
    const startTime = performance.now();
+   const resJson = res.json;
+   res.json = (body: unknown) => {
+      res.locals.body = body;
+      return resJson.call(res, body);
+   };
    res.on("finish", () => {
       const endTime = performance.now();
       logger.info(`${req.method} ${req.url}`, {
          statusCode: res.statusCode,
          duration: endTime - startTime,
+         payload: req.body,
+         response: res.locals.body,
+         params: req.params,
+         query: req.query,
       });
    });
    next();

package/src/server.ts

@@ -92,7 +92,7 @@ const databaseController = new DatabaseController(projectStore);
 const queryController = new QueryController(projectStore);
 const scheduleController = new ScheduleController(projectStore);
 
-const mcpApp = express();
+export const mcpApp = express();
 
 mcpApp.use(MCP_ENDPOINT, express.json());
 mcpApp.use(MCP_ENDPOINT, cors());
@@ -208,6 +208,13 @@ const setVersionIdError = (res: express.Response) => {
 app.use(cors());
 app.use(bodyParser.json());
 
+app.get(`${API_PREFIX}/status`, async (_req, res) => {
+   res.status(200).json({
+      timestamp: Date.now(),
+      projects: await projectStore.listProjects(),
+   });
+});
+
 app.get(`${API_PREFIX}/watch-mode/status`, watchModeController.getWatchStatus);
 app.post(`${API_PREFIX}/watch-mode/start`, watchModeController.startWatching);
 app.post(`${API_PREFIX}/watch-mode/stop`, watchModeController.stopWatchMode);
@@ -224,7 +231,9 @@ app.get(`${API_PREFIX}/projects`, async (_req, res) => {
 
 app.post(`${API_PREFIX}/projects`, async (req, res) => {
    try {
-      res.status(200).json(await projectStore.addProject(req.body));
+      logger.info("Adding project", { body: req.body });
+      const project = await projectStore.addProject(req.body);
+      res.status(200).json(await project.serialize());
    } catch (error) {
       logger.error(error);
       const { json, status } = internalErrorToHttpError(error as Error);
@@ -238,7 +247,7 @@ app.get(`${API_PREFIX}/projects/:projectName`, async (req, res) => {
          req.params.projectName,
          req.query.reload === "true",
       );
-      res.status(200).json(project.metadata);
+      res.status(200).json(await project.serialize());
    } catch (error) {
       logger.error(error);
       const { json, status } = internalErrorToHttpError(error as Error);
@@ -248,7 +257,8 @@ app.get(`${API_PREFIX}/projects/:projectName`, async (req, res) => {
 
 app.patch(`${API_PREFIX}/projects/:projectName`, async (req, res) => {
    try {
-      res.status(200).json(await projectStore.updateProject(req.body));
+      const project = await projectStore.updateProject(req.body);
+      res.status(200).json(await project.serialize());
    } catch (error) {
       logger.error(error);
       const { json, status } = internalErrorToHttpError(error as Error);
@@ -258,9 +268,8 @@ app.patch(`${API_PREFIX}/projects/:projectName`, async (req, res) => {
 
 app.delete(`${API_PREFIX}/projects/:projectName`, async (req, res) => {
    try {
-      res.status(200).json(
-         await projectStore.deleteProject(req.params.projectName),
-      );
+      const project = await projectStore.deleteProject(req.params.projectName);
+      res.status(200).json(await project.serialize());
    } catch (error) {
       logger.error(error);
       const { json, status } = internalErrorToHttpError(error as Error);
@@ -302,12 +311,11 @@ app.get(
    `${API_PREFIX}/projects/:projectName/connections/:connectionName/test`,
    async (req, res) => {
       try {
-         res.status(200).json(
-            await connectionController.testConnection(
-               req.params.projectName,
-               req.params.connectionName,
-            ),
+         const connectionStatus = await connectionController.testConnection(
+            req.params.projectName,
+            req.params.connectionName,
          );
+         res.status(200).json(connectionStatus);
       } catch (error) {
          logger.error(error);
          const { json, status } = internalErrorToHttpError(error as Error);
@@ -451,9 +459,11 @@ app.get(`${API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
 
 app.post(`${API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
    try {
-      res.status(200).json(
-         await packageController.addPackage(req.params.projectName, req.body),
+      const _package = await packageController.addPackage(
+         req.params.projectName,
+         req.body,
       );
+      res.status(200).json(_package?.getPackageMetadata());
    } catch (error) {
       logger.error(error);
       const { json, status } = internalErrorToHttpError(error as Error);
@@ -711,22 +721,18 @@ app.use(
 );
 
 const mainServer = http.createServer(app);
-projectStore.finishedInitialization.then(() => {
-   mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, () => {
-      const address = mainServer.address() as AddressInfo;
-      logger.info(
-         `Publisher server listening at http://${address.address}:${address.port}`,
-      );
-      if (isDevelopment) {
-         logger.info(
-            "Running in development mode - proxying to React dev server at http://localhost:5173",
-         );
-      }
-   });
-   mcpApp.listen(MCP_PORT, PUBLISHER_HOST, () => {
+
+mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, () => {
+   const address = mainServer.address() as AddressInfo;
+   logger.info(
+      `Publisher server listening at http://${address.address}:${address.port}`,
+   );
+   if (isDevelopment) {
       logger.info(
-         `MCP server listening at http://${PUBLISHER_HOST}:${MCP_PORT}`,
+         "Running in development mode - proxying to React dev server at http://localhost:5173",
       );
-   });
+   }
+});
+mcpApp.listen(MCP_PORT, PUBLISHER_HOST, () => {
+   logger.info(`MCP server listening at http://${PUBLISHER_HOST}:${MCP_PORT}`);
 });
-

package/src/service/connection.ts

@@ -10,6 +10,7 @@ import path from "path";
 import { v4 as uuidv4 } from "uuid";
 import { components } from "../api";
 import { CONNECTIONS_MANIFEST_NAME } from "../constants";
+import { logger } from "../logger";
 
 type ApiConnection = components["schemas"]["Connection"];
 type ApiConnectionAttributes = components["schemas"]["ConnectionAttributes"];
@@ -42,184 +43,181 @@ export async function readConnectionConfig(
    return JSON.parse(connectionFileContents.toString()) as ApiConnection[];
 }
 
-export async function createConnections(basePath: string): Promise<{
+export async function createConnections(
+   basePath: string,
+   defaultConnections: ApiConnection[] = [],
+): Promise<{
    malloyConnections: Map<string, BaseConnection>;
    apiConnections: InternalConnection[];
 }> {
    const connectionMap = new Map<string, BaseConnection>();
    const connectionConfig = await readConnectionConfig(basePath);
 
-   if (connectionConfig.length > 0) {
-      connectionConfig.map(async (connection) => {
-         // This case shouldn't happen.  The package validation logic should
-         // catch it.
-         if (!connection.name) {
-            throw "Invalid connection configuration.  No name.";
+   for (const connection of [...defaultConnections, ...connectionConfig]) {
+      logger.info(`Adding connection ${connection.name}`, {
+         connection,
+      });
+      // This case shouldn't happen.  The package validation logic should
+      // catch it.
+      if (!connection.name) {
+         throw "Invalid connection configuration.  No name.";
+      }
+
+      switch (connection.type) {
+         case "postgres": {
+            const configReader = async () => {
+               if (!connection.postgresConnection) {
+                  throw "Invalid connection configuration.  No postgres connection.";
+               }
+               return {
+                  host: connection.postgresConnection.host,
+                  port: connection.postgresConnection.port,
+                  username: connection.postgresConnection.userName,
+                  password: connection.postgresConnection.password,
+                  databaseName: connection.postgresConnection.databaseName,
+                  connectionString:
+                     connection.postgresConnection.connectionString,
+               };
+            };
+            const postgresConnection = new PostgresConnection(
+               connection.name,
+               () => ({}),
+               configReader,
+            );
+            connectionMap.set(connection.name, postgresConnection);
+            connection.attributes = getConnectionAttributes(postgresConnection);
+            break;
          }
 
-         switch (connection.type) {
-            case "postgres": {
-               const configReader = async () => {
-                  if (!connection.postgresConnection) {
-                     throw "Invalid connection configuration.  No postgres connection.";
-                  }
-                  return {
-                     host: connection.postgresConnection.host,
-                     port: connection.postgresConnection.port,
-                     username: connection.postgresConnection.userName,
-                     password: connection.postgresConnection.password,
-                     databaseName: connection.postgresConnection.databaseName,
-                     connectionString:
-                        connection.postgresConnection.connectionString,
-                  };
-               };
-               const postgresConnection = new PostgresConnection(
-                  connection.name,
-                  () => ({}),
-                  configReader,
-               );
-               connectionMap.set(connection.name, postgresConnection);
-               connection.attributes =
-                  getConnectionAttributes(postgresConnection);
-               break;
+         case "mysql": {
+            if (!connection.mysqlConnection) {
+               throw "Invalid connection configuration.  No mysql connection.";
             }
+            const config = {
+               host: connection.mysqlConnection.host,
+               port: connection.mysqlConnection.port,
+               user: connection.mysqlConnection.user,
+               password: connection.mysqlConnection.password,
+               database: connection.mysqlConnection.database,
+            };
+            const mysqlConnection = new MySQLConnection(
+               connection.name,
+               config,
+            );
+            connectionMap.set(connection.name, mysqlConnection);
+            connection.attributes = getConnectionAttributes(mysqlConnection);
+            break;
+         }
 
-            case "mysql": {
-               if (!connection.mysqlConnection) {
-                  throw "Invalid connection configuration.  No mysql connection.";
-               }
-               const config = {
-                  host: connection.mysqlConnection.host,
-                  port: connection.mysqlConnection.port,
-                  user: connection.mysqlConnection.user,
-                  password: connection.mysqlConnection.password,
-                  database: connection.mysqlConnection.database,
-               };
-               const mysqlConnection = new MySQLConnection(
-                  connection.name,
-                  config,
-               );
-               connectionMap.set(connection.name, mysqlConnection);
-               connection.attributes = getConnectionAttributes(mysqlConnection);
-               break;
+         case "bigquery": {
+            if (!connection.bigqueryConnection) {
+               throw "Invalid connection configuration.  No bigquery connection.";
             }
 
-            case "bigquery": {
-               if (!connection.bigqueryConnection) {
-                  throw "Invalid connection configuration.  No bigquery connection.";
-               }
-
-               // If a service account key file is provided, we persist it to disk
-               // and pass the path to the BigQueryConnection.
-               let serviceAccountKeyPath = undefined;
-               if (connection.bigqueryConnection.serviceAccountKeyJson) {
-                  serviceAccountKeyPath = path.join(
-                     "/tmp",
-                     `${connection.name}-${uuidv4()}-service-account-key.json`,
-                  );
-                  await fs.writeFile(
-                     serviceAccountKeyPath,
-                     connection.bigqueryConnection
-                        .serviceAccountKeyJson as string,
-                  );
-               }
-
-               const bigqueryConnectionOptions = {
-                  projectId: connection.bigqueryConnection.defaultProjectId,
-                  serviceAccountKeyPath: serviceAccountKeyPath,
-                  location: connection.bigqueryConnection.location,
-                  maximumBytesBilled:
-                     connection.bigqueryConnection.maximumBytesBilled,
-                  timeoutMs:
-                     connection.bigqueryConnection.queryTimeoutMilliseconds,
-                  billingProjectId:
-                     connection.bigqueryConnection.billingProjectId,
-               };
-               const bigqueryConnection = new BigQueryConnection(
-                  connection.name,
-                  () => ({}),
-                  bigqueryConnectionOptions,
+            // If a service account key file is provided, we persist it to disk
+            // and pass the path to the BigQueryConnection.
+            let serviceAccountKeyPath = undefined;
+            if (connection.bigqueryConnection.serviceAccountKeyJson) {
+               serviceAccountKeyPath = path.join(
+                  "/tmp",
+                  `${connection.name}-${uuidv4()}-service-account-key.json`,
+               );
+               await fs.writeFile(
+                  serviceAccountKeyPath,
+                  connection.bigqueryConnection.serviceAccountKeyJson as string,
                );
-               connectionMap.set(connection.name, bigqueryConnection);
-               connection.attributes =
-                  getConnectionAttributes(bigqueryConnection);
-               break;
             }
 
-            case "snowflake": {
-               if (!connection.snowflakeConnection) {
-                  throw new Error(
-                     "Snowflake connection configuration is missing.",
-                  );
-               }
-               if (!connection.snowflakeConnection.account) {
-                  throw new Error("Snowflake account is required.");
-               }
-
-               if (!connection.snowflakeConnection.username) {
-                  throw new Error("Snowflake username is required.");
-               }
+            const bigqueryConnectionOptions = {
+               projectId: connection.bigqueryConnection.defaultProjectId,
+               serviceAccountKeyPath: serviceAccountKeyPath,
+               location: connection.bigqueryConnection.location,
+               maximumBytesBilled:
+                  connection.bigqueryConnection.maximumBytesBilled,
+               timeoutMs:
+                  connection.bigqueryConnection.queryTimeoutMilliseconds,
+               billingProjectId: connection.bigqueryConnection.billingProjectId,
+            };
+            const bigqueryConnection = new BigQueryConnection(
+               connection.name,
+               () => ({}),
+               bigqueryConnectionOptions,
+            );
+            connectionMap.set(connection.name, bigqueryConnection);
+            connection.attributes = getConnectionAttributes(bigqueryConnection);
+            break;
+         }
 
-               if (!connection.snowflakeConnection.password) {
-                  throw new Error("Snowflake password is required.");
-               }
+         case "snowflake": {
+            if (!connection.snowflakeConnection) {
+               throw new Error(
+                  "Snowflake connection configuration is missing.",
+               );
+            }
+            if (!connection.snowflakeConnection.account) {
+               throw new Error("Snowflake account is required.");
+            }
 
-               if (!connection.snowflakeConnection.warehouse) {
-                  throw new Error("Snowflake warehouse is required.");
-               }
+            if (!connection.snowflakeConnection.username) {
+               throw new Error("Snowflake username is required.");
+            }
 
-               const snowflakeConnectionOptions = {
-                  connOptions: {
-                     account: connection.snowflakeConnection.account,
-                     username: connection.snowflakeConnection.username,
-                     password: connection.snowflakeConnection.password,
-                     warehouse: connection.snowflakeConnection.warehouse,
-                     database: connection.snowflakeConnection.database,
-                     schema: connection.snowflakeConnection.schema,
-                     timeout:
-                        connection.snowflakeConnection
-                           .responseTimeoutMilliseconds,
-                  },
-               };
-               const snowflakeConnection = new SnowflakeConnection(
-                  connection.name,
-                  snowflakeConnectionOptions,
-               );
-               connectionMap.set(connection.name, snowflakeConnection);
-               connection.attributes =
-                  getConnectionAttributes(snowflakeConnection);
-               break;
+            if (!connection.snowflakeConnection.password) {
+               throw new Error("Snowflake password is required.");
             }
 
-            case "trino": {
-               if (!connection.trinoConnection) {
-                  throw new Error("Trino connection configuration is missing.");
-               }
-               const trinoConnectionOptions = {
-                  server: connection.trinoConnection.server,
-                  port: connection.trinoConnection.port,
-                  catalog: connection.trinoConnection.catalog,
-                  schema: connection.trinoConnection.schema,
-                  user: connection.trinoConnection.user,
-                  password: connection.trinoConnection.password,
-               };
-               const trinoConnection = new TrinoConnection(
-                  connection.name,
-                  {},
-                  trinoConnectionOptions,
-               );
-               connectionMap.set(connection.name, trinoConnection);
-               connection.attributes = getConnectionAttributes(trinoConnection);
-               break;
+            if (!connection.snowflakeConnection.warehouse) {
+               throw new Error("Snowflake warehouse is required.");
             }
 
-            default: {
-               throw new Error(
-                  `Unsupported connection type: ${connection.type}`,
-               );
+            const snowflakeConnectionOptions = {
+               connOptions: {
+                  account: connection.snowflakeConnection.account,
+                  username: connection.snowflakeConnection.username,
+                  password: connection.snowflakeConnection.password,
+                  warehouse: connection.snowflakeConnection.warehouse,
+                  database: connection.snowflakeConnection.database,
+                  schema: connection.snowflakeConnection.schema,
+                  timeout:
+                     connection.snowflakeConnection.responseTimeoutMilliseconds,
+               },
+            };
+            const snowflakeConnection = new SnowflakeConnection(
+               connection.name,
+               snowflakeConnectionOptions,
+            );
+            connectionMap.set(connection.name, snowflakeConnection);
+            connection.attributes =
+               getConnectionAttributes(snowflakeConnection);
+            break;
+         }
+
+         case "trino": {
+            if (!connection.trinoConnection) {
+               throw new Error("Trino connection configuration is missing.");
             }
+            const trinoConnectionOptions = {
+               server: connection.trinoConnection.server,
+               port: connection.trinoConnection.port,
+               catalog: connection.trinoConnection.catalog,
+               schema: connection.trinoConnection.schema,
+               user: connection.trinoConnection.user,
+               password: connection.trinoConnection.password,
+            };
+            const trinoConnection = new TrinoConnection(
+               connection.name,
+               {},
+               trinoConnectionOptions,
+            );
+            connectionMap.set(connection.name, trinoConnection);
+            connection.attributes = getConnectionAttributes(trinoConnection);
+            break;
          }
-      });
+
+         default: {
+            throw new Error(`Unsupported connection type: ${connection.type}`);
+         }
+      }
    }
 
    return {