@malloy-publisher/server 0.0.92 → 0.0.93

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.92",
+   "version": "0.0.93",
    "main": "dist/server.js",
    "bin": {
       "malloy-publisher": "dist/server.js"

package/src/constants.ts

@@ -1,4 +1,6 @@
 import fs from "fs";
+import os from "os";
+import path from "path";
 export const API_PREFIX = "/api/v0";
 export const README_NAME = "README.md";
 export const PUBLISHER_CONFIG_NAME = "publisher.config.json";
@@ -7,10 +9,24 @@ export const CONNECTIONS_MANIFEST_NAME = "publisher.connections.json";
 export const MODEL_FILE_SUFFIX = ".malloy";
 export const NOTEBOOK_FILE_SUFFIX = ".malloynb";
 export const ROW_LIMIT = 1000;
+export const TEMP_DIR_PATH = os.tmpdir();
+export let publisherPath: string;
 
-export let publisherPath = "/etc/publisher";
-try {
-   fs.accessSync(publisherPath, fs.constants.W_OK);
-} catch {
-   publisherPath = "/tmp/publisher";
+if (process.platform === "win32") {
+   publisherPath = path.join(
+      process.env.PROGRAMDATA || "C:\\ProgramData",
+      "publisher",
+   );
+   try {
+      fs.accessSync(publisherPath, fs.constants.W_OK);
+   } catch {
+      publisherPath = path.join(os.tmpdir(), "publisher");
+   }
+} else {
+   publisherPath = "/etc/publisher";
+   try {
+      fs.accessSync(publisherPath, fs.constants.W_OK);
+   } catch {
+      publisherPath = path.join(os.tmpdir(), "publisher");
+   }
 }

package/src/controller/connection.controller.ts

@@ -4,15 +4,15 @@ import {
    TestableConnection,
 } from "@malloydata/malloy";
 import { Connection, PersistSQLResults } from "@malloydata/malloy/connection";
-import { components } from "../api";
-import { ConnectionError } from "../errors";
+import { BadRequestError, ConnectionError } from "../errors";
 import { logger } from "../logger";
+import { components } from "../api";
 import {
    getSchemasForConnection,
    getTablesForSchema,
 } from "../service/db_utils";
 import { ProjectStore } from "../service/project_store";
-
+import { testConnectionConfig } from "../service/connection";
 type ApiConnection = components["schemas"]["Connection"];
 type ApiConnectionStatus = components["schemas"]["ConnectionStatus"];
 type ApiSqlSource = components["schemas"]["SqlSource"];
@@ -189,4 +189,33 @@ export class ConnectionController {
          throw new ConnectionError((error as Error).message);
       }
    }
+
+   public async testConnectionConfiguration(
+      connectionConfig: ApiConnection,
+   ): Promise<ApiConnectionStatus> {
+      if (
+         !connectionConfig ||
+         typeof connectionConfig !== "object" ||
+         Object.keys(connectionConfig).length === 0
+      ) {
+         throw new BadRequestError(
+            "Connection configuration is required and cannot be empty",
+         );
+      }
+
+      if (!connectionConfig.type || typeof connectionConfig.type !== "string") {
+         throw new BadRequestError(
+            "Connection type is required and must be a string",
+         );
+      }
+
+      try {
+         return await testConnectionConfig(connectionConfig);
+      } catch (error) {
+         return {
+            status: "failed",
+            errorMessage: `Connection test failed: ${(error as Error).message}`,
+         };
+      }
+   }
 }

package/src/server.ts

@@ -311,6 +311,20 @@ app.get(
    },
 );
 
+app.post(`${API_PREFIX}/connections/test`, async (req, res) => {
+   try {
+      const connectionStatus =
+         await connectionController.testConnectionConfiguration(
+            req.body.connectionConfig,
+         );
+      res.status(200).json(connectionStatus);
+   } catch (error) {
+      logger.error(error);
+      const { json, status } = internalErrorToHttpError(error as Error);
+      res.status(status).json(json);
+   }
+});
+
 app.get(
    `${API_PREFIX}/projects/:projectName/connections/:connectionName/test`,
    async (req, res) => {

package/src/service/connection.ts

@@ -4,16 +4,19 @@ import { PostgresConnection } from "@malloydata/db-postgres";
 import { SnowflakeConnection } from "@malloydata/db-snowflake";
 import { TrinoConnection } from "@malloydata/db-trino";
 import { Connection } from "@malloydata/malloy";
+import { BadRequestError } from "../errors";
+import { logAxiosError, logger } from "../logger";
 import { BaseConnection } from "@malloydata/malloy/connection";
 import fs from "fs/promises";
 import path from "path";
 import { v4 as uuidv4 } from "uuid";
 import { components } from "../api";
-import { CONNECTIONS_MANIFEST_NAME } from "../constants";
-import { logger } from "../logger";
+import { CONNECTIONS_MANIFEST_NAME, TEMP_DIR_PATH } from "../constants";
+import { AxiosError } from "axios";
 
 type ApiConnection = components["schemas"]["Connection"];
 type ApiConnectionAttributes = components["schemas"]["ConnectionAttributes"];
+type ApiConnectionStatus = components["schemas"]["ConnectionStatus"];
 
 // Extends the public API connection with the internal connection objects
 // which contains passwords and connection strings.
@@ -129,7 +132,7 @@ export async function createConnections(
             let serviceAccountKeyPath = undefined;
             if (connection.bigqueryConnection.serviceAccountKeyJson) {
                serviceAccountKeyPath = path.join(
-                  "/tmp",
+                  TEMP_DIR_PATH,
                   `${connection.name}-${uuidv4()}-service-account-key.json`,
                );
                await fs.writeFile(
@@ -255,3 +258,257 @@ function getConnectionAttributes(
       canStream: canStream,
    };
 }
+
+export async function testConnectionConfig(
+   connectionConfig: ApiConnection,
+): Promise<ApiConnectionStatus> {
+   let testResult: { status: "ok" | "failed"; errorMessage?: string };
+
+   switch (connectionConfig.type) {
+      case "postgres": {
+         if (!connectionConfig.postgresConnection) {
+            throw new Error(
+               "Invalid connection configuration. No postgres connection.",
+            );
+         }
+
+         const postgresConfig = connectionConfig.postgresConnection;
+         if (
+            !postgresConfig.host ||
+            !postgresConfig.port ||
+            !postgresConfig.userName ||
+            !postgresConfig.password ||
+            !postgresConfig.databaseName
+         ) {
+            throw new Error(
+               "PostgreSQL connection requires: host, port, userName, password, and databaseName",
+            );
+         }
+
+         const configReader = async () => {
+            return {
+               host: postgresConfig.host,
+               port: postgresConfig.port,
+               username: postgresConfig.userName,
+               password: postgresConfig.password,
+               databaseName: postgresConfig.databaseName,
+               connectionString: postgresConfig.connectionString,
+            };
+         };
+
+         const postgresConnection = new PostgresConnection(
+            "testConnection",
+            () => ({}),
+            configReader,
+         );
+
+         try {
+            await postgresConnection.test();
+            testResult = { status: "ok" };
+         } catch (error) {
+            if (error instanceof AxiosError) {
+               logAxiosError(error);
+            } else {
+               logger.error(error);
+            }
+            testResult = {
+               status: "failed",
+               errorMessage: (error as Error).message,
+            };
+         }
+         break;
+      }
+
+      case "snowflake": {
+         if (!connectionConfig.snowflakeConnection) {
+            throw new Error(
+               "Invalid connection configuration. No snowflake connection.",
+            );
+         }
+
+         const snowflakeConfig = connectionConfig.snowflakeConnection;
+         if (
+            !snowflakeConfig.account ||
+            !snowflakeConfig.username ||
+            !snowflakeConfig.password ||
+            !snowflakeConfig.warehouse ||
+            !snowflakeConfig.database ||
+            !snowflakeConfig.schema
+         ) {
+            throw new Error(
+               "Snowflake connection requires: account, username, password, warehouse, database, and schema",
+            );
+         }
+
+         const snowflakeConnectionOptions = {
+            connOptions: {
+               account: snowflakeConfig.account,
+               username: snowflakeConfig.username,
+               password: snowflakeConfig.password,
+               warehouse: snowflakeConfig.warehouse,
+               database: snowflakeConfig.database,
+               schema: snowflakeConfig.schema,
+               timeout: snowflakeConfig.responseTimeoutMilliseconds,
+            },
+         };
+         const snowflakeConnection = new SnowflakeConnection(
+            "testConnection",
+            snowflakeConnectionOptions,
+         );
+
+         try {
+            await snowflakeConnection.test();
+            testResult = { status: "ok" };
+         } catch (error) {
+            if (error instanceof AxiosError) {
+               logAxiosError(error);
+            } else {
+               logger.error(error);
+            }
+            testResult = {
+               status: "failed",
+               errorMessage: (error as Error).message,
+            };
+         }
+         break;
+      }
+
+      case "bigquery": {
+         if (!connectionConfig.bigqueryConnection) {
+            throw new Error(
+               "Invalid connection configuration. No bigquery connection.",
+            );
+         }
+
+         const bigqueryConfig = connectionConfig.bigqueryConnection;
+         if (
+            !bigqueryConfig.serviceAccountKeyJson ||
+            !bigqueryConfig.defaultProjectId
+         ) {
+            throw new Error(
+               "BigQuery connection requires: serviceAccountKeyJson and defaultProjectId",
+            );
+         }
+
+         const serviceAccountKeyPath = path.join(
+            TEMP_DIR_PATH,
+            `test-${uuidv4()}-service-account-key.json`,
+         );
+
+         try {
+            await fs.writeFile(
+               serviceAccountKeyPath,
+               connectionConfig.bigqueryConnection
+                  .serviceAccountKeyJson as string,
+            );
+
+            const bigqueryConnectionOptions = {
+               projectId: connectionConfig.bigqueryConnection.defaultProjectId,
+               serviceAccountKeyPath: serviceAccountKeyPath,
+               location: connectionConfig.bigqueryConnection.location,
+               maximumBytesBilled:
+                  connectionConfig.bigqueryConnection.maximumBytesBilled,
+               timeoutMs:
+                  connectionConfig.bigqueryConnection.queryTimeoutMilliseconds,
+               billingProjectId:
+                  connectionConfig.bigqueryConnection.billingProjectId,
+            };
+            const bigqueryConnection = new BigQueryConnection(
+               "testConnection",
+               () => ({}),
+               bigqueryConnectionOptions,
+            );
+
+            await bigqueryConnection.test();
+            testResult = { status: "ok" };
+         } catch (error) {
+            if (error instanceof AxiosError) {
+               logAxiosError(error);
+            } else {
+               logger.error(error);
+            }
+            testResult = {
+               status: "failed",
+               errorMessage: (error as Error).message,
+            };
+         } finally {
+            try {
+               await fs.unlink(serviceAccountKeyPath);
+            } catch (cleanupError) {
+               logger.warn(
+                  `Failed to cleanup temporary file ${serviceAccountKeyPath}:`,
+                  cleanupError,
+               );
+            }
+         }
+         break;
+      }
+
+      case "trino": {
+         if (!connectionConfig.trinoConnection) {
+            throw new Error("Trino connection configuration is missing.");
+         }
+
+         const trinoConfig = connectionConfig.trinoConnection;
+         if (
+            !trinoConfig.server ||
+            !trinoConfig.port ||
+            !trinoConfig.catalog ||
+            !trinoConfig.schema ||
+            !trinoConfig.user ||
+            !trinoConfig.password
+         ) {
+            throw new Error(
+               "Trino connection requires: server, port, catalog, schema, user, and password",
+            );
+         }
+
+         const trinoConnectionOptions = {
+            server: trinoConfig.server,
+            port: trinoConfig.port,
+            catalog: trinoConfig.catalog,
+            schema: trinoConfig.schema,
+            user: trinoConfig.user,
+            password: trinoConfig.password,
+         };
+         const trinoConnection = new TrinoConnection(
+            "testConnection",
+            {},
+            trinoConnectionOptions,
+         );
+
+         try {
+            await trinoConnection.test();
+            testResult = { status: "ok" };
+         } catch (error) {
+            if (error instanceof AxiosError) {
+               logAxiosError(error);
+            } else {
+               logger.error(error);
+            }
+            testResult = {
+               status: "failed",
+               errorMessage: (error as Error).message,
+            };
+         }
+         break;
+      }
+
+      default:
+         throw new BadRequestError(
+            `Unsupported connection type: ${connectionConfig.type}`,
+         );
+   }
+
+   if (testResult.status === "failed") {
+      return {
+         status: "failed",
+         errorMessage: testResult.errorMessage || "Connection test failed",
+      };
+   }
+
+   return {
+      status: "ok",
+      errorMessage: "",
+   };
+}

package/src/service/project_store.spec.ts

@@ -6,10 +6,14 @@ import { components } from "../api";
 import { isPublisherConfigFrozen } from "../config";
 import { ProjectStore } from "./project_store";
 import { Project } from "./project";
+import { TEMP_DIR_PATH } from "../constants";
 
 type Connection = components["schemas"]["Connection"];
 
-const serverRootPath = "/tmp/pathways-worker-publisher-project-store-test";
+const serverRootPath = path.join(
+   TEMP_DIR_PATH,
+   "pathways-worker-publisher-project-store-test",
+);
 const projectName = "organizationName-projectName";
 const testConnections: Connection[] = [
    {
@@ -353,7 +357,10 @@ describe("ProjectStore Service", () => {
 describe("Project Service Error Recovery", () => {
    let sandbox: sinon.SinonSandbox;
    let projectStore: ProjectStore;
-   const serverRootPath = "/tmp/pathways-worker-publisher-error-recovery-test";
+   const serverRootPath = path.join(
+      TEMP_DIR_PATH,
+      "pathways-worker-publisher-error-recovery-test",
+   );
    const projectName = "organizationName-projectName-error-recovery";
    const testConnections: Connection[] = [
       {

package/src/service/project_store.ts

@@ -473,29 +473,6 @@ export class ProjectStore {
       projectName: string,
       packageName: string,
    ) {
-      // Handle absolute paths
-      if (location.startsWith("/")) {
-         try {
-            logger.info(
-               `Mounting local directory at "${location}" to "${targetPath}"`,
-            );
-            await this.mountLocalDirectory(
-               location,
-               targetPath,
-               projectName,
-               packageName,
-            );
-            return;
-         } catch (error) {
-            logger.error(`Failed to mount local directory "${location}"`, {
-               error,
-            });
-            throw new PackageNotFoundError(
-               `Failed to mount local directory: ${location}`,
-            );
-         }
-      }
-
       // Handle GCS paths
       if (location.startsWith("gs://")) {
          try {
@@ -553,6 +530,29 @@ export class ProjectStore {
          }
       }
 
+      // Handle absolute paths
+      if (path.isAbsolute(location)) {
+         try {
+            logger.info(
+               `Mounting local directory at "${location}" to "${targetPath}"`,
+            );
+            await this.mountLocalDirectory(
+               location,
+               targetPath,
+               projectName,
+               packageName,
+            );
+            return;
+         } catch (error) {
+            logger.error(`Failed to mount local directory "${location}"`, {
+               error,
+            });
+            throw new PackageNotFoundError(
+               `Failed to mount local directory: ${location}`,
+            );
+         }
+      }
+
       // If we get here, the path format is not supported
       const errorMsg = `Invalid package path: "${location}". Must be an absolute mounted path or a GCS/S3/GitHub URI.`;
       logger.error(errorMsg, { projectName, location });