@malloy-publisher/server 0.0.91 → 0.0.93

package/src/service/connection.ts

@@ -4,16 +4,19 @@ import { PostgresConnection } from "@malloydata/db-postgres";
 import { SnowflakeConnection } from "@malloydata/db-snowflake";
 import { TrinoConnection } from "@malloydata/db-trino";
 import { Connection } from "@malloydata/malloy";
+import { BadRequestError } from "../errors";
+import { logAxiosError, logger } from "../logger";
 import { BaseConnection } from "@malloydata/malloy/connection";
 import fs from "fs/promises";
 import path from "path";
 import { v4 as uuidv4 } from "uuid";
 import { components } from "../api";
-import { CONNECTIONS_MANIFEST_NAME } from "../constants";
-import { logger } from "../logger";
+import { CONNECTIONS_MANIFEST_NAME, TEMP_DIR_PATH } from "../constants";
+import { AxiosError } from "axios";
 
 type ApiConnection = components["schemas"]["Connection"];
 type ApiConnectionAttributes = components["schemas"]["ConnectionAttributes"];
+type ApiConnectionStatus = components["schemas"]["ConnectionStatus"];
 
 // Extends the public API connection with the internal connection objects
 // which contains passwords and connection strings.
@@ -53,16 +56,26 @@ export async function createConnections(
    const connectionMap = new Map<string, BaseConnection>();
    const connectionConfig = await readConnectionConfig(basePath);
 
-   for (const connection of [...defaultConnections, ...connectionConfig]) {
+   const allConnections = [...defaultConnections, ...connectionConfig];
+
+   const processedConnections = new Set<string>();
+   const apiConnections: InternalConnection[] = [];
+
+   for (const connection of allConnections) {
+      if (connection.name && processedConnections.has(connection.name)) {
+         continue;
+      }
+
       logger.info(`Adding connection ${connection.name}`, {
          connection,
       });
-      // This case shouldn't happen.  The package validation logic should
-      // catch it.
+
       if (!connection.name) {
          throw "Invalid connection configuration.  No name.";
       }
 
+      processedConnections.add(connection.name);
+
       switch (connection.type) {
          case "postgres": {
             const configReader = async () => {
@@ -119,7 +132,7 @@ export async function createConnections(
             let serviceAccountKeyPath = undefined;
             if (connection.bigqueryConnection.serviceAccountKeyJson) {
                serviceAccountKeyPath = path.join(
-                  "/tmp",
+                  TEMP_DIR_PATH,
                   `${connection.name}-${uuidv4()}-service-account-key.json`,
                );
                await fs.writeFile(
@@ -218,11 +231,14 @@ export async function createConnections(
             throw new Error(`Unsupported connection type: ${connection.type}`);
          }
       }
+
+      // Add the connection to apiConnections (this will be sanitized when returned)
+      apiConnections.push(connection);
    }
 
    return {
       malloyConnections: connectionMap,
-      apiConnections: connectionConfig,
+      apiConnections: apiConnections,
    };
 }
 
@@ -242,3 +258,257 @@ function getConnectionAttributes(
       canStream: canStream,
    };
 }
+
+export async function testConnectionConfig(
+   connectionConfig: ApiConnection,
+): Promise<ApiConnectionStatus> {
+   let testResult: { status: "ok" | "failed"; errorMessage?: string };
+
+   switch (connectionConfig.type) {
+      case "postgres": {
+         if (!connectionConfig.postgresConnection) {
+            throw new Error(
+               "Invalid connection configuration. No postgres connection.",
+            );
+         }
+
+         const postgresConfig = connectionConfig.postgresConnection;
+         if (
+            !postgresConfig.host ||
+            !postgresConfig.port ||
+            !postgresConfig.userName ||
+            !postgresConfig.password ||
+            !postgresConfig.databaseName
+         ) {
+            throw new Error(
+               "PostgreSQL connection requires: host, port, userName, password, and databaseName",
+            );
+         }
+
+         const configReader = async () => {
+            return {
+               host: postgresConfig.host,
+               port: postgresConfig.port,
+               username: postgresConfig.userName,
+               password: postgresConfig.password,
+               databaseName: postgresConfig.databaseName,
+               connectionString: postgresConfig.connectionString,
+            };
+         };
+
+         const postgresConnection = new PostgresConnection(
+            "testConnection",
+            () => ({}),
+            configReader,
+         );
+
+         try {
+            await postgresConnection.test();
+            testResult = { status: "ok" };
+         } catch (error) {
+            if (error instanceof AxiosError) {
+               logAxiosError(error);
+            } else {
+               logger.error(error);
+            }
+            testResult = {
+               status: "failed",
+               errorMessage: (error as Error).message,
+            };
+         }
+         break;
+      }
+
+      case "snowflake": {
+         if (!connectionConfig.snowflakeConnection) {
+            throw new Error(
+               "Invalid connection configuration. No snowflake connection.",
+            );
+         }
+
+         const snowflakeConfig = connectionConfig.snowflakeConnection;
+         if (
+            !snowflakeConfig.account ||
+            !snowflakeConfig.username ||
+            !snowflakeConfig.password ||
+            !snowflakeConfig.warehouse ||
+            !snowflakeConfig.database ||
+            !snowflakeConfig.schema
+         ) {
+            throw new Error(
+               "Snowflake connection requires: account, username, password, warehouse, database, and schema",
+            );
+         }
+
+         const snowflakeConnectionOptions = {
+            connOptions: {
+               account: snowflakeConfig.account,
+               username: snowflakeConfig.username,
+               password: snowflakeConfig.password,
+               warehouse: snowflakeConfig.warehouse,
+               database: snowflakeConfig.database,
+               schema: snowflakeConfig.schema,
+               timeout: snowflakeConfig.responseTimeoutMilliseconds,
+            },
+         };
+         const snowflakeConnection = new SnowflakeConnection(
+            "testConnection",
+            snowflakeConnectionOptions,
+         );
+
+         try {
+            await snowflakeConnection.test();
+            testResult = { status: "ok" };
+         } catch (error) {
+            if (error instanceof AxiosError) {
+               logAxiosError(error);
+            } else {
+               logger.error(error);
+            }
+            testResult = {
+               status: "failed",
+               errorMessage: (error as Error).message,
+            };
+         }
+         break;
+      }
+
+      case "bigquery": {
+         if (!connectionConfig.bigqueryConnection) {
+            throw new Error(
+               "Invalid connection configuration. No bigquery connection.",
+            );
+         }
+
+         const bigqueryConfig = connectionConfig.bigqueryConnection;
+         if (
+            !bigqueryConfig.serviceAccountKeyJson ||
+            !bigqueryConfig.defaultProjectId
+         ) {
+            throw new Error(
+               "BigQuery connection requires: serviceAccountKeyJson and defaultProjectId",
+            );
+         }
+
+         const serviceAccountKeyPath = path.join(
+            TEMP_DIR_PATH,
+            `test-${uuidv4()}-service-account-key.json`,
+         );
+
+         try {
+            await fs.writeFile(
+               serviceAccountKeyPath,
+               connectionConfig.bigqueryConnection
+                  .serviceAccountKeyJson as string,
+            );
+
+            const bigqueryConnectionOptions = {
+               projectId: connectionConfig.bigqueryConnection.defaultProjectId,
+               serviceAccountKeyPath: serviceAccountKeyPath,
+               location: connectionConfig.bigqueryConnection.location,
+               maximumBytesBilled:
+                  connectionConfig.bigqueryConnection.maximumBytesBilled,
+               timeoutMs:
+                  connectionConfig.bigqueryConnection.queryTimeoutMilliseconds,
+               billingProjectId:
+                  connectionConfig.bigqueryConnection.billingProjectId,
+            };
+            const bigqueryConnection = new BigQueryConnection(
+               "testConnection",
+               () => ({}),
+               bigqueryConnectionOptions,
+            );
+
+            await bigqueryConnection.test();
+            testResult = { status: "ok" };
+         } catch (error) {
+            if (error instanceof AxiosError) {
+               logAxiosError(error);
+            } else {
+               logger.error(error);
+            }
+            testResult = {
+               status: "failed",
+               errorMessage: (error as Error).message,
+            };
+         } finally {
+            try {
+               await fs.unlink(serviceAccountKeyPath);
+            } catch (cleanupError) {
+               logger.warn(
+                  `Failed to cleanup temporary file ${serviceAccountKeyPath}:`,
+                  cleanupError,
+               );
+            }
+         }
+         break;
+      }
+
+      case "trino": {
+         if (!connectionConfig.trinoConnection) {
+            throw new Error("Trino connection configuration is missing.");
+         }
+
+         const trinoConfig = connectionConfig.trinoConnection;
+         if (
+            !trinoConfig.server ||
+            !trinoConfig.port ||
+            !trinoConfig.catalog ||
+            !trinoConfig.schema ||
+            !trinoConfig.user ||
+            !trinoConfig.password
+         ) {
+            throw new Error(
+               "Trino connection requires: server, port, catalog, schema, user, and password",
+            );
+         }
+
+         const trinoConnectionOptions = {
+            server: trinoConfig.server,
+            port: trinoConfig.port,
+            catalog: trinoConfig.catalog,
+            schema: trinoConfig.schema,
+            user: trinoConfig.user,
+            password: trinoConfig.password,
+         };
+         const trinoConnection = new TrinoConnection(
+            "testConnection",
+            {},
+            trinoConnectionOptions,
+         );
+
+         try {
+            await trinoConnection.test();
+            testResult = { status: "ok" };
+         } catch (error) {
+            if (error instanceof AxiosError) {
+               logAxiosError(error);
+            } else {
+               logger.error(error);
+            }
+            testResult = {
+               status: "failed",
+               errorMessage: (error as Error).message,
+            };
+         }
+         break;
+      }
+
+      default:
+         throw new BadRequestError(
+            `Unsupported connection type: ${connectionConfig.type}`,
+         );
+   }
+
+   if (testResult.status === "failed") {
+      return {
+         status: "failed",
+         errorMessage: testResult.errorMessage || "Connection test failed",
+      };
+   }
+
+   return {
+      status: "ok",
+      errorMessage: "",
+   };
+}

package/src/service/db_utils.ts

@@ -9,6 +9,7 @@ import { components } from "../api";
 import { logger } from "../logger";
 import {
    ApiConnection,
+   // @ts-expect-error TODO: Fix missing MysqlConnection type in API
    MysqlConnection,
    PostgresConnection,
    SnowflakeConnection,
@@ -68,7 +69,7 @@ async function getSnowflakeConnection(
    }
    return new Promise((resolve, reject) => {
       const connection = snowflake.createConnection({
-         account: apiSnowflakeConnection.account,
+         account: apiSnowflakeConnection.account as string,
          username: apiSnowflakeConnection.username,
          password: apiSnowflakeConnection.password,
          database: apiSnowflakeConnection.database,
@@ -318,7 +319,7 @@ async function getSnowflakeSchemas(
    return new Promise((resolve, reject) => {
       connection.execute({
          sqlText: "SHOW SCHEMAS",
-         complete: (err, stmt, rows) => {
+         complete: (err, _stmt, rows) => {
             if (err) {
                reject(err);
             } else {

package/src/service/model.ts

@@ -20,6 +20,7 @@ import {
    MalloySQLStatementType,
 } from "@malloydata/malloy-sql";
 import malloyPackage from "@malloydata/malloy/package.json";
+// @ts-expect-error until this gets merged https://github.com/malloydata/malloy/pull/2484
 import { DataStyles } from "@malloydata/render";
 import { metrics } from "@opentelemetry/api";
 import * as fs from "fs/promises";
@@ -42,6 +43,7 @@ import { URL_READER } from "../utils";
 type ApiCompiledModel = components["schemas"]["CompiledModel"];
 type ApiNotebookCell = components["schemas"]["NotebookCell"];
 type ApiCompiledNotebook = components["schemas"]["CompiledNotebook"];
+// @ts-expect-error TODO: Fix missing Source type in API
 type ApiSource = components["schemas"]["Source"];
 type ApiView = components["schemas"]["View"];
 type ApiQuery = components["schemas"]["Query"];

package/src/service/package.spec.ts

@@ -7,6 +7,7 @@ import { readConnectionConfig } from "./connection";
 import { ApiConnection, Model } from "./model";
 import { Package } from "./package";
 import { Scheduler } from "./scheduler";
+import { Stats } from "fs";
 
 // Minimal partial types for mocking
 type PartialScheduler = Pick<Scheduler, "list">;
@@ -55,8 +56,16 @@ describe("service/package", () => {
          { name: "testPackage", description: "Test package" },
          [],
          new Map([
-            ["model1.malloy", { getPath: () => "model1.malloy" } as any],
-            ["model2.malloynb", { getPath: () => "model2.malloynb" } as any],
+            [
+               "model1.malloy",
+               // @ts-expect-error PartialModel is a partial type
+               { getPath: () => "model1.malloy" } as PartialModel,
+            ],
+            [
+               "model2.malloynb",
+               // @ts-expect-error PartialModel is a partial type
+               { getPath: () => "model2.malloynb" } as PartialModel,
+            ],
          ]),
          undefined,
       );
@@ -78,60 +87,69 @@ describe("service/package", () => {
                   new Map(),
                ),
             ).rejects.toThrowError(
-               PackageNotFoundError,
-               "Package manifest for testPackage does not exist.",
+               new PackageNotFoundError(
+                  "Package manifest for testPackage does not exist.",
+               ),
             );
          });
-         it("should return a Package object if the package exists", async () => {
-            sinon.stub(fs, "stat").resolves();
-            const readFileStub = sinon
-               .stub(fs, "readFile")
-               .resolves(
-                  Buffer.from(JSON.stringify({ description: "Test package" })),
+         it(
+            "should return a Package object if the package exists",
+            async () => {
+               sinon.stub(fs, "stat").resolves();
+               const readFileStub = sinon
+                  .stub(fs, "readFile")
+                  .resolves(
+                     Buffer.from(
+                        JSON.stringify({ description: "Test package" }),
+                     ),
+                  );
+
+               // Still use Partial<Model> for the stub resolution type
+               type PartialModel = Pick<Model, "getPath">;
+               sinon
+                  .stub(Model, "create")
+                  // @ts-expect-error PartialModel is a partial type
+                  .resolves({ getPath: () => "model1.model" } as PartialModel);
+
+               // @ts-expect-error PartialScheduler is a partial type
+               sinon.stub(Scheduler, "create").returns({
+                  list: () => [],
+               } as PartialScheduler);
+
+               readFileStub.restore();
+               readFileStub.resolves(Buffer.from(JSON.stringify([])));
+
+               const packageInstance = await Package.create(
+                  "testProject",
+                  "testPackage",
+                  testPackageDirectory,
+                  new Map(),
                );
 
-            // Still use Partial<Model> for the stub resolution type
-            type PartialModel = Pick<Model, "getPath">;
-            sinon
-               .stub(Model, "create")
-               .resolves({ getPath: () => "model1.model" } as PartialModel);
-
-            sinon.stub(Scheduler, "create").returns({
-               list: () => [],
-            } as PartialScheduler);
-
-            readFileStub.restore();
-            readFileStub.resolves(Buffer.from(JSON.stringify([])));
-
-            const packageInstance = await Package.create(
-               "testProject",
-               "testPackage",
-               testPackageDirectory,
-               new Map(),
-            );
-
-            expect(packageInstance).toBeInstanceOf(Package);
-            expect(packageInstance.getPackageName()).toBe("testPackage");
-            expect(packageInstance.getPackageMetadata().description).toBe(
-               "Test package",
-            );
-            expect(packageInstance.listDatabases()).toEqual([
-               {
-                  path: "database.csv",
-                  type: "embedded",
-                  info: {
-                     name: "database.csv",
-                     columns: [
-                        { name: "Name", type: "string" },
-                        { name: "Value", type: "number" },
-                     ],
-                     rowCount: 3,
+               expect(packageInstance).toBeInstanceOf(Package);
+               expect(packageInstance.getPackageName()).toBe("testPackage");
+               expect(packageInstance.getPackageMetadata().description).toBe(
+                  "Test package",
+               );
+               expect(packageInstance.listDatabases()).toEqual([
+                  {
+                     path: "database.csv",
+                     type: "embedded",
+                     info: {
+                        name: "database.csv",
+                        columns: [
+                           { name: "Name", type: "string" },
+                           { name: "Value", type: "number" },
+                        ],
+                        rowCount: 3,
+                     },
                   },
-               },
-            ]);
-            expect(packageInstance.listModels()).toBeEmpty();
-            expect(packageInstance.listSchedules()).toBeEmpty();
-         });
+               ]);
+               expect(packageInstance.listModels()).toBeEmpty();
+               expect(packageInstance.listSchedules()).toBeEmpty();
+            },
+            { timeout: 15000 },
+         );
       });
 
       describe("listModels", () => {
@@ -149,7 +167,8 @@ describe("service/package", () => {
                      {
                         getPath: () => "model1.malloy",
                         getModel: () => "foo",
-                     } as any,
+                        // @ts-expect-error PartialModel is a partial type
+                     } as PartialModel,
                   ],
                   [
                      "model2.malloynb",
@@ -160,7 +179,8 @@ describe("service/package", () => {
                               message: "This is the error",
                            };
                         },
-                     } as any,
+                        // @ts-expect-error PartialModel is a partial type
+                     } as PartialModel,
                   ],
                ]),
                undefined,
@@ -169,6 +189,7 @@ describe("service/package", () => {
             const models = await packageInstance.listModels();
             expect(models).toEqual([
                {
+                  // @ts-expect-error TODO: Fix missing projectName type in API
                   projectName: "testProject",
                   packageName: "testPackage",
                   path: "model1.malloy",
@@ -179,6 +200,7 @@ describe("service/package", () => {
             const notebooks = await packageInstance.listNotebooks();
             expect(notebooks).toEqual([
                {
+                  // @ts-expect-error TODO: Fix missing projectName type in API
                   projectName: "testProject",
                   packageName: "testPackage",
                   path: "model2.malloynb",
@@ -190,7 +212,7 @@ describe("service/package", () => {
 
       describe("getDatabaseInfo", () => {
          it("should return the size of the database file", async () => {
-            sinon.stub(fs, "stat").resolves({ size: 13 } as { size: number });
+            sinon.stub(fs, "stat").resolves({ size: 13 } as Stats);
 
             // @ts-expect-error Accessing private static method for testing
             const info = await Package.getDatabaseInfo(