@malloy-publisher/server 0.0.87 → 0.0.89

package/src/service/connection.ts

@@ -1,15 +1,16 @@
-import { PostgresConnection } from "@malloydata/db-postgres";
 import { BigQueryConnection } from "@malloydata/db-bigquery";
+import { MySQLConnection } from "@malloydata/db-mysql";
+import { PostgresConnection } from "@malloydata/db-postgres";
 import { SnowflakeConnection } from "@malloydata/db-snowflake";
 import { TrinoConnection } from "@malloydata/db-trino";
-import { MySQLConnection } from "@malloydata/db-mysql";
-import { v4 as uuidv4 } from "uuid";
 import { Connection } from "@malloydata/malloy";
-import { components } from "../api";
-import path from "path";
-import fs from "fs/promises";
-import { CONNECTIONS_MANIFEST_NAME } from "../utils";
 import { BaseConnection } from "@malloydata/malloy/connection";
+import fs from "fs/promises";
+import path from "path";
+import { v4 as uuidv4 } from "uuid";
+import { components } from "../api";
+import { CONNECTIONS_MANIFEST_NAME } from "../constants";
+import { logger } from "../logger";
 
 type ApiConnection = components["schemas"]["Connection"];
 type ApiConnectionAttributes = components["schemas"]["ConnectionAttributes"];
@@ -42,184 +43,181 @@ export async function readConnectionConfig(
    return JSON.parse(connectionFileContents.toString()) as ApiConnection[];
 }
 
-export async function createConnections(basePath: string): Promise<{
+export async function createConnections(
+   basePath: string,
+   defaultConnections: ApiConnection[] = [],
+): Promise<{
    malloyConnections: Map<string, BaseConnection>;
    apiConnections: InternalConnection[];
 }> {
    const connectionMap = new Map<string, BaseConnection>();
    const connectionConfig = await readConnectionConfig(basePath);
 
-   if (connectionConfig.length > 0) {
-      connectionConfig.map(async (connection) => {
-         // This case shouldn't happen.  The package validation logic should
-         // catch it.
-         if (!connection.name) {
-            throw "Invalid connection configuration.  No name.";
+   for (const connection of [...defaultConnections, ...connectionConfig]) {
+      logger.info(`Adding connection ${connection.name}`, {
+         connection,
+      });
+      // This case shouldn't happen.  The package validation logic should
+      // catch it.
+      if (!connection.name) {
+         throw "Invalid connection configuration.  No name.";
+      }
+
+      switch (connection.type) {
+         case "postgres": {
+            const configReader = async () => {
+               if (!connection.postgresConnection) {
+                  throw "Invalid connection configuration.  No postgres connection.";
+               }
+               return {
+                  host: connection.postgresConnection.host,
+                  port: connection.postgresConnection.port,
+                  username: connection.postgresConnection.userName,
+                  password: connection.postgresConnection.password,
+                  databaseName: connection.postgresConnection.databaseName,
+                  connectionString:
+                     connection.postgresConnection.connectionString,
+               };
+            };
+            const postgresConnection = new PostgresConnection(
+               connection.name,
+               () => ({}),
+               configReader,
+            );
+            connectionMap.set(connection.name, postgresConnection);
+            connection.attributes = getConnectionAttributes(postgresConnection);
+            break;
          }
 
-         switch (connection.type) {
-            case "postgres": {
-               const configReader = async () => {
-                  if (!connection.postgresConnection) {
-                     throw "Invalid connection configuration.  No postgres connection.";
-                  }
-                  return {
-                     host: connection.postgresConnection.host,
-                     port: connection.postgresConnection.port,
-                     username: connection.postgresConnection.userName,
-                     password: connection.postgresConnection.password,
-                     databaseName: connection.postgresConnection.databaseName,
-                     connectionString:
-                        connection.postgresConnection.connectionString,
-                  };
-               };
-               const postgresConnection = new PostgresConnection(
-                  connection.name,
-                  () => ({}),
-                  configReader,
-               );
-               connectionMap.set(connection.name, postgresConnection);
-               connection.attributes =
-                  getConnectionAttributes(postgresConnection);
-               break;
+         case "mysql": {
+            if (!connection.mysqlConnection) {
+               throw "Invalid connection configuration.  No mysql connection.";
             }
+            const config = {
+               host: connection.mysqlConnection.host,
+               port: connection.mysqlConnection.port,
+               user: connection.mysqlConnection.user,
+               password: connection.mysqlConnection.password,
+               database: connection.mysqlConnection.database,
+            };
+            const mysqlConnection = new MySQLConnection(
+               connection.name,
+               config,
+            );
+            connectionMap.set(connection.name, mysqlConnection);
+            connection.attributes = getConnectionAttributes(mysqlConnection);
+            break;
+         }
 
-            case "mysql": {
-               if (!connection.mysqlConnection) {
-                  throw "Invalid connection configuration.  No mysql connection.";
-               }
-               const config = {
-                  host: connection.mysqlConnection.host,
-                  port: connection.mysqlConnection.port,
-                  user: connection.mysqlConnection.user,
-                  password: connection.mysqlConnection.password,
-                  database: connection.mysqlConnection.database,
-               };
-               const mysqlConnection = new MySQLConnection(
-                  connection.name,
-                  config,
-               );
-               connectionMap.set(connection.name, mysqlConnection);
-               connection.attributes = getConnectionAttributes(mysqlConnection);
-               break;
+         case "bigquery": {
+            if (!connection.bigqueryConnection) {
+               throw "Invalid connection configuration.  No bigquery connection.";
             }
 
-            case "bigquery": {
-               if (!connection.bigqueryConnection) {
-                  throw "Invalid connection configuration.  No bigquery connection.";
-               }
-
-               // If a service account key file is provided, we persist it to disk
-               // and pass the path to the BigQueryConnection.
-               let serviceAccountKeyPath = undefined;
-               if (connection.bigqueryConnection.serviceAccountKeyJson) {
-                  serviceAccountKeyPath = path.join(
-                     "/tmp",
-                     `${connection.name}-${uuidv4()}-service-account-key.json`,
-                  );
-                  await fs.writeFile(
-                     serviceAccountKeyPath,
-                     connection.bigqueryConnection
-                        .serviceAccountKeyJson as string,
-                  );
-               }
-
-               const bigqueryConnectionOptions = {
-                  projectId: connection.bigqueryConnection.defaultProjectId,
-                  serviceAccountKeyPath: serviceAccountKeyPath,
-                  location: connection.bigqueryConnection.location,
-                  maximumBytesBilled:
-                     connection.bigqueryConnection.maximumBytesBilled,
-                  timeoutMs:
-                     connection.bigqueryConnection.queryTimeoutMilliseconds,
-                  billingProjectId:
-                     connection.bigqueryConnection.billingProjectId,
-               };
-               const bigqueryConnection = new BigQueryConnection(
-                  connection.name,
-                  () => ({}),
-                  bigqueryConnectionOptions,
+            // If a service account key file is provided, we persist it to disk
+            // and pass the path to the BigQueryConnection.
+            let serviceAccountKeyPath = undefined;
+            if (connection.bigqueryConnection.serviceAccountKeyJson) {
+               serviceAccountKeyPath = path.join(
+                  "/tmp",
+                  `${connection.name}-${uuidv4()}-service-account-key.json`,
+               );
+               await fs.writeFile(
+                  serviceAccountKeyPath,
+                  connection.bigqueryConnection.serviceAccountKeyJson as string,
                );
-               connectionMap.set(connection.name, bigqueryConnection);
-               connection.attributes =
-                  getConnectionAttributes(bigqueryConnection);
-               break;
             }
 
-            case "snowflake": {
-               if (!connection.snowflakeConnection) {
-                  throw new Error(
-                     "Snowflake connection configuration is missing.",
-                  );
-               }
-               if (!connection.snowflakeConnection.account) {
-                  throw new Error("Snowflake account is required.");
-               }
-
-               if (!connection.snowflakeConnection.username) {
-                  throw new Error("Snowflake username is required.");
-               }
+            const bigqueryConnectionOptions = {
+               projectId: connection.bigqueryConnection.defaultProjectId,
+               serviceAccountKeyPath: serviceAccountKeyPath,
+               location: connection.bigqueryConnection.location,
+               maximumBytesBilled:
+                  connection.bigqueryConnection.maximumBytesBilled,
+               timeoutMs:
+                  connection.bigqueryConnection.queryTimeoutMilliseconds,
+               billingProjectId: connection.bigqueryConnection.billingProjectId,
+            };
+            const bigqueryConnection = new BigQueryConnection(
+               connection.name,
+               () => ({}),
+               bigqueryConnectionOptions,
+            );
+            connectionMap.set(connection.name, bigqueryConnection);
+            connection.attributes = getConnectionAttributes(bigqueryConnection);
+            break;
+         }
 
-               if (!connection.snowflakeConnection.password) {
-                  throw new Error("Snowflake password is required.");
-               }
+         case "snowflake": {
+            if (!connection.snowflakeConnection) {
+               throw new Error(
+                  "Snowflake connection configuration is missing.",
+               );
+            }
+            if (!connection.snowflakeConnection.account) {
+               throw new Error("Snowflake account is required.");
+            }
 
-               if (!connection.snowflakeConnection.warehouse) {
-                  throw new Error("Snowflake warehouse is required.");
-               }
+            if (!connection.snowflakeConnection.username) {
+               throw new Error("Snowflake username is required.");
+            }
 
-               const snowflakeConnectionOptions = {
-                  connOptions: {
-                     account: connection.snowflakeConnection.account,
-                     username: connection.snowflakeConnection.username,
-                     password: connection.snowflakeConnection.password,
-                     warehouse: connection.snowflakeConnection.warehouse,
-                     database: connection.snowflakeConnection.database,
-                     schema: connection.snowflakeConnection.schema,
-                     timeout:
-                        connection.snowflakeConnection
-                           .responseTimeoutMilliseconds,
-                  },
-               };
-               const snowflakeConnection = new SnowflakeConnection(
-                  connection.name,
-                  snowflakeConnectionOptions,
-               );
-               connectionMap.set(connection.name, snowflakeConnection);
-               connection.attributes =
-                  getConnectionAttributes(snowflakeConnection);
-               break;
+            if (!connection.snowflakeConnection.password) {
+               throw new Error("Snowflake password is required.");
             }
 
-            case "trino": {
-               if (!connection.trinoConnection) {
-                  throw new Error("Trino connection configuration is missing.");
-               }
-               const trinoConnectionOptions = {
-                  server: connection.trinoConnection.server,
-                  port: connection.trinoConnection.port,
-                  catalog: connection.trinoConnection.catalog,
-                  schema: connection.trinoConnection.schema,
-                  user: connection.trinoConnection.user,
-                  password: connection.trinoConnection.password,
-               };
-               const trinoConnection = new TrinoConnection(
-                  connection.name,
-                  {},
-                  trinoConnectionOptions,
-               );
-               connectionMap.set(connection.name, trinoConnection);
-               connection.attributes = getConnectionAttributes(trinoConnection);
-               break;
+            if (!connection.snowflakeConnection.warehouse) {
+               throw new Error("Snowflake warehouse is required.");
             }
 
-            default: {
-               throw new Error(
-                  `Unsupported connection type: ${connection.type}`,
-               );
+            const snowflakeConnectionOptions = {
+               connOptions: {
+                  account: connection.snowflakeConnection.account,
+                  username: connection.snowflakeConnection.username,
+                  password: connection.snowflakeConnection.password,
+                  warehouse: connection.snowflakeConnection.warehouse,
+                  database: connection.snowflakeConnection.database,
+                  schema: connection.snowflakeConnection.schema,
+                  timeout:
+                     connection.snowflakeConnection.responseTimeoutMilliseconds,
+               },
+            };
+            const snowflakeConnection = new SnowflakeConnection(
+               connection.name,
+               snowflakeConnectionOptions,
+            );
+            connectionMap.set(connection.name, snowflakeConnection);
+            connection.attributes =
+               getConnectionAttributes(snowflakeConnection);
+            break;
+         }
+
+         case "trino": {
+            if (!connection.trinoConnection) {
+               throw new Error("Trino connection configuration is missing.");
             }
+            const trinoConnectionOptions = {
+               server: connection.trinoConnection.server,
+               port: connection.trinoConnection.port,
+               catalog: connection.trinoConnection.catalog,
+               schema: connection.trinoConnection.schema,
+               user: connection.trinoConnection.user,
+               password: connection.trinoConnection.password,
+            };
+            const trinoConnection = new TrinoConnection(
+               connection.name,
+               {},
+               trinoConnectionOptions,
+            );
+            connectionMap.set(connection.name, trinoConnection);
+            connection.attributes = getConnectionAttributes(trinoConnection);
+            break;
          }
-      });
+
+         default: {
+            throw new Error(`Unsupported connection type: ${connection.type}`);
+         }
+      }
    }
 
    return {

package/src/service/model.ts

@@ -25,6 +25,11 @@ import { metrics } from "@opentelemetry/api";
 import * as fs from "fs/promises";
 import * as path from "path";
 import { components } from "../api";
+import {
+   MODEL_FILE_SUFFIX,
+   NOTEBOOK_FILE_SUFFIX,
+   ROW_LIMIT,
+} from "../constants";
 import { HackyDataStylesAccumulator } from "../data_styles";
 import {
    BadRequestError,
@@ -32,12 +37,7 @@ import {
    ModelNotFoundError,
 } from "../errors";
 import { logger } from "../logger";
-import {
-   MODEL_FILE_SUFFIX,
-   NOTEBOOK_FILE_SUFFIX,
-   ROW_LIMIT,
-   URL_READER,
-} from "../utils";
+import { URL_READER } from "../utils";
 
 type ApiCompiledModel = components["schemas"]["CompiledModel"];
 type ApiNotebookCell = components["schemas"]["NotebookCell"];

package/src/service/package.spec.ts

@@ -4,13 +4,18 @@ import { join } from "path";
 import sinon from "sinon";
 import { PackageNotFoundError } from "../errors";
 import { readConnectionConfig } from "./connection";
-import { Model } from "./model";
+import { ApiConnection, Model } from "./model";
 import { Package } from "./package";
 import { Scheduler } from "./scheduler";
 
 // Minimal partial types for mocking
 type PartialScheduler = Pick<Scheduler, "list">;
 
+const connectionMocks: ApiConnection[] = [
+   { name: "conn1", type: "postgres", postgresConnection: {} },
+   { name: "conn2", type: "bigquery", bigqueryConnection: {} },
+];
+
 describe("service/package", () => {
    const testPackageDirectory = "testPackage";
 
@@ -24,10 +29,7 @@ describe("service/package", () => {
          join(testPackageDirectory, "database.csv"),
          parquetBuffer,
       );
-      const content = JSON.stringify([
-         { name: "conn1", type: "database" },
-         { name: "conn2", type: "api" },
-      ]);
+      const content = JSON.stringify(connectionMocks);
       await fs.writeFile(
          join(testPackageDirectory, "publisher.connections.json"),
          content,
@@ -82,7 +84,7 @@ describe("service/package", () => {
          });
          it("should return a Package object if the package exists", async () => {
             sinon.stub(fs, "stat").resolves();
-            sinon
+            const readFileStub = sinon
                .stub(fs, "readFile")
                .resolves(
                   Buffer.from(JSON.stringify({ description: "Test package" })),
@@ -98,6 +100,9 @@ describe("service/package", () => {
                list: () => [],
             } as PartialScheduler);
 
+            readFileStub.restore();
+            readFileStub.resolves(Buffer.from(JSON.stringify([])));
+
             const packageInstance = await Package.create(
                "testProject",
                "testPackage",
@@ -221,10 +226,7 @@ describe("service/package", () => {
             sinon.stub(fs, "stat").resolves();
             const config = await readConnectionConfig(testPackageDirectory);
 
-            expect(config).toEqual([
-               { name: "conn1", type: "database" },
-               { name: "conn2", type: "api" },
-            ]);
+            expect(config).toEqual(connectionMocks);
          });
       });
    });

package/src/service/package.ts

@@ -11,14 +11,14 @@ import {
 import { metrics } from "@opentelemetry/api";
 import recursive from "recursive-readdir";
 import { components } from "../api";
-import { API_PREFIX } from "../constants";
-import { PackageNotFoundError } from "../errors";
-import { logger } from "../logger";
 import {
+   API_PREFIX,
    MODEL_FILE_SUFFIX,
    NOTEBOOK_FILE_SUFFIX,
    PACKAGE_MANIFEST_NAME,
-} from "../utils";
+} from "../constants";
+import { PackageNotFoundError } from "../errors";
+import { logger } from "../logger";
 import { createConnections } from "./connection";
 import { Model } from "./model";
 import { Scheduler } from "./scheduler";
@@ -101,7 +101,10 @@ export class Package {
             unit: "ms",
          });
          const connections = new Map<string, Connection>(projectConnections);
-
+         logger.info(`Project connections: ${connections.size}`, {
+            connections,
+            projectConnections,
+         });
          // Package connections override project connections.
          const { malloyConnections: packageConnections } =
             await createConnections(packagePath);
@@ -147,6 +150,11 @@ export class Package {
             malloy_package_name: packageName,
             status: "success",
          });
+         logger.info(`Successfully loaded package ${packageName}`, {
+            packageName,
+            duration: executionTime,
+            unit: "ms",
+         });
          return new Package(
             projectName,
             packageName,
@@ -158,15 +166,14 @@ export class Package {
          );
       } catch (error) {
          logger.error(`Error loading package ${packageName}`, { error });
+         console.error(error);
          const endTime = performance.now();
          const executionTime = endTime - startTime;
          this.packageLoadHistogram.record(executionTime, {
             malloy_package_name: packageName,
             status: "error",
          });
-         throw new Error(`Error loading package ${packageName}`, {
-            cause: error,
-         });
+         throw error;
       }
    }