@malloy-publisher/server 0.0.193 → 0.0.195

package/dist/server.mjs

@@ -217105,6 +217105,7 @@ import {
   SnowflakeConnection
 } from "@malloydata/db-snowflake";
 import"@malloydata/db-trino";
+import"@malloydata/db-databricks";
 import {
   contextOverlay,
   MalloyConfig
@@ -220380,6 +220381,7 @@ import fs from "fs/promises";
 import path2 from "path";
 
 // src/service/connection_config.ts
+import { createPrivateKey } from "crypto";
 import path from "path";
 var PUBLISHER_DUCKDB_API_FIELDS = new Set(["attachedDatabases"]);
 function normalizeSnowflakePrivateKey(privateKey) {
@@ -220398,6 +220400,12 @@ function normalizeSnowflakePrivateKey(privateKey) {
         endRegex: /-----END\s+PRIVATE\s+KEY-----/i,
         beginMarker: "-----BEGIN PRIVATE KEY-----",
         endMarker: "-----END PRIVATE KEY-----"
+      },
+      {
+        beginRegex: /-----BEGIN\s+RSA\s+PRIVATE\s+KEY-----/i,
+        endRegex: /-----END\s+RSA\s+PRIVATE\s+KEY-----/i,
+        beginMarker: "-----BEGIN RSA PRIVATE KEY-----",
+        endMarker: "-----END RSA PRIVATE KEY-----"
       }
     ];
     for (const pattern of keyPatterns) {
@@ -220424,6 +220432,21 @@ ${pattern.endMarker}
     privateKeyContent += `
 `;
   }
+  if (/-----BEGIN\s+RSA\s+PRIVATE\s+KEY-----/i.test(privateKeyContent)) {
+    try {
+      privateKeyContent = createPrivateKey({
+        key: privateKeyContent,
+        format: "pem"
+      }).export({ type: "pkcs8", format: "pem" }).toString();
+    } catch (err) {
+      throw new Error(`Failed to convert Snowflake RSA private key (PKCS#1) to PKCS#8: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    if (!privateKeyContent.endsWith(`
+`)) {
+      privateKeyContent += `
+`;
+    }
+  }
   return privateKeyContent;
 }
 function validateDuckdbApiSurface(connection) {
@@ -220467,6 +220490,13 @@ function getStaticConnectionAttributes(type) {
         canPersist: true,
         canStream: false
       };
+    case "databricks":
+      return {
+        dialectName: "databricks",
+        isPool: false,
+        canPersist: true,
+        canStream: false
+      };
     case "mysql":
       return {
         dialectName: "mysql",
@@ -220544,6 +220574,28 @@ function validateConnectionShape(connection) {
         throw new Error("Trino connection configuration is missing.");
       }
       break;
+    case "databricks": {
+      const databricks = connection.databricksConnection;
+      if (!databricks) {
+        throw new Error("Databricks connection configuration is missing.");
+      }
+      if (!databricks.host) {
+        throw new Error("Databricks host is required.");
+      }
+      if (!databricks.path) {
+        throw new Error("Databricks SQL warehouse HTTP path is required.");
+      }
+      const hasToken = !!databricks.token;
+      const hasOAuth = !!databricks.oauthClientId && !!databricks.oauthClientSecret;
+      if (!hasToken && !hasOAuth) {
+        throw new Error("Databricks requires either a personal access token or OAuth M2M client ID and secret.");
+      }
+      const hasDefaultCatalog = !!databricks.defaultCatalog;
+      if (!hasDefaultCatalog) {
+        throw new Error("Databricks default catalog is required.");
+      }
+      break;
+    }
     case "snowflake": {
       const snowflakeConnection = connection.snowflakeConnection;
       if (!snowflakeConnection) {
@@ -220661,6 +220713,21 @@ function assembleProjectConnections(connections = [], projectPath = "") {
         };
         break;
       }
+      case "databricks": {
+        const databricks = connection.databricksConnection;
+        pojo.connections[connection.name] = {
+          is: "databricks",
+          host: databricks?.host,
+          path: databricks?.path,
+          token: databricks?.token,
+          oauthClientId: databricks?.oauthClientId,
+          oauthClientSecret: databricks?.oauthClientSecret,
+          defaultCatalog: databricks?.defaultCatalog,
+          defaultSchema: databricks?.defaultSchema,
+          setupSQL: databricks?.setupSQL
+        };
+        break;
+      }
       case "duckdb": {
         if (attachedDatabases.some((database) => database.name === connection.name)) {
           throw new Error(`DuckDB attached database names cannot conflict with connection name ${connection.name}`);
@@ -222066,6 +222133,60 @@ async function getSchemasForTrino(connection, malloyConnection) {
     throw new Error(`Failed to get schemas for Trino connection ${connection.name}: ${error.message}`);
   }
 }
+async function getSchemasForDatabricks(connection, malloyConnection) {
+  if (!connection.databricksConnection) {
+    throw new Error("Databricks connection is required");
+  }
+  try {
+    const configuredSchema = connection.databricksConnection.defaultSchema;
+    let allRows = [];
+    if (connection.databricksConnection.defaultCatalog) {
+      const catalog = connection.databricksConnection.defaultCatalog;
+      const result = await malloyConnection.runSQL(`SELECT schema_name FROM ${catalog}.information_schema.schemata ORDER BY schema_name`);
+      const rows = standardizeRunSQLResult2(result);
+      allRows = rows.map((row) => {
+        const r = row;
+        return {
+          catalog,
+          schema: String(r.schema_name ?? r.Schema ?? "")
+        };
+      });
+    } else {
+      const catalogsResult = await malloyConnection.runSQL(`SHOW CATALOGS`);
+      const catalogNames = standardizeRunSQLResult2(catalogsResult).map((row) => {
+        const r = row;
+        return String(r.catalog ?? r.Catalog ?? r.catalog_name ?? "");
+      });
+      for (const catalog of catalogNames) {
+        try {
+          const result = await malloyConnection.runSQL(`SELECT schema_name FROM ${catalog}.information_schema.schemata ORDER BY schema_name`);
+          const rows = standardizeRunSQLResult2(result);
+          for (const row of rows) {
+            const r = row;
+            allRows.push({
+              catalog,
+              schema: String(r.schema_name ?? r.Schema ?? "")
+            });
+          }
+        } catch (catalogError) {
+          logger.warn(`Failed to list schemas for Databricks catalog ${catalog}`, { error: catalogError });
+        }
+      }
+    }
+    logger.info("allRows for Schemas for Databricks", { allRows });
+    return allRows.map(({ catalog, schema }) => {
+      const name = connection.databricksConnection?.defaultCatalog ? schema : `${catalog}.${schema}`;
+      return {
+        name,
+        isHidden: ["information_schema"].includes(schema),
+        isDefault: configuredSchema ? schema === configuredSchema : false
+      };
+    });
+  } catch (error) {
+    logger.error(`Error getting schemas for Databricks connection ${connection.name}`, { error });
+    throw new Error(`Failed to get schemas for Databricks connection ${connection.name}: ${error.message}`);
+  }
+}
 async function getSchemasForDuckDB(connection, malloyConnection) {
   if (!connection.duckdbConnection) {
     throw new Error("DuckDB connection is required");
@@ -222176,6 +222297,8 @@ async function getSchemasForConnection(connection, malloyConnection) {
       return getSchemasForSnowflake(connection, malloyConnection);
     case "trino":
       return getSchemasForTrino(connection, malloyConnection);
+    case "databricks":
+      return getSchemasForDatabricks(connection, malloyConnection);
     case "duckdb":
       return getSchemasForDuckDB(connection, malloyConnection);
     case "motherduck":
@@ -222358,6 +222481,8 @@ async function listTablesForSchema(connection, schemaName, malloyConnection, tab
       return listTablesForSnowflake(connection, schemaName, malloyConnection, tableNames);
     case "trino":
       return listTablesForTrino(connection, schemaName, malloyConnection, tableNames);
+    case "databricks":
+      return listTablesForDatabricks(connection, schemaName, malloyConnection, tableNames);
     case "duckdb":
       return listTablesForDuckDB(connection, schemaName, malloyConnection, tableNames);
     case "motherduck":
@@ -222486,6 +222611,37 @@ async function listTablesForTrino(connection, schemaName, malloyConnection, tabl
     throw new Error(`Failed to get tables for Trino schema ${schemaName} in connection ${connection.name}: ${error.message}`);
   }
 }
+async function listTablesForDatabricks(connection, schemaName, malloyConnection, tableNames) {
+  if (!connection.databricksConnection) {
+    throw new Error("Databricks connection is required");
+  }
+  try {
+    let catalogPrefix;
+    let schemaOnly;
+    let resourcePrefix;
+    if (connection.databricksConnection.defaultCatalog) {
+      catalogPrefix = `${connection.databricksConnection.defaultCatalog}.`;
+      schemaOnly = schemaName;
+      resourcePrefix = `${connection.databricksConnection.defaultCatalog}.${schemaName}`;
+    } else {
+      const dotIdx = schemaName.indexOf(".");
+      if (dotIdx > 0) {
+        catalogPrefix = `${schemaName.substring(0, dotIdx)}.`;
+        schemaOnly = schemaName.substring(dotIdx + 1);
+      } else {
+        catalogPrefix = "";
+        schemaOnly = schemaName;
+      }
+      resourcePrefix = schemaName;
+    }
+    const result = await malloyConnection.runSQL(`SELECT table_name, column_name, data_type FROM ${catalogPrefix}information_schema.columns WHERE table_schema = '${schemaOnly}' ${sqlInFilter("table_name", tableNames)} ORDER BY table_name, ordinal_position`);
+    const rows = standardizeRunSQLResult2(result);
+    return groupColumnRowsIntoTables(rows, (t) => `${resourcePrefix}.${t}`);
+  } catch (error) {
+    logger.error(`Error getting tables for Databricks schema ${schemaName} in connection ${connection.name}`, { error });
+    throw new Error(`Failed to get tables for Databricks schema ${schemaName} in connection ${connection.name}: ${error.message}`);
+  }
+}
 async function listTablesForDuckDB(connection, schemaName, malloyConnection, tableNames) {
   if (!connection.duckdbConnection) {
     throw new Error("DuckDB connection is required");
@@ -228884,7 +229040,12 @@ var getPublisherConfig = (serverRoot) => {
     const fileContent = fs2.readFileSync(publisherConfigPath, "utf8");
     rawConfig = JSON.parse(fileContent);
   } catch (error) {
-    logger.error(`Failed to parse ${PUBLISHER_CONFIG_NAME}. Using default empty config.`, { error });
+    const message = error instanceof Error ? error.message : String(error);
+    logger.error(`Failed to parse ${publisherConfigPath}: ${message}. Using default empty config.`, {
+      path: publisherConfigPath,
+      error: message,
+      stack: error instanceof Error ? error.stack : undefined
+    });
     return {
       frozenConfig: false,
       projects: []
@@ -231417,6 +231578,9 @@ class Project {
       this.metadata.readme = payload.readme;
       await this.writeProjectReadme(payload.readme);
     }
+    if (payload.materializationStorage !== undefined) {
+      this.metadata.materializationStorage = payload.materializationStorage;
+    }
     if (payload.connections) {
       const payloadConnections = payload.connections;
       await this.runConnectionUpdateExclusive(async () => {
@@ -232268,6 +232432,9 @@ class ProjectStore {
     if (!newProject.metadata)
       newProject.metadata = {};
     newProject.metadata.location = absoluteProjectPath;
+    if (project.materializationStorage !== undefined) {
+      newProject.metadata.materializationStorage = project.materializationStorage;
+    }
     this.projects.set(projectName, newProject);
     project?.packages?.forEach((_package) => {
       if (_package.name) {

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.193",
+   "version": "0.0.195",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"
@@ -35,6 +35,7 @@
       "@azure/storage-blob": "^12.26.0",
       "@google-cloud/storage": "^7.16.0",
       "@malloydata/db-bigquery": "^0.0.383",
+      "@malloydata/db-databricks": "^0.0.383",
       "@malloydata/db-duckdb": "^0.0.383",
       "@malloydata/db-mysql": "^0.0.383",
       "@malloydata/db-postgres": "^0.0.383",

package/src/config.ts

@@ -90,9 +90,14 @@ export const getPublisherConfig = (serverRoot: string): PublisherConfig => {
       const fileContent = fs.readFileSync(publisherConfigPath, "utf8");
       rawConfig = JSON.parse(fileContent);
    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
       logger.error(
-         `Failed to parse ${PUBLISHER_CONFIG_NAME}. Using default empty config.`,
-         { error },
+         `Failed to parse ${publisherConfigPath}: ${message}. Using default empty config.`,
+         {
+            path: publisherConfigPath,
+            error: message,
+            stack: error instanceof Error ? error.stack : undefined,
+         },
       );
       return {
          frozenConfig: false,

package/src/dto/connection.dto.spec.ts

@@ -5,6 +5,7 @@ import { validate } from "class-validator";
 import {
    BigqueryConnectionDto,
    ConnectionDto,
+   DatabricksConnectionDto,
    PostgresConnectionDto,
    SnowflakeConnectionDto,
    MysqlConnectionDto,
@@ -100,6 +101,56 @@ describe("dto/connection", () => {
          expect(errors).toHaveLength(0);
       });
 
+      it("should validate a valid DatabricksConnection object", async () => {
+         const validData = {
+            host: "dbc-xxxxxxxx-xxxx.cloud.databricks.com",
+            path: "/sql/1.0/warehouses/abc123",
+            token: "dapiXXXX",
+            oauthClientId: "client-id",
+            oauthClientSecret: "client-secret",
+            defaultCatalog: "main",
+            defaultSchema: "default",
+            setupSQL: "USE CATALOG main",
+         };
+         const databricksConnection = plainToInstance(
+            DatabricksConnectionDto,
+            validData,
+         );
+
+         const errors = await validate(databricksConnection);
+         expect(errors).toHaveLength(0);
+      });
+
+      it("should return errors for invalid DatabricksConnection object", async () => {
+         const invalidData = {
+            host: 123, // Invalid type
+            path: false, // Invalid type
+         };
+         const databricksConnection = plainToInstance(
+            DatabricksConnectionDto,
+            invalidData,
+         );
+
+         const errors = await validate(databricksConnection);
+         expect(errors).toHaveLength(2);
+      });
+
+      it("should validate a valid Connection object with databricks type", async () => {
+         const validData = {
+            name: "My Databricks Connection",
+            type: "databricks",
+            databricksConnection: {
+               host: "dbc-xxxxxxxx-xxxx.cloud.databricks.com",
+               path: "/sql/1.0/warehouses/abc123",
+               token: "dapiXXXX",
+            },
+         };
+         const connection = plainToInstance(ConnectionDto, validData);
+
+         const errors = await validate(connection);
+         expect(errors).toHaveLength(0);
+      });
+
       it("should validate a valid Connection object with postgres type", async () => {
          const validData = {
             name: "My Postgres Connection",

package/src/dto/connection.dto.ts

@@ -129,6 +129,40 @@ export class SnowflakeConnectionDto {
    responseTimeoutMilliseconds?: number;
 }
 
+export class DatabricksConnectionDto {
+   @IsOptional()
+   @IsString()
+   host?: string;
+
+   @IsOptional()
+   @IsString()
+   path?: string;
+
+   @IsOptional()
+   @IsString()
+   token?: string;
+
+   @IsOptional()
+   @IsString()
+   oauthClientId?: string;
+
+   @IsOptional()
+   @IsString()
+   oauthClientSecret?: string;
+
+   @IsOptional()
+   @IsString()
+   defaultCatalog?: string;
+
+   @IsOptional()
+   @IsString()
+   defaultSchema?: string;
+
+   @IsOptional()
+   @IsString()
+   setupSQL?: string;
+}
+
 export class TrinoConnectionDto {
    @IsOptional()
    @IsString()
@@ -176,6 +210,7 @@ export class ConnectionDto implements ApiConnection {
       "bigquery",
       "snowflake",
       "trino",
+      "databricks",
       "mysql",
       "duckdb",
       "motherduck",
@@ -186,6 +221,7 @@ export class ConnectionDto implements ApiConnection {
       | "bigquery"
       | "snowflake"
       | "trino"
+      | "databricks"
       | "mysql"
       | "duckdb"
       | "motherduck"
@@ -211,6 +247,11 @@ export class ConnectionDto implements ApiConnection {
    @Type(() => TrinoConnectionDto)
    TrinoConnection?: TrinoConnectionDto;
 
+   @IsOptional()
+   @ValidateNested()
+   @Type(() => DatabricksConnectionDto)
+   databricksConnection?: DatabricksConnectionDto;
+
    @IsOptional()
    @ValidateNested()
    @Type(() => DuckdbConnectionDto)

package/src/service/connection.ts

@@ -12,6 +12,7 @@ import {
 } from "@malloydata/db-snowflake";
 import type { TrinoConnection } from "@malloydata/db-trino";
 import "@malloydata/db-trino";
+import "@malloydata/db-databricks";
 import {
    Connection,
    contextOverlay,
@@ -50,6 +51,7 @@ export type InternalConnection = ApiConnection & {
    bigqueryConnection?: components["schemas"]["BigqueryConnection"];
    snowflakeConnection?: components["schemas"]["SnowflakeConnection"];
    trinoConnection?: components["schemas"]["TrinoConnection"];
+   databricksConnection?: components["schemas"]["DatabricksConnection"];
    mysqlConnection?: components["schemas"]["MysqlConnection"];
    duckdbConnection?: components["schemas"]["DuckdbConnection"];
    ducklakeConnection?: components["schemas"]["DucklakeConnection"];

package/src/service/connection_config.spec.ts

@@ -0,0 +1,168 @@
+import { generateKeyPairSync } from "crypto";
+import { describe, expect, it } from "bun:test";
+import {
+   assembleProjectConnections,
+   normalizeSnowflakePrivateKey,
+} from "./connection_config";
+import { components } from "../api";
+
+type ApiConnection = components["schemas"]["Connection"];
+
+describe("assembleProjectConnections — databricks", () => {
+   const validBase: ApiConnection = {
+      name: "dbx",
+      type: "databricks",
+      databricksConnection: {
+         host: "dbc.cloud.databricks.com",
+         path: "/sql/1.0/warehouses/abc",
+         token: "dapiXXXX",
+         defaultCatalog: "main",
+         defaultSchema: "default",
+      },
+   };
+
+   it("emits a databricks core entry with all known fields preserved", () => {
+      const { pojo, apiConnections } = assembleProjectConnections([validBase]);
+
+      const entry = pojo.connections["dbx"];
+      expect(entry.is).toBe("databricks");
+      expect(entry.host).toBe("dbc.cloud.databricks.com");
+      expect(entry.path).toBe("/sql/1.0/warehouses/abc");
+      expect(entry.token).toBe("dapiXXXX");
+      expect(entry.defaultCatalog).toBe("main");
+      expect(entry.defaultSchema).toBe("default");
+
+      expect(apiConnections).toHaveLength(1);
+      expect(apiConnections[0].attributes?.dialectName).toBe("databricks");
+   });
+
+   it("accepts OAuth M2M auth (clientId + secret) without a token", () => {
+      const conn: ApiConnection = {
+         name: "dbx-oauth",
+         type: "databricks",
+         databricksConnection: {
+            host: "dbc.cloud.databricks.com",
+            path: "/sql/1.0/warehouses/abc",
+            oauthClientId: "client-id",
+            oauthClientSecret: "client-secret",
+            defaultCatalog: "main",
+         },
+      };
+      const { pojo } = assembleProjectConnections([conn]);
+      const entry = pojo.connections["dbx-oauth"];
+      expect(entry.is).toBe("databricks");
+      expect(entry.oauthClientId).toBe("client-id");
+      expect(entry.oauthClientSecret).toBe("client-secret");
+      expect(entry.token).toBeUndefined();
+   });
+
+   it("rejects connections missing the databricksConnection block", () => {
+      const conn: ApiConnection = {
+         name: "dbx",
+         type: "databricks",
+      };
+      expect(() => assembleProjectConnections([conn])).toThrow(
+         "Databricks connection configuration is missing.",
+      );
+   });
+
+   it("rejects connections with a missing host", () => {
+      const conn: ApiConnection = {
+         ...validBase,
+         databricksConnection: {
+            ...validBase.databricksConnection!,
+            host: undefined,
+         },
+      };
+      expect(() => assembleProjectConnections([conn])).toThrow(
+         "Databricks host is required",
+      );
+   });
+
+   it("rejects connections with a missing path", () => {
+      const conn: ApiConnection = {
+         ...validBase,
+         databricksConnection: {
+            ...validBase.databricksConnection!,
+            path: undefined,
+         },
+      };
+      expect(() => assembleProjectConnections([conn])).toThrow(
+         "Databricks SQL warehouse HTTP path is required",
+      );
+   });
+
+   it("rejects when defaultCatalog is missing", () => {
+      const conn: ApiConnection = {
+         name: "dbx",
+         type: "databricks",
+         databricksConnection: {
+            host: "dbc.cloud.databricks.com",
+            path: "/sql/1.0/warehouses/abc",
+            token: "dapiXXXX",
+            // defaultCatalog deliberately omitted
+         },
+      };
+      expect(() => assembleProjectConnections([conn])).toThrow(
+         "Databricks default catalog is required",
+      );
+   });
+
+   it("rejects when neither token nor full OAuth credentials are provided", () => {
+      const conn: ApiConnection = {
+         name: "dbx",
+         type: "databricks",
+         databricksConnection: {
+            host: "dbc.cloud.databricks.com",
+            path: "/sql/1.0/warehouses/abc",
+            // Only oauthClientId, missing secret → rejected.
+            oauthClientId: "client-id",
+            defaultCatalog: "main",
+         },
+      };
+      expect(() => assembleProjectConnections([conn])).toThrow(
+         "Databricks requires",
+      );
+   });
+});
+
+describe("normalizeSnowflakePrivateKey", () => {
+   const { privateKey: pkcs8Pem } = generateKeyPairSync("rsa", {
+      modulusLength: 2048,
+      privateKeyEncoding: { type: "pkcs8", format: "pem" },
+      publicKeyEncoding: { type: "spki", format: "pem" },
+   });
+   const { privateKey: pkcs1Pem } = generateKeyPairSync("rsa", {
+      modulusLength: 2048,
+      privateKeyEncoding: { type: "pkcs1", format: "pem" },
+      publicKeyEncoding: { type: "pkcs1", format: "pem" },
+   });
+
+   it("passes a multi-line PKCS#8 key through and adds a trailing newline", () => {
+      const trimmed = (pkcs8Pem as string).trimEnd();
+      const result = normalizeSnowflakePrivateKey(trimmed);
+      expect(result).toContain("-----BEGIN PRIVATE KEY-----");
+      expect(result.endsWith("\n")).toBe(true);
+   });
+
+   it("converts a multi-line PKCS#1 RSA key to PKCS#8", () => {
+      const result = normalizeSnowflakePrivateKey(pkcs1Pem as string);
+      expect(result).toContain("-----BEGIN PRIVATE KEY-----");
+      expect(result).not.toContain("BEGIN RSA PRIVATE KEY");
+      expect(result.endsWith("\n")).toBe(true);
+   });
+
+   it("converts a single-line PKCS#1 RSA key to PKCS#8", () => {
+      const singleLine = (pkcs1Pem as string).replace(/\n/g, "");
+      const result = normalizeSnowflakePrivateKey(singleLine);
+      expect(result).toContain("-----BEGIN PRIVATE KEY-----");
+      expect(result).not.toContain("BEGIN RSA PRIVATE KEY");
+   });
+
+   it("reconstructs a single-line PKCS#8 key without conversion", () => {
+      const singleLine = (pkcs8Pem as string).replace(/\n/g, "");
+      const result = normalizeSnowflakePrivateKey(singleLine);
+      expect(result.startsWith("-----BEGIN PRIVATE KEY-----\n")).toBe(true);
+      expect(result.endsWith("-----END PRIVATE KEY-----\n")).toBe(true);
+   });
+});