@malloy-publisher/server 0.0.192 → 0.0.194

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.192",
+   "version": "0.0.194",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"
@@ -34,15 +34,16 @@
       "@azure/identity": "^4.13.0",
       "@azure/storage-blob": "^12.26.0",
       "@google-cloud/storage": "^7.16.0",
-      "@malloydata/db-bigquery": "^0.0.377",
-      "@malloydata/db-duckdb": "^0.0.377",
-      "@malloydata/db-mysql": "^0.0.377",
-      "@malloydata/db-postgres": "^0.0.377",
-      "@malloydata/db-snowflake": "^0.0.377",
-      "@malloydata/db-trino": "^0.0.377",
-      "@malloydata/malloy": "^0.0.377",
-      "@malloydata/malloy-sql": "^0.0.377",
-      "@malloydata/render-validator": "^0.0.377",
+      "@malloydata/db-bigquery": "^0.0.383",
+      "@malloydata/db-databricks": "^0.0.383",
+      "@malloydata/db-duckdb": "^0.0.383",
+      "@malloydata/db-mysql": "^0.0.383",
+      "@malloydata/db-postgres": "^0.0.383",
+      "@malloydata/db-snowflake": "^0.0.383",
+      "@malloydata/db-trino": "^0.0.383",
+      "@malloydata/malloy": "^0.0.383",
+      "@malloydata/malloy-sql": "^0.0.383",
+      "@malloydata/render-validator": "^0.0.383",
       "@modelcontextprotocol/sdk": "^1.13.2",
       "@opentelemetry/api": "^1.9.0",
       "@opentelemetry/auto-instrumentations-node": "^0.57.0",

package/src/config.ts

@@ -90,9 +90,14 @@ export const getPublisherConfig = (serverRoot: string): PublisherConfig => {
       const fileContent = fs.readFileSync(publisherConfigPath, "utf8");
       rawConfig = JSON.parse(fileContent);
    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
       logger.error(
-         `Failed to parse ${PUBLISHER_CONFIG_NAME}. Using default empty config.`,
-         { error },
+         `Failed to parse ${publisherConfigPath}: ${message}. Using default empty config.`,
+         {
+            path: publisherConfigPath,
+            error: message,
+            stack: error instanceof Error ? error.stack : undefined,
+         },
       );
       return {
          frozenConfig: false,

package/src/controller/connection.controller.ts

@@ -4,11 +4,13 @@ import { components } from "../api";
 import { BadRequestError, ConnectionError } from "../errors";
 import { logger } from "../logger";
 import { testConnectionConfig } from "../service/connection";
+import { validateDuckdbApiSurface } from "../service/connection_config";
 import { ConnectionService } from "../service/connection_service";
 import {
    getSchemasForConnection,
    listTablesForSchema,
 } from "../service/db_utils";
+import type { Project } from "../service/project";
 import { ProjectStore } from "../service/project_store";
 
 type ApiConnection = components["schemas"]["Connection"];
@@ -83,6 +85,23 @@ function validateAzureAttachedDatabases(connectionConfig: ApiConnection): void {
    }
 }
 
+function validateAdminAuthoredConnection(
+   connectionName: string,
+   connectionConfig: ApiConnection,
+): void {
+   if (connectionName === "duckdb" || connectionConfig.name === "duckdb") {
+      throw new BadRequestError(
+         "DuckDB connection name cannot be 'duckdb'; it is reserved for Publisher package sandboxes.",
+      );
+   }
+
+   try {
+      validateDuckdbApiSurface(connectionConfig);
+   } catch (error) {
+      throw new BadRequestError((error as Error).message);
+   }
+}
+
 export class ConnectionController {
    private projectStore: ProjectStore;
    private connectionService: ConnectionService;
@@ -95,28 +114,62 @@ export class ConnectionController {
     * Gets the appropriate Malloy connection for a given connection name.
     * For DuckDB connections, retrieves from package level; for others, from project level.
     */
+   private getApiConnectionForLookup(
+      project: Project,
+      connectionName: string,
+   ): ApiConnection {
+      if (connectionName === "duckdb") {
+         return {
+            name: "duckdb",
+            type: "duckdb",
+            duckdbConnection: { attachedDatabases: [] },
+         };
+      }
+      return project.getApiConnection(connectionName);
+   }
+
    private async getMalloyConnection(
       projectName: string,
       connectionName: string,
+      packageName?: string,
    ): Promise<Connection> {
       const project = await this.projectStore.getProject(projectName, false);
-      const connection = project.getApiConnection(connectionName);
 
-      // For DuckDB connections, get the connection from a package
-      if (connection.name === "duckdb" && connection.type === "duckdb") {
+      // "duckdb" is the per-package sandbox; its rootDirectory is the
+      // package's directory. There is no project-level "duckdb" — the name is
+      // reserved at config time. So the lookup is intrinsically per-package
+      // and the caller must say which package to use.
+      if (connectionName === "duckdb") {
          const packages = await project.listPackages();
          if (packages.length === 0) {
-            return project.getMalloyConnection(connectionName);
+            // Fall through to project; this will surface the standard
+            // "connection not found" rather than silently inventing one.
+            return await project.getMalloyConnection(connectionName);
+         }
+         if (packageName) {
+            const known = packages.some((p) => p.name === packageName);
+            if (!known) {
+               throw new BadRequestError(
+                  `Package "${packageName}" not found in project "${projectName}"`,
+               );
+            }
+            const pkg = await project.getPackage(packageName);
+            return await pkg.getMalloyConnection(connectionName);
          }
-         // For now, use the first package's DuckDB connection
-         const packageName = packages[0].name;
-         if (!packageName) {
-            throw new ConnectionError("Package name is undefined");
+         if (packages.length === 1) {
+            const onlyPackage = packages[0].name;
+            if (!onlyPackage) {
+               throw new ConnectionError("Package name is undefined");
+            }
+            const pkg = await project.getPackage(onlyPackage);
+            return await pkg.getMalloyConnection(connectionName);
          }
-         const pkg = await project.getPackage(packageName);
-         return pkg.getMalloyConnection(connectionName);
+         throw new BadRequestError(
+            `Ambiguous "duckdb" connection lookup: project "${projectName}" has multiple packages. ` +
+               `Use /projects/${projectName}/packages/{packageName}/connections/duckdb/... to disambiguate.`,
+         );
       } else {
-         return project.getMalloyConnection(connectionName);
+         return await project.getMalloyConnection(connectionName);
       }
    }
 
@@ -188,33 +241,46 @@ export class ConnectionController {
       return project.listApiConnections();
    }
 
-   // Lists schemas (namespaces) available in a connection
+   // Lists schemas (namespaces) available in a connection.
+   // For "duckdb", the per-package sandbox, packageName disambiguates which
+   // package's DuckDB to browse in a multi-package project.
    public async listSchemas(
       projectName: string,
       connectionName: string,
+      packageName?: string,
    ): Promise<ApiSchema[]> {
       const project = await this.projectStore.getProject(projectName, false);
-      const connection = project.getApiConnection(connectionName);
+      const connection = this.getApiConnectionForLookup(
+         project,
+         connectionName,
+      );
       const malloyConnection = await this.getMalloyConnection(
          projectName,
          connectionName,
+         packageName,
       );
 
       return getSchemasForConnection(connection, malloyConnection);
    }
 
-   // Lists tables available in a schema. For postgres the schema is usually "public"
+   // Lists tables available in a schema. For postgres the schema is usually "public".
+   // packageName disambiguates per-package "duckdb" lookups (see listSchemas).
    public async listTables(
       projectName: string,
       connectionName: string,
       schemaName: string,
       tableNames?: string[],
+      packageName?: string,
    ): Promise<ApiTable[]> {
       const project = await this.projectStore.getProject(projectName, false);
-      const connection = project.getApiConnection(connectionName);
+      const connection = this.getApiConnectionForLookup(
+         project,
+         connectionName,
+      );
       const malloyConnection = await this.getMalloyConnection(
          projectName,
          connectionName,
+         packageName,
       );
 
       return listTablesForSchema(
@@ -229,10 +295,12 @@ export class ConnectionController {
       projectName: string,
       connectionName: string,
       sqlStatement: string,
+      packageName?: string,
    ): Promise<ApiSqlSource> {
       const malloyConnection = await this.getMalloyConnection(
          projectName,
          connectionName,
+         packageName,
       );
       try {
          const schema = await (
@@ -265,14 +333,19 @@ export class ConnectionController {
       connectionName: string,
       schemaName: string,
       tablePath: string,
+      packageName?: string,
    ): Promise<ApiTable> {
       const malloyConnection = await this.getMalloyConnection(
          projectName,
          connectionName,
+         packageName,
       );
       // Use getApiConnection to get the unwrapped ApiConnection config, consistent with listSchemas and listTables.
       const project = await this.projectStore.getProject(projectName, false);
-      const connection = project.getApiConnection(connectionName);
+      const connection = this.getApiConnectionForLookup(
+         project,
+         connectionName,
+      );
 
       // TODO: Move this database connection logic to the db_utils.ts file -- and
       // ultimately into a connection-specific class.
@@ -346,10 +419,12 @@ export class ConnectionController {
       connectionName: string,
       sqlStatement: string,
       options: string,
+      packageName?: string,
    ): Promise<ApiQueryData> {
       const malloyConnection = await this.getMalloyConnection(
          projectName,
          connectionName,
+         packageName,
       );
 
       let runSQLOptions: RunSQLOptions = {};
@@ -377,10 +452,12 @@ export class ConnectionController {
       projectName: string,
       connectionName: string,
       sqlStatement: string,
+      packageName?: string,
    ): Promise<ApiTemporaryTable> {
       const malloyConnection = await this.getMalloyConnection(
          projectName,
          connectionName,
+         packageName,
       );
 
       try {
@@ -397,17 +474,13 @@ export class ConnectionController {
    }
 
    public async testConnectionConfiguration(
-      connectionConfig: ApiConnection,
+      input: ApiConnection & { config?: ApiConnection },
    ): Promise<ApiConnectionStatus> {
-      if (
-         connectionConfig &&
-         "config" in connectionConfig &&
-         typeof (connectionConfig as Record<string, unknown>).config ===
-            "object"
-      ) {
-         connectionConfig = (connectionConfig as Record<string, unknown>)
-            .config as ApiConnection;
-      }
+      // Some clients wrap the payload as { config: <connection> }; unwrap.
+      const connectionConfig: ApiConnection =
+         input.config && typeof input.config === "object"
+            ? input.config
+            : input;
 
       if (
          !connectionConfig ||
@@ -453,6 +526,7 @@ export class ConnectionController {
       }
 
       validateAzureAttachedDatabases(connectionConfig);
+      validateAdminAuthoredConnection(connectionName, connectionConfig);
 
       logger.info(
          `Creating connection "${connectionName}" in project "${projectName}"`,
@@ -478,7 +552,8 @@ export class ConnectionController {
          throw new BadRequestError("Connection payload is required");
       }
 
-      validateAzureAttachedDatabases(connection as ApiConnection);
+      validateAzureAttachedDatabases(connection);
+      validateAdminAuthoredConnection(connectionName, connection);
 
       logger.info(
          `Updating connection "${connectionName}" in project "${projectName}"`,

package/src/dto/connection.dto.spec.ts

@@ -5,6 +5,7 @@ import { validate } from "class-validator";
 import {
    BigqueryConnectionDto,
    ConnectionDto,
+   DatabricksConnectionDto,
    PostgresConnectionDto,
    SnowflakeConnectionDto,
    MysqlConnectionDto,
@@ -82,9 +83,13 @@ describe("dto/connection", () => {
             account: "my-account",
             username: "user",
             password: "pass",
+            privateKey:
+               "-----BEGIN PRIVATE KEY-----\\nabc\\n-----END PRIVATE KEY-----",
+            privateKeyPass: "secret",
             warehouse: "my-warehouse",
             database: "my-database",
             schema: "my-schema",
+            role: "analyst",
             responseTimeoutMilliseconds: 5000,
          };
          const snowflakeConnection = plainToInstance(
@@ -96,6 +101,56 @@ describe("dto/connection", () => {
          expect(errors).toHaveLength(0);
       });
 
+      it("should validate a valid DatabricksConnection object", async () => {
+         const validData = {
+            host: "dbc-xxxxxxxx-xxxx.cloud.databricks.com",
+            path: "/sql/1.0/warehouses/abc123",
+            token: "dapiXXXX",
+            oauthClientId: "client-id",
+            oauthClientSecret: "client-secret",
+            defaultCatalog: "main",
+            defaultSchema: "default",
+            setupSQL: "USE CATALOG main",
+         };
+         const databricksConnection = plainToInstance(
+            DatabricksConnectionDto,
+            validData,
+         );
+
+         const errors = await validate(databricksConnection);
+         expect(errors).toHaveLength(0);
+      });
+
+      it("should return errors for invalid DatabricksConnection object", async () => {
+         const invalidData = {
+            host: 123, // Invalid type
+            path: false, // Invalid type
+         };
+         const databricksConnection = plainToInstance(
+            DatabricksConnectionDto,
+            invalidData,
+         );
+
+         const errors = await validate(databricksConnection);
+         expect(errors).toHaveLength(2);
+      });
+
+      it("should validate a valid Connection object with databricks type", async () => {
+         const validData = {
+            name: "My Databricks Connection",
+            type: "databricks",
+            databricksConnection: {
+               host: "dbc-xxxxxxxx-xxxx.cloud.databricks.com",
+               path: "/sql/1.0/warehouses/abc123",
+               token: "dapiXXXX",
+            },
+         };
+         const connection = plainToInstance(ConnectionDto, validData);
+
+         const errors = await validate(connection);
+         expect(errors).toHaveLength(0);
+      });
+
       it("should validate a valid Connection object with postgres type", async () => {
          const validData = {
             name: "My Postgres Connection",

package/src/dto/connection.dto.ts

@@ -1,14 +1,18 @@
 import { Type } from "class-transformer";
 import {
    IsEnum,
+   IsArray,
    IsNumber,
    IsOptional,
    IsString,
    ValidateNested,
 } from "class-validator";
 import "reflect-metadata";
+import { components } from "../api";
 import { ApiConnection } from "../service/model";
 
+type AttachedDatabase = components["schemas"]["AttachedDatabase"];
+
 export class PostgresConnectionDto {
    @IsOptional()
    @IsString()
@@ -96,6 +100,14 @@ export class SnowflakeConnectionDto {
    @IsString()
    password?: string;
 
+   @IsOptional()
+   @IsString()
+   privateKey?: string;
+
+   @IsOptional()
+   @IsString()
+   privateKeyPass?: string;
+
    @IsOptional()
    @IsString()
    warehouse?: string;
@@ -108,11 +120,49 @@ export class SnowflakeConnectionDto {
    @IsString()
    schema?: string;
 
+   @IsOptional()
+   @IsString()
+   role?: string;
+
    @IsOptional()
    @IsNumber()
    responseTimeoutMilliseconds?: number;
 }
 
+export class DatabricksConnectionDto {
+   @IsOptional()
+   @IsString()
+   host?: string;
+
+   @IsOptional()
+   @IsString()
+   path?: string;
+
+   @IsOptional()
+   @IsString()
+   token?: string;
+
+   @IsOptional()
+   @IsString()
+   oauthClientId?: string;
+
+   @IsOptional()
+   @IsString()
+   oauthClientSecret?: string;
+
+   @IsOptional()
+   @IsString()
+   defaultCatalog?: string;
+
+   @IsOptional()
+   @IsString()
+   defaultSchema?: string;
+
+   @IsOptional()
+   @IsString()
+   setupSQL?: string;
+}
+
 export class TrinoConnectionDto {
    @IsOptional()
    @IsString()
@@ -143,14 +193,39 @@ export class TrinoConnectionDto {
    peakaKey?: string;
 }
 
+export class DuckdbConnectionDto {
+   @IsOptional()
+   @IsArray()
+   attachedDatabases?: AttachedDatabase[];
+}
+
 export class ConnectionDto implements ApiConnection {
    @IsOptional()
    @IsString()
    name?: string;
 
    @IsOptional()
-   @IsEnum(["postgres", "bigquery", "snowflake", "trino"])
-   type?: "postgres" | "bigquery" | "snowflake" | "trino";
+   @IsEnum([
+      "postgres",
+      "bigquery",
+      "snowflake",
+      "trino",
+      "databricks",
+      "mysql",
+      "duckdb",
+      "motherduck",
+      "ducklake",
+   ])
+   type?:
+      | "postgres"
+      | "bigquery"
+      | "snowflake"
+      | "trino"
+      | "databricks"
+      | "mysql"
+      | "duckdb"
+      | "motherduck"
+      | "ducklake";
 
    @IsOptional()
    @ValidateNested()
@@ -171,4 +246,14 @@ export class ConnectionDto implements ApiConnection {
    @ValidateNested()
    @Type(() => TrinoConnectionDto)
    TrinoConnection?: TrinoConnectionDto;
+
+   @IsOptional()
+   @ValidateNested()
+   @Type(() => DatabricksConnectionDto)
+   databricksConnection?: DatabricksConnectionDto;
+
+   @IsOptional()
+   @ValidateNested()
+   @Type(() => DuckdbConnectionDto)
+   duckdbConnection?: DuckdbConnectionDto;
 }