@malloy-publisher/server 0.0.192 → 0.0.194

package/src/service/connection_service.spec.ts

@@ -611,6 +611,56 @@ describe("service/connection_service", () => {
          ).toBe(true);
       });
 
+      it("should defer DuckDB file cleanup until the previous config release callback", async () => {
+         const connectionName = "local-file";
+         const duckdbConnection: ApiConnection = {
+            name: connectionName,
+            type: "duckdb",
+            duckdbConnection: {
+               attachedDatabases: [],
+            },
+         };
+         const mockDbConnection = {
+            id: "conn-123",
+            name: duckdbConnection.name,
+            type: "duckdb",
+            config: duckdbConnection,
+         };
+         const mockProject = {
+            listApiConnections: sinon.stub().returns([duckdbConnection]),
+            getApiConnection: sinon.stub().returns(duckdbConnection),
+            updateConnections: sinon.stub(),
+            deleteDuckDBConnection: sinon.stub().resolves(),
+            metadata: { location: "/test/path" },
+         };
+         const getConnectionStub = sinon.stub(
+            connectionService,
+            "getConnection",
+         );
+         getConnectionStub.resolves({
+            dbConnection: mockDbConnection,
+            repository: mockRepository,
+         } as unknown as Awaited<
+            ReturnType<typeof connectionService.getConnection>
+         >);
+         (mockProjectStore.getProject as sinon.SinonStub).resolves(mockProject);
+
+         await connectionService.deleteConnection(
+            "test-project",
+            connectionName,
+         );
+
+         expect(mockProject.deleteDuckDBConnection.called).toBe(false);
+         const releaseCallback = mockProject.updateConnections.getCall(0)
+            .args[2] as () => Promise<void>;
+
+         expect(typeof releaseCallback).toBe("function");
+         await releaseCallback();
+         expect(
+            mockProject.deleteDuckDBConnection.calledWith(connectionName),
+         ).toBe(true);
+      });
+
       it("should throw FrozenConfigError when config is frozen", async () => {
          mockProjectStore.publisherConfigIsFrozen = true;
 

package/src/service/connection_service.ts

@@ -1,10 +1,66 @@
 import { components } from "../api";
 import { ConnectionNotFoundError, FrozenConfigError } from "../errors";
 import { logger } from "../logger";
-import { createProjectConnections } from "./connection";
+import { buildProjectMalloyConfig } from "./connection";
 import { ProjectStore } from "./project_store";
 
 type ApiConnection = components["schemas"]["Connection"];
+type ReleaseCallback = () => Promise<void>;
+type ConnectionUpdateProject = {
+   runConnectionUpdateExclusive?: <T>(fn: () => Promise<T>) => Promise<T>;
+   updateConnections?: (
+      nextMalloyConfig: ReturnType<typeof buildProjectMalloyConfig>,
+      apiConnections?: ApiConnection[],
+      afterPreviousRelease?: ReleaseCallback,
+   ) => void;
+   deleteConnection?: (connectionName: string) => Promise<void>;
+   deleteDuckDBConnection?: (connectionName: string) => Promise<void>;
+   deleteDuckLakeConnection?: (connectionName: string) => Promise<void>;
+};
+
+async function runProjectConnectionUpdate<T>(
+   project: ConnectionUpdateProject,
+   fn: () => Promise<T>,
+): Promise<T> {
+   if (project.runConnectionUpdateExclusive) {
+      return project.runConnectionUpdateExclusive(fn);
+   }
+   return fn();
+}
+
+function updateProjectConnections(
+   project: ConnectionUpdateProject,
+   nextMalloyConfig: ReturnType<typeof buildProjectMalloyConfig>,
+   afterPreviousRelease?: ReleaseCallback,
+): void {
+   project.updateConnections?.(
+      nextMalloyConfig,
+      nextMalloyConfig.apiConnections,
+      afterPreviousRelease,
+   );
+}
+
+function buildDeletedConnectionCleanup(
+   project: ConnectionUpdateProject,
+   deletedConnection: ApiConnection,
+   connectionName: string,
+): ReleaseCallback | undefined {
+   if (
+      deletedConnection.type === "duckdb" &&
+      typeof project.deleteDuckDBConnection === "function"
+   ) {
+      return () => project.deleteDuckDBConnection!(connectionName);
+   }
+
+   if (
+      deletedConnection.type === "ducklake" &&
+      typeof project.deleteDuckLakeConnection === "function"
+   ) {
+      return () => project.deleteDuckLakeConnection!(connectionName);
+   }
+
+   return undefined;
+}
 
 export class ConnectionService {
    private projectStore: ProjectStore;
@@ -74,21 +130,21 @@ export class ConnectionService {
 
       // Update in-memory connections
       const project = await this.projectStore.getProject(projectName, false);
-      const existingConnections = project.listApiConnections();
-
-      const { malloyConnections, apiConnections } =
-         await createProjectConnections(
+      await runProjectConnectionUpdate(project, async () => {
+         const existingConnections = project.listApiConnections();
+         const nextMalloyConfig = buildProjectMalloyConfig(
             [...existingConnections, connection],
             project.metadata.location || "",
          );
 
-      project.updateConnections(malloyConnections, apiConnections);
+         await this.projectStore.addConnection(
+            connection,
+            dbProject.id,
+            repository,
+         );
 
-      await this.projectStore.addConnection(
-         connection,
-         dbProject.id,
-         repository,
-      );
+         updateProjectConnections(project, nextMalloyConfig);
+      });
 
       logger.info(
          `Successfully added connection "${connection.name}" to project "${projectName}"`,
@@ -117,31 +173,36 @@ export class ConnectionService {
 
       // Update in-memory connections
       const project = await this.projectStore.getProject(projectName, false);
-      const existingConnections = project.listApiConnections();
+      await runProjectConnectionUpdate(project, async () => {
+         const existingConnections = project.listApiConnections();
 
-      const updatedConnection = {
-         ...dbConnection.config,
-         ...connection,
-         name: connectionName,
-      };
+         const updatedConnection = {
+            ...dbConnection.config,
+            ...connection,
+            name: connectionName,
+         };
 
-      const updatedConnections = existingConnections.map((conn) =>
-         conn.name === connectionName ? updatedConnection : conn,
-      );
+         const updatedConnections = existingConnections.map((conn) =>
+            conn.name === connectionName ? updatedConnection : conn,
+         );
 
-      const { malloyConnections, apiConnections } =
-         await createProjectConnections(
+         // Pass isUpdateConnectionRequest=true so the DuckLake wrapper
+         // re-attaches against the updated catalog/storage settings instead
+         // of trusting the prior generation's persisted attach state.
+         const nextMalloyConfig = buildProjectMalloyConfig(
             updatedConnections,
             project.metadata.location || "",
+            true,
          );
 
-      project.updateConnections(malloyConnections, apiConnections);
+         await this.projectStore.updateConnection(
+            updatedConnection,
+            dbProject.id,
+            repository,
+         );
 
-      await this.projectStore.updateConnection(
-         updatedConnection,
-         dbProject.id,
-         repository,
-      );
+         updateProjectConnections(project, nextMalloyConfig);
+      });
 
       logger.info(
          `Successfully updated connection "${connectionName}" in project "${projectName}"`,
@@ -169,10 +230,42 @@ export class ConnectionService {
 
       // Update in-memory connections
       const project = await this.projectStore.getProject(projectName, false);
-      await project.deleteConnection(connectionName);
-
-      // Delete from database
-      await repository.deleteConnection(dbConnection.id);
+      await runProjectConnectionUpdate(project, async () => {
+         if (typeof project.listApiConnections !== "function") {
+            if (typeof project.deleteConnection === "function") {
+               await project.deleteConnection(connectionName);
+            }
+            await repository.deleteConnection(dbConnection.id);
+            return;
+         }
+
+         const deletedConnection =
+            "getApiConnection" in project &&
+            typeof project.getApiConnection === "function"
+               ? project.getApiConnection(connectionName)
+               : dbConnection.config;
+         const updatedConnections = project
+            .listApiConnections()
+            .filter((connection) => connection.name !== connectionName);
+         const nextMalloyConfig = buildProjectMalloyConfig(
+            updatedConnections,
+            project.metadata.location || "",
+         );
+         const deleteConnectionFilesAfterRelease =
+            buildDeletedConnectionCleanup(
+               project,
+               deletedConnection,
+               connectionName,
+            );
+
+         await repository.deleteConnection(dbConnection.id);
+
+         updateProjectConnections(
+            project,
+            nextMalloyConfig,
+            deleteConnectionFilesAfterRelease,
+         );
+      });
 
       logger.info(
          `Successfully deleted connection "${connectionName}" from project "${projectName}"`,

package/src/service/db_utils.spec.ts

@@ -262,6 +262,61 @@ describe("listTablesForSchema", () => {
       });
    });
 
+   describe("databricks", () => {
+      it("uses defaultCatalog-prefixed information_schema.columns", async () => {
+         const conn: ApiConnection = {
+            name: "test",
+            type: "databricks",
+            databricksConnection: {
+               host: "dbc.cloud.databricks.com",
+               path: "/sql/1.0/warehouses/abc",
+               token: "dapi",
+               defaultCatalog: "main",
+            },
+         };
+         const m = mockConnection(columnRows);
+         const tables = await listTablesForSchema(conn, "default", m.conn);
+
+         expect(m.lastSQL).toContain("main.information_schema.columns");
+         expect(m.lastSQL).toContain("table_schema = 'default'");
+         expect(tables[0].resource).toBe("main.default.orders");
+      });
+
+      it("extracts catalog from schemaName when no defaultCatalog", async () => {
+         const conn: ApiConnection = {
+            name: "test",
+            type: "databricks",
+            databricksConnection: {
+               host: "dbc.cloud.databricks.com",
+               path: "/sql/1.0/warehouses/abc",
+               token: "dapi",
+            },
+         };
+         const m = mockConnection(columnRows);
+         const tables = await listTablesForSchema(conn, "main.default", m.conn);
+
+         expect(m.lastSQL).toContain("main.information_schema.columns");
+         expect(m.lastSQL).toContain("table_schema = 'default'");
+         expect(tables[0].resource).toBe("main.default.orders");
+      });
+
+      it("includes IN filter when tableNames provided", async () => {
+         const conn: ApiConnection = {
+            name: "test",
+            type: "databricks",
+            databricksConnection: {
+               host: "dbc.cloud.databricks.com",
+               path: "/sql/1.0/warehouses/abc",
+               token: "dapi",
+               defaultCatalog: "main",
+            },
+         };
+         const m = mockConnection(columnRows);
+         await listTablesForSchema(conn, "default", m.conn, ["orders"]);
+         expect(m.lastSQL).toContain("table_name IN ('orders')");
+      });
+   });
+
    describe("duckdb", () => {
       const conn: ApiConnection = {
          name: "test",
@@ -674,6 +729,112 @@ describe("getSchemasForConnection", () => {
       });
    });
 
+   describe("databricks", () => {
+      it("queries catalog.information_schema.schemata when defaultCatalog is set", async () => {
+         const conn: ApiConnection = {
+            name: "test",
+            type: "databricks",
+            databricksConnection: {
+               host: "dbc.cloud.databricks.com",
+               path: "/sql/1.0/warehouses/abc",
+               token: "dapi",
+               defaultCatalog: "main",
+               defaultSchema: "default",
+            },
+         };
+         const rows = [
+            { schema_name: "default" },
+            { schema_name: "information_schema" },
+         ];
+         const m = mockConnection(rows);
+         const schemas = await getSchemasForConnection(conn, m.conn);
+
+         expect(m.lastSQL).toContain("main.information_schema.schemata");
+         expect(schemas).toHaveLength(2);
+         expect(schemas.find((s) => s.name === "default")?.isDefault).toBe(
+            true,
+         );
+         expect(
+            schemas.find((s) => s.name === "information_schema")?.isHidden,
+         ).toBe(true);
+      });
+
+      it("falls back to SHOW CATALOGS when defaultCatalog is unset", async () => {
+         const conn: ApiConnection = {
+            name: "test",
+            type: "databricks",
+            databricksConnection: {
+               host: "dbc.cloud.databricks.com",
+               path: "/sql/1.0/warehouses/abc",
+               token: "dapi",
+            },
+         };
+         // First runSQL returns catalog list; subsequent runSQL calls (one
+         // per catalog) return schema rows. We use a dedicated mock so we
+         // can switch behavior across calls.
+         let callIndex = 0;
+         const calls: string[] = [];
+         const fakeConn = {
+            runSQL: async (sql: string) => {
+               calls.push(sql);
+               if (callIndex++ === 0) {
+                  return {
+                     rows: [{ catalog: "main" }, { catalog: "samples" }],
+                  };
+               }
+               return { rows: [{ schema_name: "default" }] };
+            },
+         } as unknown as Connection;
+
+         const schemas = await getSchemasForConnection(conn, fakeConn);
+
+         expect(calls[0]).toContain("SHOW CATALOGS");
+         expect(
+            calls.some((c) => c.includes("main.information_schema.schemata")),
+         ).toBe(true);
+         expect(
+            calls.some((c) =>
+               c.includes("samples.information_schema.schemata"),
+            ),
+         ).toBe(true);
+         // Two catalogs each contribute one schema → catalog-qualified names.
+         expect(schemas.map((s) => s.name)).toEqual([
+            "main.default",
+            "samples.default",
+         ]);
+      });
+
+      it("warns and continues when a catalog rejects information_schema", async () => {
+         const conn: ApiConnection = {
+            name: "test",
+            type: "databricks",
+            databricksConnection: {
+               host: "dbc.cloud.databricks.com",
+               path: "/sql/1.0/warehouses/abc",
+               token: "dapi",
+            },
+         };
+         let callIndex = 0;
+         const fakeConn = {
+            runSQL: async (sql: string) => {
+               if (callIndex++ === 0) {
+                  return { rows: [{ catalog: "denied" }, { catalog: "ok" }] };
+               }
+               if (sql.includes("denied")) {
+                  throw new Error("USE CATALOG denied");
+               }
+               return { rows: [{ schema_name: "default" }] };
+            },
+         } as unknown as Connection;
+
+         const schemas = await getSchemasForConnection(conn, fakeConn);
+
+         // Denied catalog is skipped, ok catalog contributes its schema.
+         expect(schemas).toHaveLength(1);
+         expect(schemas[0].name).toBe("ok.default");
+      });
+   });
+
    it("throws for unsupported connection type", async () => {
       const conn = {
          name: "test",

package/src/service/db_utils.ts

@@ -353,6 +353,82 @@ async function getSchemasForTrino(
    }
 }
 
+async function getSchemasForDatabricks(
+   connection: ApiConnection,
+   malloyConnection: Connection,
+): Promise<ApiSchema[]> {
+   if (!connection.databricksConnection) {
+      throw new Error("Databricks connection is required");
+   }
+   try {
+      const configuredSchema = connection.databricksConnection.defaultSchema;
+      let allRows: { catalog: string; schema: string }[] = [];
+
+      if (connection.databricksConnection.defaultCatalog) {
+         const catalog = connection.databricksConnection.defaultCatalog;
+         const result = await malloyConnection.runSQL(
+            `SELECT schema_name FROM ${catalog}.information_schema.schemata ORDER BY schema_name`,
+         );
+         const rows = standardizeRunSQLResult(result);
+         allRows = rows.map((row: unknown) => {
+            const r = row as Record<string, unknown>;
+            return {
+               catalog,
+               schema: String(r.schema_name ?? r.Schema ?? ""),
+            };
+         });
+      } else {
+         const catalogsResult = await malloyConnection.runSQL(`SHOW CATALOGS`);
+         const catalogNames = standardizeRunSQLResult(catalogsResult).map(
+            (row: unknown) => {
+               const r = row as Record<string, unknown>;
+               return String(r.catalog ?? r.Catalog ?? r.catalog_name ?? "");
+            },
+         );
+
+         for (const catalog of catalogNames) {
+            try {
+               const result = await malloyConnection.runSQL(
+                  `SELECT schema_name FROM ${catalog}.information_schema.schemata ORDER BY schema_name`,
+               );
+               const rows = standardizeRunSQLResult(result);
+               for (const row of rows) {
+                  const r = row as Record<string, unknown>;
+                  allRows.push({
+                     catalog,
+                     schema: String(r.schema_name ?? r.Schema ?? ""),
+                  });
+               }
+            } catch (catalogError) {
+               logger.warn(
+                  `Failed to list schemas for Databricks catalog ${catalog}`,
+                  { error: catalogError },
+               );
+            }
+         }
+      }
+      logger.info("allRows for Schemas for Databricks", { allRows });
+      return allRows.map(({ catalog, schema }) => {
+         const name = connection.databricksConnection?.defaultCatalog
+            ? schema
+            : `${catalog}.${schema}`;
+         return {
+            name,
+            isHidden: ["information_schema"].includes(schema),
+            isDefault: configuredSchema ? schema === configuredSchema : false,
+         };
+      });
+   } catch (error) {
+      logger.error(
+         `Error getting schemas for Databricks connection ${connection.name}`,
+         { error },
+      );
+      throw new Error(
+         `Failed to get schemas for Databricks connection ${connection.name}: ${(error as Error).message}`,
+      );
+   }
+}
+
 async function getSchemasForDuckDB(
    connection: ApiConnection,
    malloyConnection: Connection,
@@ -533,6 +609,8 @@ export async function getSchemasForConnection(
          return getSchemasForSnowflake(connection, malloyConnection);
       case "trino":
          return getSchemasForTrino(connection, malloyConnection);
+      case "databricks":
+         return getSchemasForDatabricks(connection, malloyConnection);
       case "duckdb":
          return getSchemasForDuckDB(connection, malloyConnection);
       case "motherduck":
@@ -823,6 +901,13 @@ export async function listTablesForSchema(
             malloyConnection,
             tableNames,
          );
+      case "databricks":
+         return listTablesForDatabricks(
+            connection,
+            schemaName,
+            malloyConnection,
+            tableNames,
+         );
       case "duckdb":
          return listTablesForDuckDB(
             connection,
@@ -1057,6 +1142,52 @@ async function listTablesForTrino(
    }
 }
 
+async function listTablesForDatabricks(
+   connection: ApiConnection,
+   schemaName: string,
+   malloyConnection: Connection,
+   tableNames?: string[],
+): Promise<ApiTable[]> {
+   if (!connection.databricksConnection) {
+      throw new Error("Databricks connection is required");
+   }
+   try {
+      let catalogPrefix: string;
+      let schemaOnly: string;
+      let resourcePrefix: string;
+
+      if (connection.databricksConnection.defaultCatalog) {
+         catalogPrefix = `${connection.databricksConnection.defaultCatalog}.`;
+         schemaOnly = schemaName;
+         resourcePrefix = `${connection.databricksConnection.defaultCatalog}.${schemaName}`;
+      } else {
+         const dotIdx = schemaName.indexOf(".");
+         if (dotIdx > 0) {
+            catalogPrefix = `${schemaName.substring(0, dotIdx)}.`;
+            schemaOnly = schemaName.substring(dotIdx + 1);
+         } else {
+            catalogPrefix = "";
+            schemaOnly = schemaName;
+         }
+         resourcePrefix = schemaName;
+      }
+
+      const result = await malloyConnection.runSQL(
+         `SELECT table_name, column_name, data_type FROM ${catalogPrefix}information_schema.columns WHERE table_schema = '${schemaOnly}' ${sqlInFilter("table_name", tableNames)} ORDER BY table_name, ordinal_position`,
+      );
+      const rows = standardizeRunSQLResult(result);
+      return groupColumnRowsIntoTables(rows, (t) => `${resourcePrefix}.${t}`);
+   } catch (error) {
+      logger.error(
+         `Error getting tables for Databricks schema ${schemaName} in connection ${connection.name}`,
+         { error },
+      );
+      throw new Error(
+         `Failed to get tables for Databricks schema ${schemaName} in connection ${connection.name}: ${(error as Error).message}`,
+      );
+   }
+}
+
 async function listTablesForDuckDB(
    connection: ApiConnection,
    schemaName: string,

package/src/service/materialization_service.spec.ts

@@ -480,10 +480,12 @@ describe("MaterializationService", () => {
             dialectName: "duckdb",
             runSQL,
          } as unknown as Connection;
-         const connections = new Map<string, Connection>([
-            ["conn", connection],
-         ]);
-         const pkg = { getConnections: () => connections };
+         const pkg = {
+            getMalloyConnection: async (name: string): Promise<Connection> => {
+               if (name === "conn") return connection;
+               throw new Error(`unknown connection: ${name}`);
+            },
+         };
          (ctx.projectStore.getProject as sinon.SinonStub).resolves({
             getPackage: sinon.stub().resolves(pkg),
          });
@@ -535,10 +537,12 @@ describe("MaterializationService", () => {
          // a vanished "ghost_conn", which used to be impossible to tear down.
          // `teardownPackage` must force-delete the row anyway so teardown
          // can complete.
-         const connections = new Map<string, Connection>([
-            ["live_conn", livingConn],
-         ]);
-         const pkg = { getConnections: () => connections };
+         const pkg = {
+            getMalloyConnection: async (name: string): Promise<Connection> => {
+               if (name === "live_conn") return livingConn;
+               throw new Error(`unknown connection: ${name}`);
+            },
+         };
          (ctx.projectStore.getProject as sinon.SinonStub).resolves({
             getPackage: sinon.stub().resolves(pkg),
          });
@@ -576,10 +580,12 @@ describe("MaterializationService", () => {
             dialectName: "duckdb",
             runSQL,
          } as unknown as Connection;
-         const connections = new Map<string, Connection>([
-            ["conn", connection],
-         ]);
-         const pkg = { getConnections: () => connections };
+         const pkg = {
+            getMalloyConnection: async (name: string): Promise<Connection> => {
+               if (name === "conn") return connection;
+               throw new Error(`unknown connection: ${name}`);
+            },
+         };
          (ctx.projectStore.getProject as sinon.SinonStub).resolves({
             getPackage: sinon.stub().resolves(pkg),
          });