@malloy-publisher/server 0.0.192 → 0.0.193

package/src/service/package.ts

@@ -2,10 +2,14 @@ import * as fs from "fs/promises";
 import * as path from "path";
 
 import { DuckDBConnection } from "@malloydata/db-duckdb";
+import "@malloydata/db-duckdb/native";
 import {
    Connection,
    ConnectionRuntime,
    EmptyURLReader,
+   FixedConnectionMap,
+   contextOverlay,
+   MalloyConfig,
    SourceDef,
 } from "@malloydata/malloy";
 import { metrics } from "@opentelemetry/api";
@@ -28,6 +32,14 @@ type ApiNotebook = components["schemas"]["Notebook"];
 export type ApiPackage = components["schemas"]["Package"];
 type ApiColumn = components["schemas"]["Column"];
 type ApiTableDescription = components["schemas"]["TableDescription"];
+// A thunk lets callers pass a live reference to the *current* project
+// MalloyConfig so the package wrapper resolves project connections against the
+// generation that's active at lookup time, not the one that was current when
+// the package was first loaded.
+type PackageConnectionInput =
+   | MalloyConfig
+   | Map<string, Connection>
+   | (() => MalloyConfig);
 
 const ENABLE_LIST_MODEL_COMPILATION = true;
 export class Package {
@@ -37,7 +49,7 @@ export class Package {
    private databases: ApiDatabase[];
    private models: Map<string, Model> = new Map();
    private packagePath: string;
-   private connections: Map<string, Connection> = new Map();
+   private malloyConfig: MalloyConfig;
    private static meter = metrics.getMeter("publisher");
    private static packageLoadHistogram = this.meter.createHistogram(
       "malloy_package_load_duration",
@@ -54,7 +66,7 @@ export class Package {
       packageMetadata: ApiPackage,
       databases: ApiDatabase[],
       models: Map<string, Model>,
-      connections: Map<string, Connection> = new Map(),
+      malloyConfig: MalloyConfig = new MalloyConfig({ connections: {} }),
    ) {
       this.projectName = projectName;
       this.packageName = packageName;
@@ -62,14 +74,14 @@ export class Package {
       this.packageMetadata = packageMetadata;
       this.databases = databases;
       this.models = models;
-      this.connections = connections;
+      this.malloyConfig = malloyConfig;
    }
 
    static async create(
       projectName: string,
       packageName: string,
       packagePath: string,
-      projectConnections: Map<string, Connection>,
+      projectMalloyConfig: PackageConnectionInput,
    ): Promise<Package> {
       const startTime = performance.now();
       await Package.validatePackageManifestExistsOrThrowError(packagePath);
@@ -97,20 +109,17 @@ export class Package {
             databaseCount: databases.length,
             duration: formatDuration(databasesTime - packageConfigTime),
          });
-         const connections = new Map<string, Connection>(projectConnections);
-
-         // Add a duckdb connection for the package.
-         const duckdbConnection = new DuckDBConnection(
-            "duckdb",
-            ":memory:",
+         const malloyConfig = Package.buildPackageMalloyConfig(
             packagePath,
+            typeof projectMalloyConfig === "function"
+               ? projectMalloyConfig
+               : () => Package.toMalloyConfig(projectMalloyConfig),
          );
-         connections.set("duckdb", duckdbConnection);
 
          const models = await Package.loadModels(
             packageName,
             packagePath,
-            connections,
+            malloyConfig,
          );
          const modelsTime = performance.now();
          logger.info("Models loaded", {
@@ -159,7 +168,7 @@ export class Package {
             packageConfig,
             databases,
             models,
-            connections,
+            malloyConfig,
          );
       } catch (error) {
          logger.error(`Error loading package ${packageName}`, { error });
@@ -190,10 +199,6 @@ export class Package {
       return this.packageName;
    }
 
-   public getPackagePath(): string {
-      return this.packagePath;
-   }
-
    public getPackageMetadata(): ApiPackage {
       return this.packageMetadata;
    }
@@ -206,19 +211,24 @@ export class Package {
       return this.models.get(modelPath);
    }
 
+   public async getMalloyConnection(
+      connectionName: string,
+   ): Promise<Connection> {
+      return this.malloyConfig.connections.lookupConnection(connectionName);
+   }
+
+   public getMalloyConfig(): MalloyConfig {
+      return this.malloyConfig;
+   }
+
+   public getPackagePath(): string {
+      return this.packagePath;
+   }
+
    public getModelPaths(): string[] {
       return Array.from(this.models.keys());
    }
 
-   /**
-    * Recompile every model in the package with the given build manifest
-    * so queries resolve persist references to materialized tables.
-    *
-    * Builds a fresh map off to the side and swaps it in at the end. If any
-    * recompile fails the whole call rejects before the swap and the live
-    * `this.models` reference remains untouched — no half-loaded state is
-    * ever observable to concurrent readers.
-    */
    public async reloadAllModels(
       buildManifest: BuildManifest["entries"],
    ): Promise<void> {
@@ -228,14 +238,13 @@ export class Package {
          modelCount: modelPaths.length,
          manifestEntryCount: Object.keys(buildManifest).length,
       });
-
       const reloaded = await Promise.all(
          modelPaths.map((modelPath) =>
             Model.create(
                this.packageName,
                this.packagePath,
                modelPath,
-               this.connections,
+               this.malloyConfig,
                { buildManifest },
             ),
          ),
@@ -247,20 +256,6 @@ export class Package {
       this.models = nextModels;
    }
 
-   public getConnections(): Map<string, Connection> {
-      return this.connections;
-   }
-
-   public getMalloyConnection(connectionName: string): Connection {
-      const connection = this.connections.get(connectionName);
-      if (!connection) {
-         throw new Error(
-            `Connection ${connectionName} not found in package ${this.packageName}`,
-         );
-      }
-      return connection;
-   }
-
    public async getModelFileText(modelPath: string): Promise<string> {
       const model = this.getModel(modelPath);
       if (!model) {
@@ -322,17 +317,64 @@ export class Package {
    private static async loadModels(
       packageName: string,
       packagePath: string,
-      connections: Map<string, Connection>,
+      malloyConfig: MalloyConfig,
    ): Promise<Map<string, Model>> {
       const modelPaths = await Package.getModelPaths(packagePath);
       const models = await Promise.all(
          modelPaths.map((modelPath) =>
-            Model.create(packageName, packagePath, modelPath, connections),
+            Model.create(packageName, packagePath, modelPath, malloyConfig),
          ),
       );
       return new Map(models.map((model) => [model.getPath(), model]));
    }
 
+   private static buildPackageMalloyConfig(
+      packagePath: string,
+      getProjectMalloyConfig: () => MalloyConfig,
+   ): MalloyConfig {
+      const malloyConfig = new MalloyConfig(
+         {
+            connections: {
+               duckdb: {
+                  is: "duckdb",
+                  databasePath: ":memory:",
+               },
+            },
+         },
+         {
+            config: contextOverlay({ rootDirectory: packagePath }),
+         },
+      );
+
+      malloyConfig.wrapConnections((base) => ({
+         lookupConnection: async (name?: string) => {
+            if (!name || name === "duckdb") {
+               return base.lookupConnection(name);
+            }
+            // Resolve against the *current* project MalloyConfig so a
+            // connection-generation swap on Project propagates without a
+            // package reload.
+            return getProjectMalloyConfig().connections.lookupConnection(name);
+         },
+      }));
+
+      return malloyConfig;
+   }
+
+   private static toMalloyConfig(
+      input: MalloyConfig | Map<string, Connection>,
+   ): MalloyConfig {
+      if (input instanceof MalloyConfig) {
+         return input;
+      }
+
+      const malloyConfig = new MalloyConfig({ connections: {} });
+      malloyConfig.wrapConnections(
+         () => new FixedConnectionMap(input, "duckdb"),
+      );
+      return malloyConfig;
+   }
+
    private static async getModelPaths(packagePath: string): Promise<string[]> {
       let files = undefined;
       try {

package/src/service/project.ts

@@ -1,6 +1,5 @@
 import type { LogMessage } from "@malloydata/malloy";
-import { FixedConnectionMap, MalloyError, Runtime } from "@malloydata/malloy";
-import { BaseConnection } from "@malloydata/malloy/connection";
+import { MalloyError, Runtime } from "@malloydata/malloy";
 import { Mutex } from "async-mutex";
 import * as fs from "fs";
 import * as path from "path";
@@ -14,9 +13,10 @@ import {
 import { logger } from "../logger";
 import { URL_READER } from "../utils";
 import {
-   createProjectConnections,
+   buildProjectMalloyConfig,
    deleteDuckLakeConnectionFile,
    InternalConnection,
+   ProjectMalloyConfig,
 } from "./connection";
 import { ApiConnection } from "./model";
 import { Package } from "./package";
@@ -35,12 +35,27 @@ interface PackageInfo {
 
 type ApiPackage = components["schemas"]["Package"];
 type ApiProject = components["schemas"]["Project"];
+type RetiredConnectionGeneration = {
+   label: string;
+   releaseConnections: () => Promise<void>;
+   timer?: ReturnType<typeof setTimeout>;
+};
+
+const RETIRED_CONNECTION_DRAIN_MS = 30_000;
 
 export class Project {
    private packages: Map<string, Package> = new Map();
+   // Lock ordering: connectionMutex (project) MUST be acquired before any
+   // packageMutex. Connection updates may invalidate cached package
+   // MalloyConfigs and force reloads, so the project lock is the outer one.
+   // Never acquire connectionMutex while holding a packageMutex — that's the
+   // AB/BA deadlock path.
    private packageMutexes = new Map<string, Mutex>();
    private packageStatuses: Map<string, PackageInfo> = new Map();
-   private malloyConnections: Map<string, BaseConnection>;
+   private malloyConfig: ProjectMalloyConfig;
+   private connectionMutex = new Mutex();
+   private retiredConnectionGenerations =
+      new Set<RetiredConnectionGeneration>();
    private apiConnections: ApiConnection[];
    private projectPath: string;
    private projectName: string;
@@ -49,12 +64,12 @@ export class Project {
    constructor(
       projectName: string,
       projectPath: string,
-      malloyConnections: Map<string, BaseConnection>,
+      malloyConfig: ProjectMalloyConfig,
       apiConnections: InternalConnection[],
    ) {
       this.projectName = projectName;
       this.projectPath = projectPath;
-      this.malloyConnections = malloyConnections;
+      this.malloyConfig = malloyConfig;
       this.apiConnections = apiConnections;
       this.metadata = {
          resource: `${API_PREFIX}/projects/${this.projectName}`,
@@ -87,30 +102,28 @@ export class Project {
       // Handle connections update
       // TODO: Update project connections should have its own API endpoint
       if (payload.connections) {
-         logger.info(
-            `Updating ${payload.connections.length} connections for project ${this.projectName}`,
-         );
-         const isUpdateConnectionRequest = true;
-         // Reload connections with full config
-         const { malloyConnections, apiConnections } =
-            await createProjectConnections(
-               payload.connections,
+         const payloadConnections = payload.connections;
+         await this.runConnectionUpdateExclusive(async () => {
+            logger.info(
+               `Updating ${payloadConnections.length} connections for project ${this.projectName}`,
+            );
+            const isUpdateConnectionRequest = true;
+            const nextMalloyConfig = buildProjectMalloyConfig(
+               payloadConnections,
                this.projectPath,
                isUpdateConnectionRequest,
             );
 
-         // Update the project's connection maps
-         this.malloyConnections = malloyConnections;
-         this.apiConnections = apiConnections;
+            this.updateConnections(nextMalloyConfig);
 
-         logger.info(
-            `Successfully updated connections for project ${this.projectName}`,
-            {
-               malloyConnections: malloyConnections.size,
-               apiConnections: apiConnections.length,
-               internalConnections: apiConnections.length,
-            },
-         );
+            logger.info(
+               `Successfully updated connections for project ${this.projectName}`,
+               {
+                  apiConnections: this.apiConnections.length,
+                  internalConnections: this.apiConnections.length,
+               },
+            );
+         });
       }
 
       return this;
@@ -128,22 +141,20 @@ export class Project {
       }
 
       logger.info(`Creating project with connection configuration`);
-      const { malloyConnections, apiConnections } =
-         await createProjectConnections(connections, projectPath);
+      const malloyConfig = buildProjectMalloyConfig(connections, projectPath);
 
       logger.info(
-         `Loaded ${malloyConnections.size + apiConnections.length} connections for project ${projectName}`,
+         `Loaded ${malloyConfig.apiConnections.length} connections for project ${projectName}`,
          {
-            malloyConnections,
-            apiConnections,
+            apiConnections: malloyConfig.apiConnections,
          },
       );
 
       const project = new Project(
          projectName,
          projectPath,
-         malloyConnections,
-         apiConnections,
+         malloyConfig,
+         malloyConfig.apiConnections,
       );
 
       return project;
@@ -203,10 +214,14 @@ export class Project {
          },
       };
 
-      // Initialize Runtime with the project's active connections
+      const pkg = await this.getPackage(packageName);
+
+      // Initialize Runtime with the package's active MalloyConfig so compile
+      // checks see the same package-scoped duckdb as execution. This runtime
+      // borrows the package config; the package/project lifecycle owns release.
       const runtime = new Runtime({
          urlReader: interceptingReader,
-         connections: new FixedConnectionMap(this.malloyConnections, "duckdb"),
+         config: pkg.getMalloyConfig(),
       });
 
       // Attempt to compile
@@ -254,14 +269,66 @@ export class Project {
       return connection;
    }
 
-   public getMalloyConnection(connectionName: string): BaseConnection {
-      const connection = this.malloyConnections.get(connectionName);
-      if (!connection) {
-         throw new ConnectionNotFoundError(
-            `Connection ${connectionName} not found`,
+   public async getMalloyConnection(connectionName: string) {
+      return this.malloyConfig.malloyConfig.connections.lookupConnection(
+         connectionName,
+      );
+   }
+
+   public getProjectMalloyConfig() {
+      return this.malloyConfig.malloyConfig;
+   }
+
+   public async runConnectionUpdateExclusive<T>(
+      fn: () => Promise<T>,
+   ): Promise<T> {
+      return this.connectionMutex.runExclusive(fn);
+   }
+
+   private retireConnectionGeneration(
+      label: string,
+      releaseConnections: () => Promise<void>,
+   ): void {
+      const generation: RetiredConnectionGeneration = {
+         label,
+         releaseConnections,
+      };
+      generation.timer = setTimeout(() => {
+         void this.releaseRetiredConnectionGeneration(generation);
+      }, RETIRED_CONNECTION_DRAIN_MS);
+      (
+         generation.timer as ReturnType<typeof setTimeout> & {
+            unref?: () => void;
+         }
+      ).unref?.();
+      this.retiredConnectionGenerations.add(generation);
+   }
+
+   private async releaseRetiredConnectionGeneration(
+      generation: RetiredConnectionGeneration,
+   ): Promise<void> {
+      if (!this.retiredConnectionGenerations.delete(generation)) return;
+
+      if (generation.timer) {
+         clearTimeout(generation.timer);
+      }
+
+      try {
+         await generation.releaseConnections();
+      } catch (error) {
+         logger.error(
+            `Error releasing retired connection generation ${generation.label}`,
+            { error },
          );
       }
-      return connection;
+   }
+
+   private async releaseAllRetiredConnectionGenerations(): Promise<void> {
+      await Promise.all(
+         [...this.retiredConnectionGenerations].map((generation) =>
+            this.releaseRetiredConnectionGeneration(generation),
+         ),
+      );
    }
 
    public async listPackages(): Promise<ApiPackage[]> {
@@ -356,8 +423,13 @@ export class Project {
                this.projectName,
                packageName,
                packagePath,
-               this.malloyConnections,
+               () => this.malloyConfig.malloyConfig,
             );
+            if (existingPackage !== undefined && reload) {
+               this.retireConnectionGeneration(`package ${packageName}`, () =>
+                  existingPackage.getMalloyConfig().releaseConnections(),
+               );
+            }
             this.packages.set(packageName, _package);
 
             // Set package status to serving
@@ -391,7 +463,7 @@ export class Project {
          `Adding package ${packageName} to project ${this.projectName}`,
          {
             packagePath,
-            malloyConnections: this.malloyConnections,
+            malloyConfig: this.malloyConfig.malloyConfig,
          },
       );
       this.setPackageStatus(packageName, PackageStatus.LOADING);
@@ -402,7 +474,7 @@ export class Project {
                this.projectName,
                packageName,
                packagePath,
-               this.malloyConnections,
+               () => this.malloyConfig.malloyConfig,
             ),
          );
       } catch (error) {
@@ -514,6 +586,8 @@ export class Project {
          this.setPackageStatus(packageName, PackageStatus.UNLOADING);
       }
 
+      await _package.getMalloyConfig().releaseConnections();
+
       try {
          await fs.promises.rm(path.join(this.projectPath, packageName), {
             recursive: true,
@@ -532,33 +606,37 @@ export class Project {
    }
 
    public updateConnections(
-      malloyConnections: Map<string, BaseConnection>,
-      apiConnections: ApiConnection[],
+      malloyConfig: ProjectMalloyConfig,
+      _apiConnections?: ApiConnection[],
+      afterPreviousRelease?: () => Promise<void>,
    ): void {
-      this.malloyConnections = malloyConnections;
-      this.apiConnections = apiConnections;
+      const previousMalloyConfig = this.malloyConfig;
+      this.malloyConfig = malloyConfig;
+      this.apiConnections = malloyConfig.apiConnections;
+
+      if (previousMalloyConfig !== malloyConfig) {
+         this.retireConnectionGeneration(
+            `project ${this.projectName}`,
+            async () => {
+               await previousMalloyConfig.releaseConnections();
+               await afterPreviousRelease?.();
+            },
+         );
+      } else {
+         void afterPreviousRelease?.();
+      }
    }
 
    public async deleteConnection(connectionName: string): Promise<void> {
-      this.malloyConnections.get(connectionName)?.close();
-      const isDeleted = this.malloyConnections.delete(connectionName);
-
       const index = this.apiConnections.findIndex(
          (conn) => conn.name === connectionName,
       );
 
-      const connectionType = this.apiConnections[index]?.type;
-      if (connectionType === "duckdb") {
-         await this.deleteDuckDBConnection(connectionName);
-      } else if (connectionType === "ducklake") {
-         await this.deleteDuckLakeConnection(connectionName);
-      }
-
       if (index !== -1) {
          this.apiConnections.splice(index, 1);
       }
 
-      if (isDeleted || index !== -1) {
+      if (index !== -1) {
          logger.info(
             `Removed connection ${connectionName} from project ${this.projectName}`,
          );
@@ -569,24 +647,36 @@ export class Project {
       }
    }
 
-   public closeAllConnections(): void {
-      // Close all Malloy connections
-      for (const [connectionName, connection] of this.malloyConnections) {
-         try {
-            connection.close();
-            logger.info(
-               `Closed connection ${connectionName} for project ${this.projectName}`,
-            );
-         } catch (error) {
+   public async closeAllConnections(): Promise<void> {
+      // Release the package-scoped MalloyConfigs (each holds the package's own
+      // sandbox `duckdb` connection) before tearing down the project config
+      // they wrap. Without this, hard unload leaks per-package DuckDB handles.
+      const packageReleases = await Promise.allSettled(
+         Array.from(this.packages.values(), (pkg) =>
+            pkg.getMalloyConfig().releaseConnections(),
+         ),
+      );
+      for (const result of packageReleases) {
+         if (result.status === "rejected") {
             logger.error(
-               `Error closing connection ${connectionName} for project ${this.projectName}`,
-               { error },
+               `Error closing package connections for project ${this.projectName}`,
+               { error: result.reason },
             );
          }
       }
+      this.packages.clear();
+      this.packageStatuses.clear();
+
+      try {
+         await this.malloyConfig.releaseConnections();
+      } catch (error) {
+         logger.error(
+            `Error closing connections for project ${this.projectName}`,
+            { error },
+         );
+      }
+      await this.releaseAllRetiredConnectionGenerations();
 
-      // Clear connection maps
-      this.malloyConnections.clear();
       this.apiConnections = [];
 
       logger.info(`Closed all connections for project ${this.projectName}`);
@@ -605,29 +695,17 @@ export class Project {
          this.projectPath,
          `${connectionName}.duckdb`,
       );
-      fs.promises
-         .access(duckdbPath)
-         .then(() => {
-            fs.promises
-               .rm(duckdbPath)
-               .then(() => {
-                  logger.info(
-                     `Removed DuckDB connection file ${connectionName} from project ${this.projectName}`,
-                  );
-               })
-               .catch((error) => {
-                  logger.error(
-                     `Failed to remove DuckDB connection file ${connectionName} from project ${this.projectName}`,
-                     { error },
-                  );
-               });
-         })
-         .catch((error) => {
-            logger.error(
-               `Failed to remove DuckDB connection file ${connectionName} from project ${this.projectName}`,
-               { error },
-            );
-         });
+      try {
+         await fs.promises.rm(duckdbPath, { force: true });
+         logger.info(
+            `Removed DuckDB connection file ${connectionName} from project ${this.projectName}`,
+         );
+      } catch (error) {
+         logger.error(
+            `Failed to remove DuckDB connection file ${connectionName} from project ${this.projectName}`,
+            { error },
+         );
+      }
    }
 
    public async deleteDuckLakeConnection(