@malloy-publisher/server 0.0.192 → 0.0.193

package/src/service/connection_service.ts

@@ -1,10 +1,66 @@
 import { components } from "../api";
 import { ConnectionNotFoundError, FrozenConfigError } from "../errors";
 import { logger } from "../logger";
-import { createProjectConnections } from "./connection";
+import { buildProjectMalloyConfig } from "./connection";
 import { ProjectStore } from "./project_store";
 
 type ApiConnection = components["schemas"]["Connection"];
+type ReleaseCallback = () => Promise<void>;
+type ConnectionUpdateProject = {
+   runConnectionUpdateExclusive?: <T>(fn: () => Promise<T>) => Promise<T>;
+   updateConnections?: (
+      nextMalloyConfig: ReturnType<typeof buildProjectMalloyConfig>,
+      apiConnections?: ApiConnection[],
+      afterPreviousRelease?: ReleaseCallback,
+   ) => void;
+   deleteConnection?: (connectionName: string) => Promise<void>;
+   deleteDuckDBConnection?: (connectionName: string) => Promise<void>;
+   deleteDuckLakeConnection?: (connectionName: string) => Promise<void>;
+};
+
+async function runProjectConnectionUpdate<T>(
+   project: ConnectionUpdateProject,
+   fn: () => Promise<T>,
+): Promise<T> {
+   if (project.runConnectionUpdateExclusive) {
+      return project.runConnectionUpdateExclusive(fn);
+   }
+   return fn();
+}
+
+function updateProjectConnections(
+   project: ConnectionUpdateProject,
+   nextMalloyConfig: ReturnType<typeof buildProjectMalloyConfig>,
+   afterPreviousRelease?: ReleaseCallback,
+): void {
+   project.updateConnections?.(
+      nextMalloyConfig,
+      nextMalloyConfig.apiConnections,
+      afterPreviousRelease,
+   );
+}
+
+function buildDeletedConnectionCleanup(
+   project: ConnectionUpdateProject,
+   deletedConnection: ApiConnection,
+   connectionName: string,
+): ReleaseCallback | undefined {
+   if (
+      deletedConnection.type === "duckdb" &&
+      typeof project.deleteDuckDBConnection === "function"
+   ) {
+      return () => project.deleteDuckDBConnection!(connectionName);
+   }
+
+   if (
+      deletedConnection.type === "ducklake" &&
+      typeof project.deleteDuckLakeConnection === "function"
+   ) {
+      return () => project.deleteDuckLakeConnection!(connectionName);
+   }
+
+   return undefined;
+}
 
 export class ConnectionService {
    private projectStore: ProjectStore;
@@ -74,21 +130,21 @@ export class ConnectionService {
 
       // Update in-memory connections
       const project = await this.projectStore.getProject(projectName, false);
-      const existingConnections = project.listApiConnections();
-
-      const { malloyConnections, apiConnections } =
-         await createProjectConnections(
+      await runProjectConnectionUpdate(project, async () => {
+         const existingConnections = project.listApiConnections();
+         const nextMalloyConfig = buildProjectMalloyConfig(
             [...existingConnections, connection],
             project.metadata.location || "",
          );
 
-      project.updateConnections(malloyConnections, apiConnections);
+         await this.projectStore.addConnection(
+            connection,
+            dbProject.id,
+            repository,
+         );
 
-      await this.projectStore.addConnection(
-         connection,
-         dbProject.id,
-         repository,
-      );
+         updateProjectConnections(project, nextMalloyConfig);
+      });
 
       logger.info(
          `Successfully added connection "${connection.name}" to project "${projectName}"`,
@@ -117,31 +173,36 @@ export class ConnectionService {
 
       // Update in-memory connections
       const project = await this.projectStore.getProject(projectName, false);
-      const existingConnections = project.listApiConnections();
+      await runProjectConnectionUpdate(project, async () => {
+         const existingConnections = project.listApiConnections();
 
-      const updatedConnection = {
-         ...dbConnection.config,
-         ...connection,
-         name: connectionName,
-      };
+         const updatedConnection = {
+            ...dbConnection.config,
+            ...connection,
+            name: connectionName,
+         };
 
-      const updatedConnections = existingConnections.map((conn) =>
-         conn.name === connectionName ? updatedConnection : conn,
-      );
+         const updatedConnections = existingConnections.map((conn) =>
+            conn.name === connectionName ? updatedConnection : conn,
+         );
 
-      const { malloyConnections, apiConnections } =
-         await createProjectConnections(
+         // Pass isUpdateConnectionRequest=true so the DuckLake wrapper
+         // re-attaches against the updated catalog/storage settings instead
+         // of trusting the prior generation's persisted attach state.
+         const nextMalloyConfig = buildProjectMalloyConfig(
             updatedConnections,
             project.metadata.location || "",
+            true,
          );
 
-      project.updateConnections(malloyConnections, apiConnections);
+         await this.projectStore.updateConnection(
+            updatedConnection,
+            dbProject.id,
+            repository,
+         );
 
-      await this.projectStore.updateConnection(
-         updatedConnection,
-         dbProject.id,
-         repository,
-      );
+         updateProjectConnections(project, nextMalloyConfig);
+      });
 
       logger.info(
          `Successfully updated connection "${connectionName}" in project "${projectName}"`,
@@ -169,10 +230,42 @@ export class ConnectionService {
 
       // Update in-memory connections
       const project = await this.projectStore.getProject(projectName, false);
-      await project.deleteConnection(connectionName);
-
-      // Delete from database
-      await repository.deleteConnection(dbConnection.id);
+      await runProjectConnectionUpdate(project, async () => {
+         if (typeof project.listApiConnections !== "function") {
+            if (typeof project.deleteConnection === "function") {
+               await project.deleteConnection(connectionName);
+            }
+            await repository.deleteConnection(dbConnection.id);
+            return;
+         }
+
+         const deletedConnection =
+            "getApiConnection" in project &&
+            typeof project.getApiConnection === "function"
+               ? project.getApiConnection(connectionName)
+               : dbConnection.config;
+         const updatedConnections = project
+            .listApiConnections()
+            .filter((connection) => connection.name !== connectionName);
+         const nextMalloyConfig = buildProjectMalloyConfig(
+            updatedConnections,
+            project.metadata.location || "",
+         );
+         const deleteConnectionFilesAfterRelease =
+            buildDeletedConnectionCleanup(
+               project,
+               deletedConnection,
+               connectionName,
+            );
+
+         await repository.deleteConnection(dbConnection.id);
+
+         updateProjectConnections(
+            project,
+            nextMalloyConfig,
+            deleteConnectionFilesAfterRelease,
+         );
+      });
 
       logger.info(
          `Successfully deleted connection "${connectionName}" from project "${projectName}"`,

package/src/service/materialization_service.spec.ts

@@ -480,10 +480,12 @@ describe("MaterializationService", () => {
             dialectName: "duckdb",
             runSQL,
          } as unknown as Connection;
-         const connections = new Map<string, Connection>([
-            ["conn", connection],
-         ]);
-         const pkg = { getConnections: () => connections };
+         const pkg = {
+            getMalloyConnection: async (name: string): Promise<Connection> => {
+               if (name === "conn") return connection;
+               throw new Error(`unknown connection: ${name}`);
+            },
+         };
          (ctx.projectStore.getProject as sinon.SinonStub).resolves({
             getPackage: sinon.stub().resolves(pkg),
          });
@@ -535,10 +537,12 @@ describe("MaterializationService", () => {
          // a vanished "ghost_conn", which used to be impossible to tear down.
          // `teardownPackage` must force-delete the row anyway so teardown
          // can complete.
-         const connections = new Map<string, Connection>([
-            ["live_conn", livingConn],
-         ]);
-         const pkg = { getConnections: () => connections };
+         const pkg = {
+            getMalloyConnection: async (name: string): Promise<Connection> => {
+               if (name === "live_conn") return livingConn;
+               throw new Error(`unknown connection: ${name}`);
+            },
+         };
          (ctx.projectStore.getProject as sinon.SinonStub).resolves({
             getPackage: sinon.stub().resolves(pkg),
          });
@@ -576,10 +580,12 @@ describe("MaterializationService", () => {
             dialectName: "duckdb",
             runSQL,
          } as unknown as Connection;
-         const connections = new Map<string, Connection>([
-            ["conn", connection],
-         ]);
-         const pkg = { getConnections: () => connections };
+         const pkg = {
+            getMalloyConnection: async (name: string): Promise<Connection> => {
+               if (name === "conn") return connection;
+               throw new Error(`unknown connection: ${name}`);
+            },
+         };
          (ctx.projectStore.getProject as sinon.SinonStub).resolves({
             getPackage: sinon.stub().resolves(pkg),
          });

package/src/service/materialization_service.ts

@@ -1,6 +1,7 @@
 import type {
    BuildGraph,
    Connection as MalloyConnection,
+   MalloyConfig,
    PersistSource,
 } from "@malloydata/malloy";
 import { Manifest } from "@malloydata/malloy";
@@ -46,6 +47,34 @@ export function stagingSuffix(buildId: string): string {
    return `_${buildId.substring(0, STAGING_BUILD_ID_LEN)}`;
 }
 
+/**
+ * Resolve a Map<name, Connection> for just the names a materialization
+ * step is about to touch. The package's MalloyConfig caches each lookup,
+ * so subsequent calls with overlapping names are cheap. A failed lookup
+ * is logged and the name is omitted from the result — downstream code
+ * (`forceDeleteRowOnMissingConnection` in teardown, or the explicit
+ * "connection X not found" check in build) handles a missing entry.
+ */
+async function resolvePackageConnections(
+   pkg: { getMalloyConnection(name: string): Promise<MalloyConnection> },
+   names: Iterable<string>,
+): Promise<Map<string, MalloyConnection>> {
+   const map = new Map<string, MalloyConnection>();
+   const seen = new Set<string>();
+   for (const name of names) {
+      if (!name || seen.has(name)) continue;
+      seen.add(name);
+      try {
+         map.set(name, await pkg.getMalloyConnection(name));
+      } catch (err) {
+         logger.warn(`Failed to resolve connection ${name}`, {
+            error: err instanceof Error ? err.message : String(err),
+         });
+      }
+   }
+   return map;
+}
+
 /**
  * Build a stable key for a `(connectionName, tableName)` pair.
  * Used to check whether a persist target was created by a previous build.
@@ -494,13 +523,17 @@ export class MaterializationService {
 
       const project = await this.projectStore.getProject(projectName, false);
       const pkg = await project.getPackage(packageName, false);
-      const connections = pkg.getConnections();
 
       const entries = await this.manifestService.listEntries(
          projectId,
          packageName,
       );
 
+      const connections = await resolvePackageConnections(
+         pkg,
+         entries.map((e) => e.connectionName),
+      );
+
       // `forceDeleteRowOnMissingConnection`: teardown is the one place
       // where we'd rather lose the manifest row than leave it pointing at
       // a vanished connection. We also deliberately omit `liveTables`:
@@ -560,7 +593,9 @@ export class MaterializationService {
       );
 
       // ── STEP 2: COMPILE & PLAN ─────────────────────────────────────
-      const { graphs, sources, connectionDigests } =
+      // `connections` is built lazily from the connection names the plan
+      // actually targets — no upfront ATTACH on every project connection.
+      const { graphs, sources, connectionDigests, connections } =
          await this.compilePackageBuildPlan(pkg, signal);
 
       if (graphs.length === 0) {
@@ -569,7 +604,6 @@ export class MaterializationService {
       }
 
       // ── STEP 3: BUILD ──────────────────────────────────────────────
-      const connections = pkg.getConnections();
       let sourcesBuilt = 0;
       let sourcesSkipped = 0;
       const sourceResults: Record<string, unknown>[] = [];
@@ -646,13 +680,15 @@ export class MaterializationService {
       pkg: {
          getModelPaths(): string[];
          getPackagePath(): string;
-         getConnections(): Map<string, MalloyConnection>;
+         getMalloyConfig(): MalloyConfig;
+         getMalloyConnection(name: string): Promise<MalloyConnection>;
       },
       signal: AbortSignal,
    ): Promise<{
       graphs: BuildGraph[];
       sources: Record<string, PersistSource>;
       connectionDigests: Record<string, string>;
+      connections: Map<string, MalloyConnection>;
    }> {
       const modelPaths = pkg.getModelPaths();
       const allGraphs: BuildGraph[] = [];
@@ -665,7 +701,7 @@ export class MaterializationService {
             await Model.getModelRuntime(
                pkg.getPackagePath(),
                modelPath,
-               pkg.getConnections(),
+               pkg.getMalloyConfig(),
             );
 
          const modelMaterializer = runtime.loadModel(modelURL, {
@@ -730,7 +766,13 @@ export class MaterializationService {
          tableOwners.set(key, sourceID);
       }
 
-      const connections = pkg.getConnections();
+      // Resolve only the connections this build plan actually targets;
+      // the package's MalloyConfig caches each lookup so the build phase
+      // sees the same Connection instance and avoids re-resolving.
+      const connections = await resolvePackageConnections(
+         pkg,
+         allGraphs.map((g) => g.connectionName),
+      );
       const connectionDigests: Record<string, string> = {};
       for (const graph of allGraphs) {
          const conn = connections.get(graph.connectionName);
@@ -739,7 +781,12 @@ export class MaterializationService {
          }
       }
 
-      return { graphs: allGraphs, sources: allSources, connectionDigests };
+      return {
+         graphs: allGraphs,
+         sources: allSources,
+         connectionDigests,
+         connections,
+      };
    }
 
    /**

package/src/service/model.ts

@@ -1,10 +1,10 @@
-import { DuckDBConnection } from "@malloydata/db-duckdb";
 import {
    Annotation,
    API,
    Connection,
    FixedConnectionMap,
    isSourceDef,
+   MalloyConfig,
    MalloyError,
    ModelDef,
    modelDefToModelInfo,
@@ -27,7 +27,6 @@ import { DataStyles } from "@malloydata/render";
 import { metrics } from "@opentelemetry/api";
 import * as fs from "fs/promises";
 import * as path from "path";
-import { fileURLToPath } from "url";
 import { components } from "../api";
 import {
    MODEL_FILE_SUFFIX,
@@ -72,6 +71,7 @@ const MALLOY_VERSION = (
 ).version;
 
 export type ModelType = "model" | "notebook";
+type ModelConnectionInput = MalloyConfig | Map<string, Connection>;
 
 interface RunnableNotebookCell {
    type: "code" | "markdown";
@@ -162,7 +162,7 @@ export class Model {
       packageName: string,
       packagePath: string,
       modelPath: string,
-      connections: Map<string, Connection>,
+      malloyConfig: ModelConnectionInput,
       options?: { buildManifest?: BuildManifest["entries"] },
    ): Promise<Model> {
       // getModelRuntime might throw a ModelNotFoundError. It's the callers responsibility
@@ -171,7 +171,7 @@ export class Model {
          await Model.getModelRuntime(
             packagePath,
             modelPath,
-            connections,
+            malloyConfig,
             options,
          );
 
@@ -672,7 +672,7 @@ export class Model {
    static async getModelRuntime(
       packagePath: string,
       modelPath: string,
-      connections: Map<string, Connection>,
+      malloyConfig: ModelConnectionInput,
       options?: { buildManifest?: BuildManifest["entries"] },
    ): Promise<{
       runtime: Runtime;
@@ -703,35 +703,32 @@ export class Model {
 
       const modelURL = new URL(`file://${fullModelPath}`);
       const baseUrl = new URL(".", modelURL);
-      const fileUrl = new URL(baseUrl.pathname, "file:");
-      const workingDirectory = fileURLToPath(fileUrl);
       const importBaseURL = new URL(baseUrl.pathname + "/", "file:");
       const urlReader = new HackyDataStylesAccumulator(URL_READER);
 
-      const duckdbConnection = connections.get("duckdb") as DuckDBConnection;
-      await duckdbConnection.runSQL(
-         `SET FILE_SEARCH_PATH='${workingDirectory}';`,
-      );
-
-      const runtimeOptions: {
-         urlReader: typeof urlReader;
-         connections: FixedConnectionMap;
-         buildManifest?: BuildManifest;
-      } = {
+      // Request runtimes borrow the cached package MalloyConfig. The package
+      // owns release; callers must not release this runtime per request.
+      const runtime = new Runtime({
          urlReader,
-         connections: new FixedConnectionMap(connections, "duckdb"),
-      };
+         config: Model.toMalloyConfig(malloyConfig),
+         buildManifest: options?.buildManifest
+            ? { entries: options.buildManifest, strict: false }
+            : undefined,
+      });
+      const dataStyles = urlReader.getHackyAccumulatedDataStyles();
+      return { runtime, modelURL, importBaseURL, dataStyles, modelType };
+   }
 
-      if (options?.buildManifest) {
-         runtimeOptions.buildManifest = {
-            entries: options.buildManifest,
-            strict: false,
-         };
+   private static toMalloyConfig(input: ModelConnectionInput): MalloyConfig {
+      if (input instanceof MalloyConfig) {
+         return input;
       }
 
-      const runtime = new Runtime(runtimeOptions);
-      const dataStyles = urlReader.getHackyAccumulatedDataStyles();
-      return { runtime, modelURL, importBaseURL, dataStyles, modelType };
+      const malloyConfig = new MalloyConfig({ connections: {} });
+      malloyConfig.wrapConnections(
+         () => new FixedConnectionMap(input, "duckdb"),
+      );
+      return malloyConfig;
    }
 
    private static getQueries(

package/src/service/package.spec.ts

@@ -1,7 +1,7 @@
 import { afterEach, beforeEach, describe, expect, it } from "bun:test";
 import { Stats } from "fs";
 import fs from "fs/promises";
-import { join } from "path";
+import { join, resolve } from "path";
 import sinon from "sinon";
 import { PackageNotFoundError } from "../errors";
 import { Model } from "./model";
@@ -155,6 +155,130 @@ describe("service/package", () => {
             },
             { timeout: 20000 },
          );
+
+         it(
+            "uses package-root-relative DuckDB file paths for nested models",
+            async () => {
+               await fs.mkdir(join(testPackageDirectory, "data"), {
+                  recursive: true,
+               });
+               await fs.mkdir(join(testPackageDirectory, "models"), {
+                  recursive: true,
+               });
+               await fs.writeFile(
+                  join(testPackageDirectory, "data", "root.csv"),
+                  "name,value\nalpha,1\nbeta,2\n",
+               );
+               await fs.writeFile(
+                  join(testPackageDirectory, "models", "root_read.malloy"),
+                  [
+                     "source: rows is duckdb.table('data/root.csv')",
+                     "query: row_count is rows -> { aggregate: c is count() }",
+                  ].join("\n"),
+               );
+
+               const absolutePackageDirectory = resolve(testPackageDirectory);
+               const packageInstance = await Package.create(
+                  "testProject",
+                  "testPackage",
+                  absolutePackageDirectory,
+                  new Map(),
+               );
+               try {
+                  const rootModel = packageInstance.getModel(
+                     "models/root_read.malloy",
+                  );
+                  expect(rootModel).toBeDefined();
+                  const rootResults = await rootModel!.getQueryResults(
+                     undefined,
+                     "row_count",
+                  );
+                  expect(
+                     (rootResults.compactResult as { c: number }[])[0]?.c,
+                  ).toBe(2);
+               } finally {
+                  await packageInstance.getMalloyConfig().releaseConnections();
+               }
+            },
+            { timeout: 20000 },
+         );
+
+         it(
+            "does not resolve DuckDB file paths relative to the model directory",
+            async () => {
+               await fs.mkdir(join(testPackageDirectory, "models"), {
+                  recursive: true,
+               });
+               await fs.writeFile(
+                  join(testPackageDirectory, "models", "sibling.csv"),
+                  "name,value\nnested,3\n",
+               );
+               await fs.writeFile(
+                  join(testPackageDirectory, "models", "sibling_read.malloy"),
+                  [
+                     "source: rows is duckdb.table('./sibling.csv')",
+                     "query: row_count is rows -> { aggregate: c is count() }",
+                  ].join("\n"),
+               );
+
+               await expect(
+                  Package.create(
+                     "testProject",
+                     "testPackage",
+                     resolve(testPackageDirectory),
+                     new Map(),
+                  ),
+               ).rejects.toThrow();
+            },
+            { timeout: 20000 },
+         );
+
+         it(
+            "does not treat package-root paths as filesystem isolation",
+            async () => {
+               const outsidePath = resolve(
+                  testPackageDirectory,
+                  "..",
+                  "outside-package.csv",
+               );
+               await fs.mkdir(join(testPackageDirectory, "models"), {
+                  recursive: true,
+               });
+               await fs.writeFile(outsidePath, "name,value\noutside,9\n");
+               await fs.writeFile(
+                  join(testPackageDirectory, "models", "outside_read.malloy"),
+                  [
+                     "source: rows is duckdb.table('../outside-package.csv')",
+                     "query: row_count is rows -> { aggregate: c is count() }",
+                  ].join("\n"),
+               );
+
+               let packageInstance: Package | undefined;
+               try {
+                  packageInstance = await Package.create(
+                     "testProject",
+                     "testPackage",
+                     resolve(testPackageDirectory),
+                     new Map(),
+                  );
+                  const outsideModel = packageInstance.getModel(
+                     "models/outside_read.malloy",
+                  );
+                  expect(outsideModel).toBeDefined();
+                  const outsideResults = await outsideModel!.getQueryResults(
+                     undefined,
+                     "row_count",
+                  );
+                  expect(
+                     (outsideResults.compactResult as { c: number }[])[0]?.c,
+                  ).toBe(1);
+               } finally {
+                  await packageInstance?.getMalloyConfig().releaseConnections();
+                  await fs.rm(outsidePath, { force: true });
+               }
+            },
+            { timeout: 20000 },
+         );
       });
 
       describe("listModels", () => {