@malloy-publisher/server 0.0.192 → 0.0.194

package/src/service/materialization_service.ts

@@ -1,6 +1,7 @@
 import type {
    BuildGraph,
    Connection as MalloyConnection,
+   MalloyConfig,
    PersistSource,
 } from "@malloydata/malloy";
 import { Manifest } from "@malloydata/malloy";
@@ -46,6 +47,34 @@ export function stagingSuffix(buildId: string): string {
    return `_${buildId.substring(0, STAGING_BUILD_ID_LEN)}`;
 }
 
+/**
+ * Resolve a Map<name, Connection> for just the names a materialization
+ * step is about to touch. The package's MalloyConfig caches each lookup,
+ * so subsequent calls with overlapping names are cheap. A failed lookup
+ * is logged and the name is omitted from the result — downstream code
+ * (`forceDeleteRowOnMissingConnection` in teardown, or the explicit
+ * "connection X not found" check in build) handles a missing entry.
+ */
+async function resolvePackageConnections(
+   pkg: { getMalloyConnection(name: string): Promise<MalloyConnection> },
+   names: Iterable<string>,
+): Promise<Map<string, MalloyConnection>> {
+   const map = new Map<string, MalloyConnection>();
+   const seen = new Set<string>();
+   for (const name of names) {
+      if (!name || seen.has(name)) continue;
+      seen.add(name);
+      try {
+         map.set(name, await pkg.getMalloyConnection(name));
+      } catch (err) {
+         logger.warn(`Failed to resolve connection ${name}`, {
+            error: err instanceof Error ? err.message : String(err),
+         });
+      }
+   }
+   return map;
+}
+
 /**
  * Build a stable key for a `(connectionName, tableName)` pair.
  * Used to check whether a persist target was created by a previous build.
@@ -494,13 +523,17 @@ export class MaterializationService {
 
       const project = await this.projectStore.getProject(projectName, false);
       const pkg = await project.getPackage(packageName, false);
-      const connections = pkg.getConnections();
 
       const entries = await this.manifestService.listEntries(
          projectId,
          packageName,
       );
 
+      const connections = await resolvePackageConnections(
+         pkg,
+         entries.map((e) => e.connectionName),
+      );
+
       // `forceDeleteRowOnMissingConnection`: teardown is the one place
       // where we'd rather lose the manifest row than leave it pointing at
       // a vanished connection. We also deliberately omit `liveTables`:
@@ -560,7 +593,9 @@ export class MaterializationService {
       );
 
       // ── STEP 2: COMPILE & PLAN ─────────────────────────────────────
-      const { graphs, sources, connectionDigests } =
+      // `connections` is built lazily from the connection names the plan
+      // actually targets — no upfront ATTACH on every project connection.
+      const { graphs, sources, connectionDigests, connections } =
          await this.compilePackageBuildPlan(pkg, signal);
 
       if (graphs.length === 0) {
@@ -569,7 +604,6 @@ export class MaterializationService {
       }
 
       // ── STEP 3: BUILD ──────────────────────────────────────────────
-      const connections = pkg.getConnections();
       let sourcesBuilt = 0;
       let sourcesSkipped = 0;
       const sourceResults: Record<string, unknown>[] = [];
@@ -646,13 +680,15 @@ export class MaterializationService {
       pkg: {
          getModelPaths(): string[];
          getPackagePath(): string;
-         getConnections(): Map<string, MalloyConnection>;
+         getMalloyConfig(): MalloyConfig;
+         getMalloyConnection(name: string): Promise<MalloyConnection>;
       },
       signal: AbortSignal,
    ): Promise<{
       graphs: BuildGraph[];
       sources: Record<string, PersistSource>;
       connectionDigests: Record<string, string>;
+      connections: Map<string, MalloyConnection>;
    }> {
       const modelPaths = pkg.getModelPaths();
       const allGraphs: BuildGraph[] = [];
@@ -665,7 +701,7 @@ export class MaterializationService {
             await Model.getModelRuntime(
                pkg.getPackagePath(),
                modelPath,
-               pkg.getConnections(),
+               pkg.getMalloyConfig(),
             );
 
          const modelMaterializer = runtime.loadModel(modelURL, {
@@ -730,7 +766,13 @@ export class MaterializationService {
          tableOwners.set(key, sourceID);
       }
 
-      const connections = pkg.getConnections();
+      // Resolve only the connections this build plan actually targets;
+      // the package's MalloyConfig caches each lookup so the build phase
+      // sees the same Connection instance and avoids re-resolving.
+      const connections = await resolvePackageConnections(
+         pkg,
+         allGraphs.map((g) => g.connectionName),
+      );
       const connectionDigests: Record<string, string> = {};
       for (const graph of allGraphs) {
          const conn = connections.get(graph.connectionName);
@@ -739,7 +781,12 @@ export class MaterializationService {
          }
       }
 
-      return { graphs: allGraphs, sources: allSources, connectionDigests };
+      return {
+         graphs: allGraphs,
+         sources: allSources,
+         connectionDigests,
+         connections,
+      };
    }
 
    /**

package/src/service/model.ts

@@ -1,10 +1,10 @@
-import { DuckDBConnection } from "@malloydata/db-duckdb";
 import {
    Annotation,
    API,
    Connection,
    FixedConnectionMap,
    isSourceDef,
+   MalloyConfig,
    MalloyError,
    ModelDef,
    modelDefToModelInfo,
@@ -27,7 +27,6 @@ import { DataStyles } from "@malloydata/render";
 import { metrics } from "@opentelemetry/api";
 import * as fs from "fs/promises";
 import * as path from "path";
-import { fileURLToPath } from "url";
 import { components } from "../api";
 import {
    MODEL_FILE_SUFFIX,
@@ -72,6 +71,7 @@ const MALLOY_VERSION = (
 ).version;
 
 export type ModelType = "model" | "notebook";
+type ModelConnectionInput = MalloyConfig | Map<string, Connection>;
 
 interface RunnableNotebookCell {
    type: "code" | "markdown";
@@ -162,7 +162,7 @@ export class Model {
       packageName: string,
       packagePath: string,
       modelPath: string,
-      connections: Map<string, Connection>,
+      malloyConfig: ModelConnectionInput,
       options?: { buildManifest?: BuildManifest["entries"] },
    ): Promise<Model> {
       // getModelRuntime might throw a ModelNotFoundError. It's the callers responsibility
@@ -171,7 +171,7 @@ export class Model {
          await Model.getModelRuntime(
             packagePath,
             modelPath,
-            connections,
+            malloyConfig,
             options,
          );
 
@@ -672,7 +672,7 @@ export class Model {
    static async getModelRuntime(
       packagePath: string,
       modelPath: string,
-      connections: Map<string, Connection>,
+      malloyConfig: ModelConnectionInput,
       options?: { buildManifest?: BuildManifest["entries"] },
    ): Promise<{
       runtime: Runtime;
@@ -703,35 +703,32 @@ export class Model {
 
       const modelURL = new URL(`file://${fullModelPath}`);
       const baseUrl = new URL(".", modelURL);
-      const fileUrl = new URL(baseUrl.pathname, "file:");
-      const workingDirectory = fileURLToPath(fileUrl);
       const importBaseURL = new URL(baseUrl.pathname + "/", "file:");
       const urlReader = new HackyDataStylesAccumulator(URL_READER);
 
-      const duckdbConnection = connections.get("duckdb") as DuckDBConnection;
-      await duckdbConnection.runSQL(
-         `SET FILE_SEARCH_PATH='${workingDirectory}';`,
-      );
-
-      const runtimeOptions: {
-         urlReader: typeof urlReader;
-         connections: FixedConnectionMap;
-         buildManifest?: BuildManifest;
-      } = {
+      // Request runtimes borrow the cached package MalloyConfig. The package
+      // owns release; callers must not release this runtime per request.
+      const runtime = new Runtime({
          urlReader,
-         connections: new FixedConnectionMap(connections, "duckdb"),
-      };
+         config: Model.toMalloyConfig(malloyConfig),
+         buildManifest: options?.buildManifest
+            ? { entries: options.buildManifest, strict: false }
+            : undefined,
+      });
+      const dataStyles = urlReader.getHackyAccumulatedDataStyles();
+      return { runtime, modelURL, importBaseURL, dataStyles, modelType };
+   }
 
-      if (options?.buildManifest) {
-         runtimeOptions.buildManifest = {
-            entries: options.buildManifest,
-            strict: false,
-         };
+   private static toMalloyConfig(input: ModelConnectionInput): MalloyConfig {
+      if (input instanceof MalloyConfig) {
+         return input;
       }
 
-      const runtime = new Runtime(runtimeOptions);
-      const dataStyles = urlReader.getHackyAccumulatedDataStyles();
-      return { runtime, modelURL, importBaseURL, dataStyles, modelType };
+      const malloyConfig = new MalloyConfig({ connections: {} });
+      malloyConfig.wrapConnections(
+         () => new FixedConnectionMap(input, "duckdb"),
+      );
+      return malloyConfig;
    }
 
    private static getQueries(

package/src/service/package.spec.ts

@@ -1,7 +1,7 @@
 import { afterEach, beforeEach, describe, expect, it } from "bun:test";
 import { Stats } from "fs";
 import fs from "fs/promises";
-import { join } from "path";
+import { join, resolve } from "path";
 import sinon from "sinon";
 import { PackageNotFoundError } from "../errors";
 import { Model } from "./model";
@@ -155,6 +155,130 @@ describe("service/package", () => {
             },
             { timeout: 20000 },
          );
+
+         it(
+            "uses package-root-relative DuckDB file paths for nested models",
+            async () => {
+               await fs.mkdir(join(testPackageDirectory, "data"), {
+                  recursive: true,
+               });
+               await fs.mkdir(join(testPackageDirectory, "models"), {
+                  recursive: true,
+               });
+               await fs.writeFile(
+                  join(testPackageDirectory, "data", "root.csv"),
+                  "name,value\nalpha,1\nbeta,2\n",
+               );
+               await fs.writeFile(
+                  join(testPackageDirectory, "models", "root_read.malloy"),
+                  [
+                     "source: rows is duckdb.table('data/root.csv')",
+                     "query: row_count is rows -> { aggregate: c is count() }",
+                  ].join("\n"),
+               );
+
+               const absolutePackageDirectory = resolve(testPackageDirectory);
+               const packageInstance = await Package.create(
+                  "testProject",
+                  "testPackage",
+                  absolutePackageDirectory,
+                  new Map(),
+               );
+               try {
+                  const rootModel = packageInstance.getModel(
+                     "models/root_read.malloy",
+                  );
+                  expect(rootModel).toBeDefined();
+                  const rootResults = await rootModel!.getQueryResults(
+                     undefined,
+                     "row_count",
+                  );
+                  expect(
+                     (rootResults.compactResult as { c: number }[])[0]?.c,
+                  ).toBe(2);
+               } finally {
+                  await packageInstance.getMalloyConfig().releaseConnections();
+               }
+            },
+            { timeout: 20000 },
+         );
+
+         it(
+            "does not resolve DuckDB file paths relative to the model directory",
+            async () => {
+               await fs.mkdir(join(testPackageDirectory, "models"), {
+                  recursive: true,
+               });
+               await fs.writeFile(
+                  join(testPackageDirectory, "models", "sibling.csv"),
+                  "name,value\nnested,3\n",
+               );
+               await fs.writeFile(
+                  join(testPackageDirectory, "models", "sibling_read.malloy"),
+                  [
+                     "source: rows is duckdb.table('./sibling.csv')",
+                     "query: row_count is rows -> { aggregate: c is count() }",
+                  ].join("\n"),
+               );
+
+               await expect(
+                  Package.create(
+                     "testProject",
+                     "testPackage",
+                     resolve(testPackageDirectory),
+                     new Map(),
+                  ),
+               ).rejects.toThrow();
+            },
+            { timeout: 20000 },
+         );
+
+         it(
+            "does not treat package-root paths as filesystem isolation",
+            async () => {
+               const outsidePath = resolve(
+                  testPackageDirectory,
+                  "..",
+                  "outside-package.csv",
+               );
+               await fs.mkdir(join(testPackageDirectory, "models"), {
+                  recursive: true,
+               });
+               await fs.writeFile(outsidePath, "name,value\noutside,9\n");
+               await fs.writeFile(
+                  join(testPackageDirectory, "models", "outside_read.malloy"),
+                  [
+                     "source: rows is duckdb.table('../outside-package.csv')",
+                     "query: row_count is rows -> { aggregate: c is count() }",
+                  ].join("\n"),
+               );
+
+               let packageInstance: Package | undefined;
+               try {
+                  packageInstance = await Package.create(
+                     "testProject",
+                     "testPackage",
+                     resolve(testPackageDirectory),
+                     new Map(),
+                  );
+                  const outsideModel = packageInstance.getModel(
+                     "models/outside_read.malloy",
+                  );
+                  expect(outsideModel).toBeDefined();
+                  const outsideResults = await outsideModel!.getQueryResults(
+                     undefined,
+                     "row_count",
+                  );
+                  expect(
+                     (outsideResults.compactResult as { c: number }[])[0]?.c,
+                  ).toBe(1);
+               } finally {
+                  await packageInstance?.getMalloyConfig().releaseConnections();
+                  await fs.rm(outsidePath, { force: true });
+               }
+            },
+            { timeout: 20000 },
+         );
       });
 
       describe("listModels", () => {

package/src/service/package.ts

@@ -2,10 +2,14 @@ import * as fs from "fs/promises";
 import * as path from "path";
 
 import { DuckDBConnection } from "@malloydata/db-duckdb";
+import "@malloydata/db-duckdb/native";
 import {
    Connection,
    ConnectionRuntime,
    EmptyURLReader,
+   FixedConnectionMap,
+   contextOverlay,
+   MalloyConfig,
    SourceDef,
 } from "@malloydata/malloy";
 import { metrics } from "@opentelemetry/api";
@@ -28,6 +32,14 @@ type ApiNotebook = components["schemas"]["Notebook"];
 export type ApiPackage = components["schemas"]["Package"];
 type ApiColumn = components["schemas"]["Column"];
 type ApiTableDescription = components["schemas"]["TableDescription"];
+// A thunk lets callers pass a live reference to the *current* project
+// MalloyConfig so the package wrapper resolves project connections against the
+// generation that's active at lookup time, not the one that was current when
+// the package was first loaded.
+type PackageConnectionInput =
+   | MalloyConfig
+   | Map<string, Connection>
+   | (() => MalloyConfig);
 
 const ENABLE_LIST_MODEL_COMPILATION = true;
 export class Package {
@@ -37,7 +49,7 @@ export class Package {
    private databases: ApiDatabase[];
    private models: Map<string, Model> = new Map();
    private packagePath: string;
-   private connections: Map<string, Connection> = new Map();
+   private malloyConfig: MalloyConfig;
    private static meter = metrics.getMeter("publisher");
    private static packageLoadHistogram = this.meter.createHistogram(
       "malloy_package_load_duration",
@@ -54,7 +66,7 @@ export class Package {
       packageMetadata: ApiPackage,
       databases: ApiDatabase[],
       models: Map<string, Model>,
-      connections: Map<string, Connection> = new Map(),
+      malloyConfig: MalloyConfig = new MalloyConfig({ connections: {} }),
    ) {
       this.projectName = projectName;
       this.packageName = packageName;
@@ -62,14 +74,14 @@ export class Package {
       this.packageMetadata = packageMetadata;
       this.databases = databases;
       this.models = models;
-      this.connections = connections;
+      this.malloyConfig = malloyConfig;
    }
 
    static async create(
       projectName: string,
       packageName: string,
       packagePath: string,
-      projectConnections: Map<string, Connection>,
+      projectMalloyConfig: PackageConnectionInput,
    ): Promise<Package> {
       const startTime = performance.now();
       await Package.validatePackageManifestExistsOrThrowError(packagePath);
@@ -97,20 +109,17 @@ export class Package {
             databaseCount: databases.length,
             duration: formatDuration(databasesTime - packageConfigTime),
          });
-         const connections = new Map<string, Connection>(projectConnections);
-
-         // Add a duckdb connection for the package.
-         const duckdbConnection = new DuckDBConnection(
-            "duckdb",
-            ":memory:",
+         const malloyConfig = Package.buildPackageMalloyConfig(
             packagePath,
+            typeof projectMalloyConfig === "function"
+               ? projectMalloyConfig
+               : () => Package.toMalloyConfig(projectMalloyConfig),
          );
-         connections.set("duckdb", duckdbConnection);
 
          const models = await Package.loadModels(
             packageName,
             packagePath,
-            connections,
+            malloyConfig,
          );
          const modelsTime = performance.now();
          logger.info("Models loaded", {
@@ -159,7 +168,7 @@ export class Package {
             packageConfig,
             databases,
             models,
-            connections,
+            malloyConfig,
          );
       } catch (error) {
          logger.error(`Error loading package ${packageName}`, { error });
@@ -190,10 +199,6 @@ export class Package {
       return this.packageName;
    }
 
-   public getPackagePath(): string {
-      return this.packagePath;
-   }
-
    public getPackageMetadata(): ApiPackage {
       return this.packageMetadata;
    }
@@ -206,19 +211,24 @@ export class Package {
       return this.models.get(modelPath);
    }
 
+   public async getMalloyConnection(
+      connectionName: string,
+   ): Promise<Connection> {
+      return this.malloyConfig.connections.lookupConnection(connectionName);
+   }
+
+   public getMalloyConfig(): MalloyConfig {
+      return this.malloyConfig;
+   }
+
+   public getPackagePath(): string {
+      return this.packagePath;
+   }
+
    public getModelPaths(): string[] {
       return Array.from(this.models.keys());
    }
 
-   /**
-    * Recompile every model in the package with the given build manifest
-    * so queries resolve persist references to materialized tables.
-    *
-    * Builds a fresh map off to the side and swaps it in at the end. If any
-    * recompile fails the whole call rejects before the swap and the live
-    * `this.models` reference remains untouched — no half-loaded state is
-    * ever observable to concurrent readers.
-    */
    public async reloadAllModels(
       buildManifest: BuildManifest["entries"],
    ): Promise<void> {
@@ -228,14 +238,13 @@ export class Package {
          modelCount: modelPaths.length,
          manifestEntryCount: Object.keys(buildManifest).length,
       });
-
       const reloaded = await Promise.all(
          modelPaths.map((modelPath) =>
             Model.create(
                this.packageName,
                this.packagePath,
                modelPath,
-               this.connections,
+               this.malloyConfig,
                { buildManifest },
             ),
          ),
@@ -247,20 +256,6 @@ export class Package {
       this.models = nextModels;
    }
 
-   public getConnections(): Map<string, Connection> {
-      return this.connections;
-   }
-
-   public getMalloyConnection(connectionName: string): Connection {
-      const connection = this.connections.get(connectionName);
-      if (!connection) {
-         throw new Error(
-            `Connection ${connectionName} not found in package ${this.packageName}`,
-         );
-      }
-      return connection;
-   }
-
    public async getModelFileText(modelPath: string): Promise<string> {
       const model = this.getModel(modelPath);
       if (!model) {
@@ -322,17 +317,64 @@ export class Package {
    private static async loadModels(
       packageName: string,
       packagePath: string,
-      connections: Map<string, Connection>,
+      malloyConfig: MalloyConfig,
    ): Promise<Map<string, Model>> {
       const modelPaths = await Package.getModelPaths(packagePath);
       const models = await Promise.all(
          modelPaths.map((modelPath) =>
-            Model.create(packageName, packagePath, modelPath, connections),
+            Model.create(packageName, packagePath, modelPath, malloyConfig),
          ),
       );
       return new Map(models.map((model) => [model.getPath(), model]));
    }
 
+   private static buildPackageMalloyConfig(
+      packagePath: string,
+      getProjectMalloyConfig: () => MalloyConfig,
+   ): MalloyConfig {
+      const malloyConfig = new MalloyConfig(
+         {
+            connections: {
+               duckdb: {
+                  is: "duckdb",
+                  databasePath: ":memory:",
+               },
+            },
+         },
+         {
+            config: contextOverlay({ rootDirectory: packagePath }),
+         },
+      );
+
+      malloyConfig.wrapConnections((base) => ({
+         lookupConnection: async (name?: string) => {
+            if (!name || name === "duckdb") {
+               return base.lookupConnection(name);
+            }
+            // Resolve against the *current* project MalloyConfig so a
+            // connection-generation swap on Project propagates without a
+            // package reload.
+            return getProjectMalloyConfig().connections.lookupConnection(name);
+         },
+      }));
+
+      return malloyConfig;
+   }
+
+   private static toMalloyConfig(
+      input: MalloyConfig | Map<string, Connection>,
+   ): MalloyConfig {
+      if (input instanceof MalloyConfig) {
+         return input;
+      }
+
+      const malloyConfig = new MalloyConfig({ connections: {} });
+      malloyConfig.wrapConnections(
+         () => new FixedConnectionMap(input, "duckdb"),
+      );
+      return malloyConfig;
+   }
+
    private static async getModelPaths(packagePath: string): Promise<string[]> {
       let files = undefined;
       try {