@malloy-publisher/server 0.0.195 → 0.0.196

package/src/service/package.ts

@@ -6,9 +6,9 @@ import "@malloydata/db-duckdb/native";
 import {
    Connection,
    ConnectionRuntime,
+   contextOverlay,
    EmptyURLReader,
    FixedConnectionMap,
-   contextOverlay,
    MalloyConfig,
    SourceDef,
 } from "@malloydata/malloy";
@@ -32,8 +32,8 @@ type ApiNotebook = components["schemas"]["Notebook"];
 export type ApiPackage = components["schemas"]["Package"];
 type ApiColumn = components["schemas"]["Column"];
 type ApiTableDescription = components["schemas"]["TableDescription"];
-// A thunk lets callers pass a live reference to the *current* project
-// MalloyConfig so the package wrapper resolves project connections against the
+// A thunk lets callers pass a live reference to the *current* environment
+// MalloyConfig so the package wrapper resolves environment connections against the
 // generation that's active at lookup time, not the one that was current when
 // the package was first loaded.
 type PackageConnectionInput =
@@ -43,7 +43,7 @@ type PackageConnectionInput =
 
 const ENABLE_LIST_MODEL_COMPILATION = true;
 export class Package {
-   private projectName: string;
+   private environmentName: string;
    private packageName: string;
    private packageMetadata: ApiPackage;
    private databases: ApiDatabase[];
@@ -60,7 +60,7 @@ export class Package {
    );
 
    constructor(
-      projectName: string,
+      environmentName: string,
       packageName: string,
       packagePath: string,
       packageMetadata: ApiPackage,
@@ -68,7 +68,7 @@ export class Package {
       models: Map<string, Model>,
       malloyConfig: MalloyConfig = new MalloyConfig({ connections: {} }),
    ) {
-      this.projectName = projectName;
+      this.environmentName = environmentName;
       this.packageName = packageName;
       this.packagePath = packagePath;
       this.packageMetadata = packageMetadata;
@@ -78,10 +78,10 @@ export class Package {
    }
 
    static async create(
-      projectName: string,
+      environmentName: string,
       packageName: string,
       packagePath: string,
-      projectMalloyConfig: PackageConnectionInput,
+      environmentMalloyConfig: PackageConnectionInput,
    ): Promise<Package> {
       const startTime = performance.now();
       await Package.validatePackageManifestExistsOrThrowError(packagePath);
@@ -100,7 +100,7 @@ export class Package {
                packageConfigTime - manifestValidationTime,
             ),
          });
-         packageConfig.resource = `${API_PREFIX}/projects/${projectName}/packages/${packageName}`;
+         packageConfig.resource = `${API_PREFIX}/environments/${environmentName}/packages/${packageName}`;
 
          const databases = await Package.readDatabases(packagePath);
          const databasesTime = performance.now();
@@ -111,9 +111,9 @@ export class Package {
          });
          const malloyConfig = Package.buildPackageMalloyConfig(
             packagePath,
-            typeof projectMalloyConfig === "function"
-               ? projectMalloyConfig
-               : () => Package.toMalloyConfig(projectMalloyConfig),
+            typeof environmentMalloyConfig === "function"
+               ? environmentMalloyConfig
+               : () => Package.toMalloyConfig(environmentMalloyConfig),
          );
 
          const models = await Package.loadModels(
@@ -162,7 +162,7 @@ export class Package {
             duration: formatDuration(executionTime),
          });
          return new Package(
-            projectName,
+            environmentName,
             packageName,
             packagePath,
             packageConfig,
@@ -283,7 +283,7 @@ export class Package {
                   }
                }
                return {
-                  projectName: this.projectName,
+                  environmentName: this.environmentName,
                   path: modelPath,
                   packageName: this.packageName,
                   error,
@@ -305,7 +305,7 @@ export class Package {
                   error = this.models.get(modelPath)?.getNotebookError();
                }
                return {
-                  projectName: this.projectName,
+                  environmentName: this.environmentName,
                   packageName: this.packageName,
                   path: modelPath,
                   error: error?.message,
@@ -330,7 +330,7 @@ export class Package {
 
    private static buildPackageMalloyConfig(
       packagePath: string,
-      getProjectMalloyConfig: () => MalloyConfig,
+      getEnvironmentMalloyConfig: () => MalloyConfig,
    ): MalloyConfig {
       const malloyConfig = new MalloyConfig(
          {
@@ -351,10 +351,12 @@ export class Package {
             if (!name || name === "duckdb") {
                return base.lookupConnection(name);
             }
-            // Resolve against the *current* project MalloyConfig so a
-            // connection-generation swap on Project propagates without a
+            // Resolve against the *current* environment MalloyConfig so a
+            // connection-generation swap on Environment propagates without a
             // package reload.
-            return getProjectMalloyConfig().connections.lookupConnection(name);
+            return getEnvironmentMalloyConfig().connections.lookupConnection(
+               name,
+            );
          },
       }));
 
@@ -495,8 +497,8 @@ export class Package {
       this.packageName = name;
    }
 
-   public setProjectName(projectName: string) {
-      this.projectName = projectName;
+   public setEnvironmentName(environmentName: string) {
+      this.environmentName = environmentName;
    }
 
    public setPackageMetadata(packageMetadata: ApiPackage) {

package/src/service/resolve_environment.ts

@@ -0,0 +1,15 @@
+import { EnvironmentNotFoundError } from "../errors";
+import { ResourceRepository } from "../storage/DatabaseInterface";
+
+export async function resolveEnvironmentId(
+   repository: ResourceRepository,
+   environmentName: string,
+): Promise<string> {
+   const dbEnvironment = await repository.getEnvironmentByName(environmentName);
+   if (!dbEnvironment) {
+      throw new EnvironmentNotFoundError(
+         `Environment '${environmentName}' not found`,
+      );
+   }
+   return dbEnvironment.id;
+}

package/src/storage/DatabaseInterface.ts

@@ -9,20 +9,26 @@ export interface DatabaseConnection {
 }
 
 export interface ResourceRepository {
-   // Projects
-   listProjects(): Promise<Project[]>;
-   getProjectById(id: string): Promise<Project | null>;
-   getProjectByName(name: string): Promise<Project | null>;
-   createProject(
-      project: Omit<Project, "id" | "createdAt" | "updatedAt">,
-   ): Promise<Project>;
-   updateProject(id: string, updates: Partial<Project>): Promise<Project>;
-   deleteProject(id: string): Promise<void>;
+   // Environments
+   listEnvironments(): Promise<Environment[]>;
+   getEnvironmentById(id: string): Promise<Environment | null>;
+   getEnvironmentByName(name: string): Promise<Environment | null>;
+   createEnvironment(
+      environment: Omit<Environment, "id" | "createdAt" | "updatedAt">,
+   ): Promise<Environment>;
+   updateEnvironment(
+      id: string,
+      updates: Partial<Environment>,
+   ): Promise<Environment>;
+   deleteEnvironment(id: string): Promise<void>;
 
    // Packages
-   listPackages(projectId: string): Promise<Package[]>;
+   listPackages(environmentId: string): Promise<Package[]>;
    getPackageById(id: string): Promise<Package | null>;
-   getPackageByName(projectId: string, name: string): Promise<Package | null>;
+   getPackageByName(
+      environmentId: string,
+      name: string,
+   ): Promise<Package | null>;
    createPackage(
       pkg: Omit<Package, "id" | "createdAt" | "updatedAt">,
    ): Promise<Package>;
@@ -30,10 +36,10 @@ export interface ResourceRepository {
    deletePackage(id: string): Promise<void>;
 
    // Connections
-   listConnections(projectId: string): Promise<Connection[]>;
+   listConnections(environmentId: string): Promise<Connection[]>;
    getConnectionById(id: string): Promise<Connection | null>;
    getConnectionByName(
-      projectId: string,
+      environmentId: string,
       name: string,
    ): Promise<Connection | null>;
    createConnection(
@@ -47,17 +53,17 @@ export interface ResourceRepository {
 
    // Materializations
    listMaterializations(
-      projectId: string,
+      environmentId: string,
       packageName: string,
       options?: { limit?: number; offset?: number },
    ): Promise<Materialization[]>;
    getMaterializationById(id: string): Promise<Materialization | null>;
    getActiveMaterialization(
-      projectId: string,
+      environmentId: string,
       packageName: string,
    ): Promise<Materialization | null>;
    createMaterialization(
-      projectId: string,
+      environmentId: string,
       packageName: string,
       status?: MaterializationStatus,
       metadata?: Record<string, unknown> | null,
@@ -75,7 +81,7 @@ export interface ResourceRepository {
    deleteMaterialization(id: string): Promise<void>;
    // Build Manifests
    listManifestEntries(
-      projectId: string,
+      environmentId: string,
       packageName: string,
    ): Promise<ManifestEntry[]>;
    upsertManifestEntry(
@@ -84,7 +90,7 @@ export interface ResourceRepository {
    deleteManifestEntry(id: string): Promise<void>;
 }
 
-export interface Project {
+export interface Environment {
    id: string;
    name: string;
    path: string;
@@ -96,7 +102,7 @@ export interface Project {
 
 export interface Package {
    id: string;
-   projectId: string;
+   environmentId: string;
    name: string;
    description?: string;
    manifestPath: string;
@@ -107,7 +113,7 @@ export interface Package {
 
 export interface Connection {
    id: string;
-   projectId: string;
+   environmentId: string;
    name: string;
    type: "bigquery" | "postgres" | "duckdb" | "mysql" | "snowflake" | "trino";
    config: Record<string, unknown>;
@@ -124,7 +130,7 @@ export type MaterializationStatus =
 
 export interface Materialization {
    id: string;
-   projectId: string;
+   environmentId: string;
    packageName: string;
    status: MaterializationStatus;
    startedAt: Date | null;
@@ -137,7 +143,7 @@ export interface Materialization {
 
 export interface ManifestEntry {
    id: string;
-   projectId: string;
+   environmentId: string;
    packageName: string;
    buildId: string;
    tableName: string;
@@ -163,9 +169,12 @@ export interface BuildManifest {
  * orchestrated mode swaps in a DuckLakeManifestStore.
  */
 export interface ManifestStore {
-   getManifest(projectId: string, packageName: string): Promise<BuildManifest>;
+   getManifest(
+      environmentId: string,
+      packageName: string,
+   ): Promise<BuildManifest>;
    writeEntry(
-      projectId: string,
+      environmentId: string,
       packageName: string,
       buildId: string,
       tableName: string,
@@ -174,7 +183,7 @@ export interface ManifestStore {
    ): Promise<void>;
    deleteEntry(id: string): Promise<void>;
    listEntries(
-      projectId: string,
+      environmentId: string,
       packageName: string,
    ): Promise<ManifestEntry[]>;
 }

package/src/storage/StorageManager.mock.ts

@@ -7,12 +7,12 @@ export class StorageManager {
 
    getRepository() {
       return {
-         listProjects: async (): Promise<unknown[]> => [],
-         createProject: async (data: MockData): Promise<MockData> => ({
+         listEnvironments: async (): Promise<unknown[]> => [],
+         createEnvironment: async (data: MockData): Promise<MockData> => ({
             id: "test-id",
             ...data,
          }),
-         updateProject: async (
+         updateEnvironment: async (
             id: string,
             data: MockData,
          ): Promise<MockData> => ({

package/src/storage/StorageManager.ts

@@ -1,3 +1,4 @@
+import * as crypto from "crypto";
 import { logger } from "../logger";
 import {
    DatabaseConnection,
@@ -42,10 +43,23 @@ function escapeSQL(value: string): string {
    return value.replace(/'/g, "''");
 }
 
+function configKey(c: DuckLakeManifestConfig): string {
+   return `${c.catalogUrl}|${c.dataPath}`;
+}
+
+function catalogNameForConfig(c: DuckLakeManifestConfig): string {
+   const hash = crypto
+      .createHash("sha256")
+      .update(configKey(c))
+      .digest("hex")
+      .slice(0, 8);
+   return `manifest_lake_${hash}`;
+}
+
 /**
- * Manages the storage backend (DuckDB, Postgres, etc.) and per-project
- * manifest stores. Projects without `materializationStorage` config use
- * the default DuckDB manifest store. Projects with the config get a
+ * Manages the storage backend (DuckDB, Postgres, etc.) and per-environment
+ * manifest stores. Environments without `materializationStorage` config use
+ * the default DuckDB manifest store. Environments with the config get a
  * DuckLake-backed store attached lazily on first access.
  */
 export class StorageManager {
@@ -54,11 +68,15 @@ export class StorageManager {
    private repository: ResourceRepository | null = null;
    private defaultManifestStore: ManifestStore | null = null;
 
-   /** Per-project DuckLake manifest stores, keyed by projectId. */
-   private projectManifestStores = new Map<string, ManifestStore>();
+   /** Per-environment DuckLake manifest stores, keyed by environmentId. */
+   private environmentManifestStores = new Map<string, ManifestStore>();
 
-   /** Tracks which catalogs have been attached to avoid duplicate ATTACHes. */
-   private attachedCatalogs = new Set<string>();
+   /**
+    * Tracks attached DuckLake catalogs as `configKey -> catalogName`. Each
+    * unique materializationStorage config gets its own ATTACHment under a
+    * deterministic catalog name, so multiple configs can coexist on one worker.
+    */
+   private attachedCatalogs = new Map<string, string>();
 
    private config: StorageConfig;
 
@@ -104,33 +122,45 @@ export class StorageManager {
    }
 
    /**
-    * Lazily initializes a DuckLake manifest store for a project.
-    * The DuckLake catalog is attached to the shared DuckDB connection
-    * and persists for the lifetime of the process.
+    * Lazily initializes a DuckLake manifest store for an environment.
+    *
+    * One shared catalog per materializationStorage config: every environment
+    * pointing at the same (catalogUrl, dataPath) shares one `build_manifests`
+    * table inside it, partitioned by `environment_id` (set to the environment's name
+    * so it's stable across worker replicas — required for cross-pod manifest
+    * visibility in orchestrated mode). Different configs (e.g. different
+    * orgs) attach as separate catalogs under distinct deterministic aliases.
     */
-   async initializeDuckLakeForProject(
-      projectId: string,
+   async initializeDuckLakeForEnvironment(
+      environmentId: string,
+      environmentName: string,
       config: DuckLakeManifestConfig,
    ): Promise<void> {
       if (!this.duckDbConnection) {
          throw new Error("Storage not initialized. Call initialize() first.");
       }
 
-      const catalogName = `manifest_lake_${projectId.replace(/[^a-zA-Z0-9_]/g, "_")}`;
-
-      if (!this.attachedCatalogs.has(catalogName)) {
+      const key = configKey(config);
+      let catalogName = this.attachedCatalogs.get(key);
+      if (!catalogName) {
+         // Catalog name derived from the config so multiple configs can coexist as
+         // separate ATTACHments without colliding on the name.
+         catalogName = catalogNameForConfig(config);
          await this.attachDuckLakeCatalog(config, catalogName);
+         this.attachedCatalogs.set(key, catalogName);
       }
 
       const store = new DuckLakeManifestStore(
          this.duckDbConnection,
          catalogName,
+         environmentName,
       );
       await store.bootstrapSchema();
 
-      this.projectManifestStores.set(projectId, store);
-      logger.info("DuckLake manifest store initialized for project", {
-         projectId,
+      this.environmentManifestStores.set(environmentId, store);
+      logger.info("DuckLake manifest store initialized for environment", {
+         environmentId,
+         environmentName,
          catalogName,
       });
    }
@@ -155,7 +185,14 @@ export class StorageManager {
          config.dataPath.startsWith("s3://");
 
       let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
-      const attachOpts: string[] = [`DATA_PATH '${escapedDataPath}'`];
+      const attachOpts: string[] = [
+         `DATA_PATH '${escapedDataPath}'`,
+         // The manifest table is small relational metadata (one row per build).
+         // Set a high inlining limit so writes always land transactionally in
+         // the postgres catalog rather than as parquet files in object storage,
+         // sidestepping object-storage auth issues entirely for this path.
+         "DATA_INLINING_ROW_LIMIT 100000",
+      ];
       if (isCloudStorage) {
          attachOpts.push("OVERRIDE_DATA_PATH true");
       }
@@ -163,8 +200,6 @@ export class StorageManager {
 
       logger.info(`Attaching DuckLake manifest catalog: ${attachCmd}`);
       await connection.run(attachCmd);
-
-      this.attachedCatalogs.add(catalogName);
    }
 
    getRepository(): ResourceRepository {
@@ -175,15 +210,16 @@ export class StorageManager {
    }
 
    /**
-    * Returns the manifest store for a project. If the project has a
+    * Returns the manifest store for an environment. If the environment has a
     * DuckLake store configured, returns that; otherwise returns the
     * default DuckDB-backed store.
     */
-   getManifestStore(projectId?: string): ManifestStore {
-      if (projectId) {
-         const projectStore = this.projectManifestStores.get(projectId);
-         if (projectStore) {
-            return projectStore;
+   getManifestStore(environmentId?: string): ManifestStore {
+      if (environmentId) {
+         const environmentStore =
+            this.environmentManifestStores.get(environmentId);
+         if (environmentStore) {
+            return environmentStore;
          }
       }
       if (!this.defaultManifestStore) {
@@ -199,7 +235,7 @@ export class StorageManager {
          this.duckDbConnection = null;
          this.repository = null;
          this.defaultManifestStore = null;
-         this.projectManifestStores.clear();
+         this.environmentManifestStores.clear();
          this.attachedCatalogs.clear();
       }
    }

package/src/storage/duckdb/ConnectionRepository.ts

@@ -12,11 +12,11 @@ export class ConnectionRepository {
       return new Date();
    }
 
-   async listConnections(projectId: string): Promise<Connection[]> {
+   async listConnections(environmentId: string): Promise<Connection[]> {
       try {
          const rows = await this.db.all<Record<string, unknown>>(
-            "SELECT * FROM connections WHERE project_id = ? ORDER BY name",
-            [projectId],
+            "SELECT * FROM connections WHERE environment_id = ? ORDER BY name",
+            [environmentId],
          );
          return rows.map(this.mapToConnection);
       } catch (err: unknown) {
@@ -35,12 +35,12 @@ export class ConnectionRepository {
    }
 
    async getConnectionByName(
-      projectId: string,
+      environmentId: string,
       name: string,
    ): Promise<Connection | null> {
       const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM connections WHERE project_id = ? AND name = ?",
-         [projectId, name],
+         "SELECT * FROM connections WHERE environment_id = ? AND name = ?",
+         [environmentId, name],
       );
       return row ? this.mapToConnection(row) : null;
    }
@@ -55,11 +55,11 @@ export class ConnectionRepository {
          const configJson = JSON.stringify(connection.config);
 
          await this.db.run(
-            `INSERT INTO connections (id, project_id, name, type, config, created_at, updated_at)
+            `INSERT INTO connections (id, environment_id, name, type, config, created_at, updated_at)
         VALUES (?, ?, ?, ?, ?, ?, ?)`,
             [
                id,
-               connection.projectId,
+               connection.environmentId,
                connection.name,
                connection.type,
                configJson,
@@ -123,14 +123,16 @@ export class ConnectionRepository {
       await this.db.run("DELETE FROM connections WHERE id = ?", [id]);
    }
 
-   async deleteConnectionsByProjectId(id: string): Promise<void> {
-      await this.db.run("DELETE FROM connections WHERE project_id = ?", [id]);
+   async deleteConnectionsByEnvironmentId(id: string): Promise<void> {
+      await this.db.run("DELETE FROM connections WHERE environment_id = ?", [
+         id,
+      ]);
    }
 
    private mapToConnection(row: Record<string, unknown>): Connection {
       return {
          id: row.id as string,
-         projectId: row.project_id as string,
+         environmentId: row.environment_id as string,
          name: row.name as string,
          type: row.type as Connection["type"],
          config: JSON.parse(row.config as string),

package/src/storage/duckdb/DuckDBConnection.ts

@@ -93,7 +93,7 @@ export class DuckDBConnection implements DatabaseConnection {
       return this.mutex.runExclusive(async () => {
          return new Promise<boolean>((resolve) => {
             this.connection!.all(
-               "SELECT name FROM sqlite_master WHERE type='table' AND name='projects'",
+               "SELECT name FROM sqlite_master WHERE type='table' AND name='environments'",
                (err, rows) => {
                   if (err) {
                      resolve(false);

package/src/storage/duckdb/DuckDBManifestStore.ts

@@ -22,11 +22,11 @@ export class DuckDBManifestStore implements ManifestStore {
     * reference has no manifest entry (e.g. before the first materialization).
     */
    async getManifest(
-      projectId: string,
+      environmentId: string,
       packageName: string,
    ): Promise<BuildManifest> {
       const entries = await this.repository.listManifestEntries(
-         projectId,
+         environmentId,
          packageName,
       );
       const manifest: BuildManifest = {
@@ -40,7 +40,7 @@ export class DuckDBManifestStore implements ManifestStore {
    }
 
    async writeEntry(
-      projectId: string,
+      environmentId: string,
       packageName: string,
       buildId: string,
       tableName: string,
@@ -48,7 +48,7 @@ export class DuckDBManifestStore implements ManifestStore {
       connectionName: string,
    ): Promise<void> {
       await this.repository.upsertManifestEntry({
-         projectId,
+         environmentId,
          packageName,
          buildId,
          tableName,
@@ -62,9 +62,9 @@ export class DuckDBManifestStore implements ManifestStore {
    }
 
    async listEntries(
-      projectId: string,
+      environmentId: string,
       packageName: string,
    ): Promise<ManifestEntry[]> {
-      return this.repository.listManifestEntries(projectId, packageName);
+      return this.repository.listManifestEntries(environmentId, packageName);
    }
 }