@malloy-publisher/server 0.0.197-dev → 0.0.197

package/src/server.ts

@@ -36,6 +36,7 @@ import { logger, loggerMiddleware } from "./logger";
 import { ManifestController } from "./controller/manifest.controller";
 import { MaterializationController } from "./controller/materialization.controller";
 import { initializeMcpServer } from "./mcp/server";
+import { registerLegacyRoutes } from "./server-old";
 import { EnvironmentStore } from "./service/environment_store";
 import { ManifestService } from "./service/manifest_service";
 import { MaterializationService } from "./service/materialization_service";
@@ -50,6 +51,8 @@ export function normalizeQueryArray(value: unknown): string[] | undefined {
 // Parse command line arguments
 function parseArgs() {
    const args = process.argv.slice(2);
+   let sawServerRoot = false;
+   let sawConfig = false;
    for (let i = 0; i < args.length; i++) {
       const arg = args[i];
       if (arg === "--port" && args[i + 1]) {
@@ -59,8 +62,13 @@ function parseArgs() {
          process.env.PUBLISHER_HOST = args[i + 1];
          i++;
       } else if (arg === "--server_root" && args[i + 1]) {
+         sawServerRoot = true;
          process.env.SERVER_ROOT = args[i + 1];
          i++;
+      } else if (arg === "--config" && args[i + 1]) {
+         sawConfig = true;
+         process.env.PUBLISHER_CONFIG_PATH = args[i + 1];
+         i++;
       } else if (arg === "--mcp_port" && args[i + 1]) {
          process.env.MCP_PORT = args[i + 1];
          i++;
@@ -90,6 +98,9 @@ function parseArgs() {
          console.log(
             "  --server_root <path>   Root directory to serve files from (default: .)",
          );
+         console.log(
+            "  --config <path>        Path to publisher.config.json (default: <server_root>/publisher.config.json; falls back to bundled DuckDB-only sample config if missing)",
+         );
          console.log(
             "  --mcp_port <number>    Port for MCP server (default: 4040)",
          );
@@ -106,6 +117,16 @@ function parseArgs() {
          process.exit(0);
       }
    }
+   // Zero-config invocation (`npx @malloy-publisher/server`) opts in to
+   // the bundled DuckDB-only sample config so the Quick Start works
+   // without any flags. Any explicit --server_root or --config disables
+   // this — the user told us where to look. Skip in NODE_ENV=test so
+   // specs that import this module for utility helpers (e.g.
+   // db_utils.spec.ts -> normalizeQueryArray) don't get the bundled
+   // default leaked into their EnvironmentStore construction.
+   if (!sawServerRoot && !sawConfig && process.env.NODE_ENV !== "test") {
+      process.env.PUBLISHER_USE_BUNDLED_DEFAULT = "true";
+   }
 }
 
 // Parse CLI arguments before setting up constants
@@ -1364,6 +1385,21 @@ app.post(
    },
 );
 
+// Register legacy `/projects/...` routes for backwards compatibility with
+// clients that haven't migrated to `/environments/...` yet. Must be added
+// before the SPA catch-all below.
+registerLegacyRoutes(app, {
+   environmentStore,
+   connectionController,
+   modelController,
+   packageController,
+   databaseController,
+   queryController,
+   compileController,
+   materializationController,
+   manifestController,
+});
+
 // Modify the catch-all route to only serve index.html in production
 if (!isDevelopment) {
    app.get("*", (_req, res) => res.sendFile(path.resolve(ROOT, "index.html")));

package/src/service/connection.spec.ts

@@ -1129,10 +1129,14 @@ describe("connection integration tests", () => {
                   ],
                   testEnvironmentPath,
                ),
-            ).rejects.toThrow(/cannot be 'duckdb'/);
+            ).rejects.toThrow(/'duckdb' is reserved/);
          });
 
          it("should reject DuckDB connections with no attachments", async () => {
+            // Env-level DuckDB connections must declare at least one
+            // attached foreign database; the empty-array case is operator
+            // confusion (the per-package "duckdb" sandbox already covers
+            // the plain-in-memory use case).
             await expect(
                createEnvironmentConnections(
                   [
@@ -1144,9 +1148,7 @@ describe("connection integration tests", () => {
                   ],
                   testEnvironmentPath,
                ),
-            ).rejects.toThrow(
-               "DuckDB connection must have at least one attached database",
-            );
+            ).rejects.toThrow(/has no attached databases/);
          });
 
          it("should reject unsupported DuckDB connector fields", async () => {

package/src/service/connection.ts

@@ -25,6 +25,7 @@ import fs from "fs/promises";
 import path from "path";
 import { components } from "../api";
 import { logAxiosError, logger } from "../logger";
+import { redactPgSecrets } from "../pg_helpers";
 import {
    assembleEnvironmentConnections,
    CoreConnectionEntry,
@@ -365,13 +366,17 @@ async function attachDuckLake(
    const pgConnString: string = buildPgConnectionString(pg);
    // Attach DuckLake with Postgres catalog and cloud storage data path in READ_ONLY mode
    // The client manages metadata - we only read from the catalogs
-   logger.info(`pgConnString: ${pgConnString}`);
+   logger.info(`pgConnString: ${redactPgSecrets(pgConnString)}`);
    const escapedPgConnString = escapeSQL(pgConnString);
-   logger.info(`Final escaped connection string: ${escapedPgConnString}`);
+   logger.info(
+      `Final escaped connection string: ${redactPgSecrets(escapedPgConnString)}`,
+   );
    const escapedBucketUrl = escapeSQL(ducklakeConfig.storage.bucketUrl);
    logger.info(`escapedBucketUrl: ${escapedBucketUrl}`);
    const attachCommand = `ATTACH OR REPLACE 'ducklake:postgres:${escapedPgConnString}' AS ${dbName} (DATA_PATH '${escapedBucketUrl}', OVERRIDE_DATA_PATH true, READ_ONLY true);`;
-   logger.info(`Attaching DuckLake database using command: ${attachCommand}`);
+   logger.info(
+      `Attaching DuckLake database using command: ${redactPgSecrets(attachCommand)}`,
+   );
    try {
       await connection.runSQL(attachCommand);
       logger.info(

package/src/service/connection_config.ts

@@ -272,7 +272,7 @@ function validateConnectionShape(connection: ApiConnection): void {
                connection.duckdbConnection.attachedDatabases ?? [];
             if (attached.length === 0) {
                throw new Error(
-                  "DuckDB connection must have at least one attached database",
+                  `DuckDB connection "${connection.name}" has no attached databases. Add at least one foreign database (BigQuery, Snowflake, Postgres, GCS, S3, Azure) to attachedDatabases, or remove this connection entirely — each package already gets a per-package DuckDB sandbox named "duckdb" automatically.`,
                );
             }
          }
@@ -359,7 +359,7 @@ export function assembleEnvironmentConnections(
 
       if (connection.name === "duckdb") {
          throw new Error(
-            "DuckDB connection name cannot be 'duckdb'; it is reserved for Publisher package sandboxes.",
+            "Connection name 'duckdb' is reserved for per-package sandboxes. Choose a different name for environment-level DuckDB connections (e.g. 'shared_duckdb').",
          );
       }
 

package/src/service/environment.ts

@@ -389,6 +389,16 @@ export class Environment {
       }
    }
 
+   /** One mutex per package name; never replace after create (avoids parallel loads). */
+   private getOrCreatePackageMutex(packageName: string): Mutex {
+      let packageMutex = this.packageMutexes.get(packageName);
+      if (packageMutex === undefined) {
+         packageMutex = new Mutex();
+         this.packageMutexes.set(packageName, packageMutex);
+      }
+      return packageMutex;
+   }
+
    public async getPackage(
       packageName: string,
       reload: boolean = false,
@@ -399,25 +409,23 @@ export class Environment {
          return _package;
       }
 
-      // We need to acquire the mutex to prevent a thundering herd of requests from creating the
-      // package multiple times.
-      let packageMutex = this.packageMutexes.get(packageName);
-      if (packageMutex?.isLocked()) {
+      // Serialize load per package name so concurrent callers share one Mutex and
+      // failed loads cannot rm the tree while another load is still scanning it.
+      const packageMutex = this.getOrCreatePackageMutex(packageName);
+
+      if (packageMutex.isLocked()) {
          logger.debug(
             `Package ${packageName} is being loaded, waiting for unlock...`,
          );
          await packageMutex.waitForUnlock();
          logger.debug(`Package ${packageName} unlocked`);
          const existingPackage = this.packages.get(packageName);
-         if (existingPackage) {
+         if (existingPackage !== undefined && !reload) {
             logger.debug(`Package ${packageName} loaded by another request`);
             return existingPackage;
          }
-         // If package still doesn't exist after unlock, it might have failed to load
-         // Continue to try loading it ourselves
+         // Reload, or prior load failed — continue under the same mutex.
       }
-      packageMutex = new Mutex();
-      this.packageMutexes.set(packageName, packageMutex);
 
       return packageMutex.runExclusive(async () => {
          // Double-check after acquiring mutex
@@ -479,24 +487,44 @@ export class Environment {
             malloyConfig: this.malloyConfig.malloyConfig,
          },
       );
-      this.setPackageStatus(packageName, PackageStatus.LOADING);
-      try {
-         this.packages.set(
-            packageName,
-            await Package.create(
-               this.environmentName,
-               packageName,
-               packagePath,
-               () => this.malloyConfig.malloyConfig,
-            ),
+
+      const packageMutex = this.getOrCreatePackageMutex(packageName);
+      if (packageMutex.isLocked()) {
+         logger.debug(
+            `Package ${packageName} is being loaded, waiting before addPackage...`,
          );
-      } catch (error) {
-         logger.error("Error adding package", { error });
-         this.deletePackageStatus(packageName);
-         throw error;
+         await packageMutex.waitForUnlock();
+         const alreadyLoaded = this.packages.get(packageName);
+         if (alreadyLoaded !== undefined) {
+            return alreadyLoaded;
+         }
       }
-      this.setPackageStatus(packageName, PackageStatus.SERVING);
-      return this.packages.get(packageName);
+
+      return packageMutex.runExclusive(async () => {
+         const existingPackage = this.packages.get(packageName);
+         if (existingPackage !== undefined) {
+            return existingPackage;
+         }
+
+         this.setPackageStatus(packageName, PackageStatus.LOADING);
+         try {
+            this.packages.set(
+               packageName,
+               await Package.create(
+                  this.environmentName,
+                  packageName,
+                  packagePath,
+                  () => this.malloyConfig.malloyConfig,
+               ),
+            );
+         } catch (error) {
+            logger.error("Error adding package", { error });
+            this.deletePackageStatus(packageName);
+            throw error;
+         }
+         this.setPackageStatus(packageName, PackageStatus.SERVING);
+         return this.packages.get(packageName);
+      });
    }
 
    private async writePackageManifest(

package/src/service/environment_store.spec.ts

@@ -355,6 +355,15 @@ describe("EnvironmentStore Service", () => {
       expect(projects.length).toBe(2);
       expect(projects.map((p) => p.name)).toContain(projectName1);
       expect(projects.map((p) => p.name)).toContain(projectName2);
+
+      // All envs initialized cleanly → status is "serving" (not
+      // "degraded") and there's no failedEnvironments key on the
+      // response. This is the happy-path companion to the
+      // "should skip a project with invalid startup connection config"
+      // test which exercises the degraded path.
+      const status = await newEnvironmentStore.getStatus();
+      expect(status.operationalState).toBe("serving");
+      expect(status.failedEnvironments).toBeUndefined();
    });
 
    it("should skip a project with invalid startup connection config", async () => {
@@ -427,6 +436,16 @@ describe("EnvironmentStore Service", () => {
       await expect(
          newEnvironmentStore.getEnvironment(invalidProjectName),
       ).rejects.toThrow();
+
+      // The skipped environment should surface in the status response
+      // so external callers (CI smoke tests, dashboards) can tell the
+      // server is only partially serving.
+      const status = await newEnvironmentStore.getStatus();
+      expect(status.operationalState).toBe("degraded");
+      expect(status.failedEnvironments).toBeDefined();
+      expect(status.failedEnvironments?.map((f) => f.name)).toContain(
+         invalidProjectName,
+      );
    });
 
    it("should handle project updates", async () => {

package/src/service/environment_store.ts

@@ -25,7 +25,12 @@ import {
    FrozenConfigError,
    PackageNotFoundError,
 } from "../errors";
-import { getOperationalState, markNotReady, markReady } from "../health";
+import {
+   getOperationalState,
+   markDegraded,
+   markNotReady,
+   markReady,
+} from "../health";
 import { formatDuration, logger } from "../logger";
 import { Connection } from "../storage/DatabaseInterface";
 import { StorageConfig, StorageManager } from "../storage/StorageManager";
@@ -96,6 +101,7 @@ export class EnvironmentStore {
    public publisherConfigIsFrozen: boolean;
    public finishedInitialization: Promise<void>;
    private isInitialized: boolean = false;
+   private failedEnvironments: Array<{ name: string; error: string }> = [];
    public storageManager: StorageManager;
    private s3Client = new S3({
       followRegionRedirects: true,
@@ -142,6 +148,10 @@ export class EnvironmentStore {
          `Error initializing environment${label}; skipping environment`,
          this.extractErrorDataFromError(error),
       );
+      this.failedEnvironments.push({
+         name: environmentName ?? "<unknown>",
+         error: error instanceof Error ? error.message : String(error),
+      });
    }
 
    private async initialize() {
@@ -275,7 +285,11 @@ export class EnvironmentStore {
          }
 
          this.isInitialized = true;
-         markReady();
+         if (this.failedEnvironments.length > 0) {
+            markDegraded();
+         } else {
+            markReady();
+         }
          const initializationDuration = performance.now() - initialTime;
          logger.info(
             `Environment store successfully initialized in ${formatDuration(initializationDuration)}`,
@@ -689,6 +703,11 @@ export class EnvironmentStore {
          frozenConfig: isPublisherConfigFrozen(this.serverRootPath),
          operationalState:
             getOperationalState() as components["schemas"]["ServerStatus"]["operationalState"],
+         ...(this.failedEnvironments.length > 0 && {
+            failedEnvironments: [
+               ...this.failedEnvironments,
+            ] as components["schemas"]["ServerStatus"]["failedEnvironments"],
+         }),
       };
 
       const environments = await this.listEnvironments(true);

package/src/service/package.ts

@@ -24,6 +24,7 @@ import {
 import { PackageNotFoundError } from "../errors";
 import { formatDuration, logger } from "../logger";
 import { BuildManifest } from "../storage/DatabaseInterface";
+import { ignoreDotfiles } from "../utils";
 import { Model } from "./model";
 
 type ApiDatabase = components["schemas"]["Database"];
@@ -42,6 +43,7 @@ type PackageConnectionInput =
    | (() => MalloyConfig);
 
 const ENABLE_LIST_MODEL_COMPILATION = true;
+
 export class Package {
    private environmentName: string;
    private packageName: string;
@@ -380,7 +382,7 @@ export class Package {
    private static async getModelPaths(packagePath: string): Promise<string[]> {
       let files = undefined;
       try {
-         files = await recursive(packagePath);
+         files = await recursive(packagePath, [ignoreDotfiles]);
       } catch (error) {
          logger.error(error);
          throw new PackageNotFoundError(
@@ -449,8 +451,7 @@ export class Package {
    private static async getDatabasePaths(
       packagePath: string,
    ): Promise<string[]> {
-      let files = undefined;
-      files = await recursive(packagePath);
+      const files = await recursive(packagePath, [ignoreDotfiles]);
       return files
          .map((fullPath: string) => {
             return path.relative(packagePath, fullPath).replace(/\\/g, "/");

package/src/storage/StorageManager.ts

@@ -1,5 +1,13 @@
+import { Mutex } from "async-mutex";
 import * as crypto from "crypto";
+import { ConnectionAuthError } from "../errors";
 import { logger } from "../logger";
+import {
+   handlePgAttachError,
+   pgConnectTimeoutSeconds,
+   redactPgSecrets,
+   withPgConnectTimeout,
+} from "../pg_helpers";
 import {
    DatabaseConnection,
    ManifestStore,
@@ -78,6 +86,13 @@ export class StorageManager {
     */
    private attachedCatalogs = new Map<string, string>();
 
+   // Serializes DuckLake catalog attaches. Concurrent POST /environments calls
+   // hitting the same DuckDB connection would otherwise race on extension
+   // autoload (httpfs/azure/etc.), where multiple connections download the
+   // extension to `.tmp-<uuid>` files in parallel; only one wins the rename
+   // and the rest crash with "Could not remove file ... No such file or directory".
+   private duckLakeAttachMutex: Mutex = new Mutex();
+
    private config: StorageConfig;
 
    constructor(config: StorageConfig) {
@@ -141,14 +156,18 @@ export class StorageManager {
       }
 
       const key = configKey(config);
-      let catalogName = this.attachedCatalogs.get(key);
-      if (!catalogName) {
-         // Catalog name derived from the config so multiple configs can coexist as
-         // separate ATTACHments without colliding on the name.
-         catalogName = catalogNameForConfig(config);
-         await this.attachDuckLakeCatalog(config, catalogName);
-         this.attachedCatalogs.set(key, catalogName);
-      }
+      const catalogName = await this.duckLakeAttachMutex.runExclusive(
+         async () => {
+            const existing = this.attachedCatalogs.get(key);
+            if (existing) return existing;
+            // Catalog name derived from the config so multiple configs can coexist as
+            // separate ATTACHments without colliding on the name.
+            const name = catalogNameForConfig(config);
+            await this.attachDuckLakeCatalog(config, name);
+            this.attachedCatalogs.set(key, name);
+            return name;
+         },
+      );
 
       const store = new DuckLakeManifestStore(
          this.duckDbConnection,
@@ -178,12 +197,31 @@ export class StorageManager {
          await connection.run("INSTALL postgres; LOAD postgres;");
       }
 
-      const escapedCatalogUrl = escapeSQL(config.catalogUrl);
+      // For PG-backed catalogs, inject connect_timeout so a wedged libpq
+      // handshake fails the caller in seconds rather than hanging the
+      // worker until the K8s liveness probe trips (the 2026-05 incident).
+      // Non-PG catalogs (e.g. SQLite, MySQL) pass through unchanged.
+      const catalogUrl = isPostgres
+         ? withPgConnectTimeout(config.catalogUrl, pgConnectTimeoutSeconds())
+         : config.catalogUrl;
+
+      const escapedCatalogUrl = escapeSQL(catalogUrl);
       const escapedDataPath = escapeSQL(config.dataPath);
       const isCloudStorage =
          config.dataPath.startsWith("gs://") ||
          config.dataPath.startsWith("s3://");
 
+      // Pre-install httpfs explicitly so the ATTACH below doesn't trigger
+      // DuckDB's autoloader. The autoloader downloads extensions to
+      // `<ext>.tmp-<uuid>` and races when multiple connections within the
+      // same process hit it concurrently — losers fail with
+      // "Could not remove file ... No such file or directory" on cleanup
+      // of their .tmp file. INSTALL/LOAD here is idempotent and serialized
+      // by the caller's mutex.
+      if (isCloudStorage) {
+         await connection.run("INSTALL httpfs; LOAD httpfs;");
+      }
+
       let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
       const attachOpts: string[] = [
          `DATA_PATH '${escapedDataPath}'`,
@@ -193,13 +231,35 @@ export class StorageManager {
          // sidestepping object-storage auth issues entirely for this path.
          "DATA_INLINING_ROW_LIMIT 100000",
       ];
+
       if (isCloudStorage) {
          attachOpts.push("OVERRIDE_DATA_PATH true");
       }
       attachCmd += ` (${attachOpts.join(", ")});`;
 
-      logger.info(`Attaching DuckLake manifest catalog: ${attachCmd}`);
-      await connection.run(attachCmd);
+      logger.info(
+         `Attaching DuckLake manifest catalog: ${redactPgSecrets(attachCmd)}`,
+      );
+      try {
+         await connection.run(attachCmd);
+      } catch (error) {
+         const outcome = handlePgAttachError(
+            error,
+            `DuckLake catalog credentials rejected for ${catalogName}`,
+         );
+         if (outcome.action === "swallow") {
+            logger.info(
+               `DuckLake catalog ${catalogName} is already attached, skipping`,
+            );
+            return;
+         }
+         if (outcome.error instanceof ConnectionAuthError) {
+            logger.warn("DuckLake catalog credentials rejected", {
+               catalogName,
+            });
+         }
+         throw outcome.error;
+      }
    }
 
    getRepository(): ResourceRepository {

package/src/storage/duckdb/schema.ts

@@ -17,6 +17,15 @@ export async function initializeSchema(
       );
       await dropAllTables(db);
    } else {
+      // TODO: Remove this during projects cleanup
+      // If a pre-rename `projects` schema is on disk, the new
+      // CREATE TABLE IF NOT EXISTS pass below would silently leave child
+      // tables on the old `project_id` column and the first query against
+      // `environment_id` would crash. Drop the legacy tables (with a loud
+      // warning) so the fresh schema can be created cleanly. This is
+      // destructive — operators upgrading should re-create their environments
+      // and packages via the API after the upgrade.
+      await dropLegacyProjectSchema(db);
       logger.info("Creating database schema for the first time...");
    }
 
@@ -125,6 +134,38 @@ export async function initializeSchema(
    );
 }
 
+// TODO: Remove this during projects cleanup
+// Tables in the pre-rename schema, listed children-first so DROP order
+// satisfies foreign-key dependencies on the legacy `projects` table.
+const LEGACY_TABLES_DROP_ORDER = [
+   "build_manifests",
+   "materializations",
+   "packages",
+   "connections",
+   "projects",
+] as const;
+
+async function dropLegacyProjectSchema(db: DuckDBConnection): Promise<void> {
+   const legacy = await db.all<{ name: string }>(
+      "SELECT name FROM sqlite_master WHERE type='table' AND name='projects'",
+   );
+   if (!legacy || legacy.length === 0) {
+      return;
+   }
+
+   logger.warn(
+      "Detected legacy 'projects' schema. Dropping legacy tables; existing environments/packages/connections/materializations data will be lost. Re-create them via the API after upgrade.",
+   );
+
+   for (const table of LEGACY_TABLES_DROP_ORDER) {
+      try {
+         await db.run(`DROP TABLE IF EXISTS ${table}`);
+      } catch (err) {
+         logger.warn(`Failed to drop legacy table ${table}:`, err);
+      }
+   }
+}
+
 async function dropAllTables(db: DuckDBConnection): Promise<void> {
    const tables = [
       "build_manifests",

package/src/utils.ts

@@ -1,5 +1,6 @@
 import { URLReader } from "@malloydata/malloy";
 import * as fs from "fs";
+import * as path from "path";
 import { fileURLToPath } from "url";
 
 export const URL_READER: URLReader = {
@@ -11,3 +12,13 @@ export const URL_READER: URLReader = {
       return fs.promises.readFile(path, "utf8");
    },
 };
+
+/**
+ * Skip dotfiles/dotdirs (.vscode, .git, .DS_Store, etc.) when walking a
+ * package tree. These come from editors/VCS, never contain Malloy models
+ * or databases, and have been a source of spurious ENOENTs when their
+ * contents disappear mid-scan.
+ */
+export function ignoreDotfiles(file: string): boolean {
+   return path.basename(file).startsWith(".");
+}

package/tests/harness/rest_e2e.ts

@@ -12,8 +12,8 @@ export interface RestE2EEnv {
  * reuses the cached Express app and binds on an OS-assigned port
  * to avoid collisions.
  *
- * Callers are responsible for creating any test-specific projects
- * via the REST API (POST /api/v0/projects) and cleaning them up.
+ * Callers are responsible for creating any test-specific environments
+ * via the REST API (POST /api/v0/environments) and cleaning them up.
  */
 export async function startRestE2E(): Promise<
    RestE2EEnv & { stop(): Promise<void> }