@malloy-publisher/server 0.0.198-dev → 0.0.198-dev1

package/src/service/package_race.spec.ts

@@ -0,0 +1,208 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import * as fs from "fs/promises";
+import * as os from "os";
+import * as path from "path";
+import { Environment } from "./environment";
+
+/**
+ * Race-condition regression tests for the package-directory pipeline.
+ *
+ * Three tests, all deterministic without timing-based flakiness:
+ *
+ *  1. **Behavioral race repro** — concurrently install (rewrite the
+ *     package directory) and read (`getModelFileText`); assert no
+ *     `ENOENT` is observed. On the pre-fix code, the read would fail
+ *     mid-rewrite. With the per-package mutex now covering both paths,
+ *     all reads succeed.
+ *
+ *  2. **Mutex coverage** — manually hold `withPackageLock` and assert
+ *     that a concurrent reader is pending until released. Pins the
+ *     invariant that readers actually take the lock.
+ *
+ *  3. **Download does not block compile** — start an `installPackage`
+ *     whose downloader never resolves on its own, then assert that
+ *     `getModelFileText` resolves promptly. This pins the Phase 1 /
+ *     Phase 2 split — if a future regression accidentally moves the
+ *     download inside the lock, this test fails.
+ */
+describe("package directory race", () => {
+   let rootDir: string;
+   let envPath: string;
+   let fixtureDir: string;
+
+   const PUBLISHER_JSON = JSON.stringify({
+      name: "pkg",
+      description: "race-test fixture",
+   });
+   const MODEL_MALLOY = `source: ones is duckdb.sql("SELECT 1 as x")\n`;
+
+   async function writeFixture(targetDir: string): Promise<void> {
+      await fs.mkdir(targetDir, { recursive: true });
+      await fs.writeFile(
+         path.join(targetDir, "publisher.json"),
+         PUBLISHER_JSON,
+      );
+      await fs.writeFile(path.join(targetDir, "model.malloy"), MODEL_MALLOY);
+   }
+
+   async function copyDir(src: string, dst: string): Promise<void> {
+      await fs.mkdir(dst, { recursive: true });
+      await fs.cp(src, dst, { recursive: true });
+   }
+
+   beforeEach(async () => {
+      rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-race-"));
+      envPath = path.join(rootDir, "env");
+      fixtureDir = path.join(rootDir, "fixture");
+      await fs.mkdir(envPath, { recursive: true });
+      await writeFixture(fixtureDir);
+   });
+
+   afterEach(async () => {
+      await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
+   });
+
+   it("(A) concurrent installs and reads never observe a half-rewritten tree", async () => {
+      const env = await Environment.create("testEnv", envPath, []);
+
+      // Initial install to populate the canonical path.
+      await env.installPackage("pkg", (stagingPath) =>
+         copyDir(fixtureDir, stagingPath),
+      );
+
+      const ITERATIONS = 30;
+      const errors: unknown[] = [];
+      let mutatorDone = false;
+
+      // Mutator loop: re-install the package over and over. Each iteration
+      // exercises the full Phase 1 (no-lock) + Phase 2 (locked) swap.
+      const mutator = (async () => {
+         try {
+            for (let i = 0; i < ITERATIONS; i++) {
+               try {
+                  await env.installPackage("pkg", (stagingPath) =>
+                     copyDir(fixtureDir, stagingPath),
+                  );
+               } catch (err) {
+                  errors.push({ kind: "install", err });
+               }
+            }
+         } finally {
+            mutatorDone = true;
+         }
+      })();
+
+      // Reader loop: hammer `getModelFileText` while installs run. On the
+      // pre-fix code (no lock on reads), the read would sometimes hit ENOENT
+      // because the canonical dir was momentarily missing during the rename
+      // window. With the per-package mutex covering reads as well, this
+      // window is never observable.
+      const reader = (async () => {
+         while (!mutatorDone) {
+            try {
+               const text = await env.getModelFileText("pkg", "model.malloy");
+               expect(text).toBe(MODEL_MALLOY);
+            } catch (err) {
+               errors.push({ kind: "read", err });
+            }
+         }
+      })();
+
+      await mutator;
+      await reader;
+
+      // Any error here means the lock wasn't actually covering one of the
+      // sides — that's the regression we're guarding against.
+      if (errors.length > 0) {
+         throw new Error(
+            `Observed ${errors.length} race-window error(s): ${JSON.stringify(
+               errors.slice(0, 3),
+               (_k, v) => (v instanceof Error ? `${v.name}: ${v.message}` : v),
+            )}`,
+         );
+      }
+   }, 60_000);
+
+   it("(B) compile-time disk reads queue behind withPackageLock", async () => {
+      const env = await Environment.create("testEnv", envPath, []);
+      await env.installPackage("pkg", (stagingPath) =>
+         copyDir(fixtureDir, stagingPath),
+      );
+
+      const lockEntered = defer<void>();
+      const releaseLock = defer<void>();
+
+      // Hold the per-package mutex from "outside" — simulates a mutator
+      // (install / delete / writePackageManifest) being in flight.
+      const lockHolder = env.withPackageLock("pkg", async () => {
+         lockEntered.resolve();
+         await releaseLock.promise;
+      });
+
+      await lockEntered.promise;
+
+      // While the lock is held, the reader must NOT make progress.
+      const readPromise = env.getModelFileText("pkg", "model.malloy");
+      const TIMEOUT_SENTINEL = Symbol("timeout");
+      const raced = await Promise.race([
+         readPromise,
+         new Promise<typeof TIMEOUT_SENTINEL>((resolve) =>
+            setTimeout(() => resolve(TIMEOUT_SENTINEL), 50),
+         ),
+      ]);
+      expect(raced).toBe(TIMEOUT_SENTINEL);
+
+      // Release the lock; the reader must now complete.
+      releaseLock.resolve();
+      await lockHolder;
+      const text = await readPromise;
+      expect(text).toBe(MODEL_MALLOY);
+   }, 15_000);
+
+   it("(C) a slow download does not block concurrent reads", async () => {
+      const env = await Environment.create("testEnv", envPath, []);
+      // Initial install to make the package present.
+      await env.installPackage("pkg", (stagingPath) =>
+         copyDir(fixtureDir, stagingPath),
+      );
+
+      const downloadGate = defer<void>();
+
+      // Kick off an install whose Phase 1 downloader stalls until we open
+      // the gate. Phase 2 (the brief locked swap) cannot run until then.
+      const slowInstall = env.installPackage("pkg", async (stagingPath) => {
+         await downloadGate.promise;
+         await copyDir(fixtureDir, stagingPath);
+      });
+
+      // The reader must resolve well before we open the gate, proving the
+      // per-package mutex is NOT held during Phase 1.
+      const readStart = Date.now();
+      const text = await env.getModelFileText("pkg", "model.malloy");
+      const readElapsedMs = Date.now() - readStart;
+
+      expect(text).toBe(MODEL_MALLOY);
+      // 1s is generous; in practice this resolves in single-digit ms.
+      expect(readElapsedMs).toBeLessThan(1_000);
+
+      // Now open the gate and let the install complete.
+      downloadGate.resolve();
+      await slowInstall;
+   }, 15_000);
+});
+
+interface Deferred<T> {
+   promise: Promise<T>;
+   resolve: (value: T) => void;
+   reject: (reason?: unknown) => void;
+}
+
+function defer<T>(): Deferred<T> {
+   let resolve!: (value: T) => void;
+   let reject!: (reason?: unknown) => void;
+   const promise = new Promise<T>((res, rej) => {
+      resolve = res;
+      reject = rej;
+   });
+   return { promise, resolve, reject };
+}

package/src/storage/StorageManager.ts

@@ -1,5 +1,13 @@
+import { Mutex } from "async-mutex";
 import * as crypto from "crypto";
+import { ConnectionAuthError } from "../errors";
 import { logger } from "../logger";
+import {
+   handlePgAttachError,
+   pgConnectTimeoutSeconds,
+   redactPgSecrets,
+   withPgConnectTimeout,
+} from "../pg_helpers";
 import {
    DatabaseConnection,
    ManifestStore,
@@ -78,6 +86,13 @@ export class StorageManager {
     */
    private attachedCatalogs = new Map<string, string>();
 
+   // Serializes DuckLake catalog attaches. Concurrent POST /environments calls
+   // hitting the same DuckDB connection would otherwise race on extension
+   // autoload (httpfs/azure/etc.), where multiple connections download the
+   // extension to `.tmp-<uuid>` files in parallel; only one wins the rename
+   // and the rest crash with "Could not remove file ... No such file or directory".
+   private duckLakeAttachMutex: Mutex = new Mutex();
+
    private config: StorageConfig;
 
    constructor(config: StorageConfig) {
@@ -141,14 +156,18 @@ export class StorageManager {
       }
 
       const key = configKey(config);
-      let catalogName = this.attachedCatalogs.get(key);
-      if (!catalogName) {
-         // Catalog name derived from the config so multiple configs can coexist as
-         // separate ATTACHments without colliding on the name.
-         catalogName = catalogNameForConfig(config);
-         await this.attachDuckLakeCatalog(config, catalogName);
-         this.attachedCatalogs.set(key, catalogName);
-      }
+      const catalogName = await this.duckLakeAttachMutex.runExclusive(
+         async () => {
+            const existing = this.attachedCatalogs.get(key);
+            if (existing) return existing;
+            // Catalog name derived from the config so multiple configs can coexist as
+            // separate ATTACHments without colliding on the name.
+            const name = catalogNameForConfig(config);
+            await this.attachDuckLakeCatalog(config, name);
+            this.attachedCatalogs.set(key, name);
+            return name;
+         },
+      );
 
       const store = new DuckLakeManifestStore(
          this.duckDbConnection,
@@ -178,12 +197,31 @@ export class StorageManager {
          await connection.run("INSTALL postgres; LOAD postgres;");
       }
 
-      const escapedCatalogUrl = escapeSQL(config.catalogUrl);
+      // For PG-backed catalogs, inject connect_timeout so a wedged libpq
+      // handshake fails the caller in seconds rather than hanging the
+      // worker until the K8s liveness probe trips (the 2026-05 incident).
+      // Non-PG catalogs (e.g. SQLite, MySQL) pass through unchanged.
+      const catalogUrl = isPostgres
+         ? withPgConnectTimeout(config.catalogUrl, pgConnectTimeoutSeconds())
+         : config.catalogUrl;
+
+      const escapedCatalogUrl = escapeSQL(catalogUrl);
       const escapedDataPath = escapeSQL(config.dataPath);
       const isCloudStorage =
          config.dataPath.startsWith("gs://") ||
          config.dataPath.startsWith("s3://");
 
+      // Pre-install httpfs explicitly so the ATTACH below doesn't trigger
+      // DuckDB's autoloader. The autoloader downloads extensions to
+      // `<ext>.tmp-<uuid>` and races when multiple connections within the
+      // same process hit it concurrently — losers fail with
+      // "Could not remove file ... No such file or directory" on cleanup
+      // of their .tmp file. INSTALL/LOAD here is idempotent and serialized
+      // by the caller's mutex.
+      if (isCloudStorage) {
+         await connection.run("INSTALL httpfs; LOAD httpfs;");
+      }
+
       let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
       const attachOpts: string[] = [
          `DATA_PATH '${escapedDataPath}'`,
@@ -193,13 +231,35 @@ export class StorageManager {
          // sidestepping object-storage auth issues entirely for this path.
          "DATA_INLINING_ROW_LIMIT 100000",
       ];
+
       if (isCloudStorage) {
          attachOpts.push("OVERRIDE_DATA_PATH true");
       }
       attachCmd += ` (${attachOpts.join(", ")});`;
 
-      logger.info(`Attaching DuckLake manifest catalog: ${attachCmd}`);
-      await connection.run(attachCmd);
+      logger.info(
+         `Attaching DuckLake manifest catalog: ${redactPgSecrets(attachCmd)}`,
+      );
+      try {
+         await connection.run(attachCmd);
+      } catch (error) {
+         const outcome = handlePgAttachError(
+            error,
+            `DuckLake catalog credentials rejected for ${catalogName}`,
+         );
+         if (outcome.action === "swallow") {
+            logger.info(
+               `DuckLake catalog ${catalogName} is already attached, skipping`,
+            );
+            return;
+         }
+         if (outcome.error instanceof ConnectionAuthError) {
+            logger.warn("DuckLake catalog credentials rejected", {
+               catalogName,
+            });
+         }
+         throw outcome.error;
+      }
    }
 
    getRepository(): ResourceRepository {

package/src/storage/duckdb/schema.ts

@@ -17,6 +17,15 @@ export async function initializeSchema(
       );
       await dropAllTables(db);
    } else {
+      // TODO: Remove this during projects cleanup
+      // If a pre-rename `projects` schema is on disk, the new
+      // CREATE TABLE IF NOT EXISTS pass below would silently leave child
+      // tables on the old `project_id` column and the first query against
+      // `environment_id` would crash. Drop the legacy tables (with a loud
+      // warning) so the fresh schema can be created cleanly. This is
+      // destructive — operators upgrading should re-create their environments
+      // and packages via the API after the upgrade.
+      await dropLegacyProjectSchema(db);
       logger.info("Creating database schema for the first time...");
    }
 
@@ -125,6 +134,38 @@ export async function initializeSchema(
    );
 }
 
+// TODO: Remove this during projects cleanup
+// Tables in the pre-rename schema, listed children-first so DROP order
+// satisfies foreign-key dependencies on the legacy `projects` table.
+const LEGACY_TABLES_DROP_ORDER = [
+   "build_manifests",
+   "materializations",
+   "packages",
+   "connections",
+   "projects",
+] as const;
+
+async function dropLegacyProjectSchema(db: DuckDBConnection): Promise<void> {
+   const legacy = await db.all<{ name: string }>(
+      "SELECT name FROM sqlite_master WHERE type='table' AND name='projects'",
+   );
+   if (!legacy || legacy.length === 0) {
+      return;
+   }
+
+   logger.warn(
+      "Detected legacy 'projects' schema. Dropping legacy tables; existing environments/packages/connections/materializations data will be lost. Re-create them via the API after upgrade.",
+   );
+
+   for (const table of LEGACY_TABLES_DROP_ORDER) {
+      try {
+         await db.run(`DROP TABLE IF EXISTS ${table}`);
+      } catch (err) {
+         logger.warn(`Failed to drop legacy table ${table}:`, err);
+      }
+   }
+}
+
 async function dropAllTables(db: DuckDBConnection): Promise<void> {
    const tables = [
       "build_manifests",

package/src/utils.ts

@@ -1,5 +1,6 @@
 import { URLReader } from "@malloydata/malloy";
 import * as fs from "fs";
+import * as path from "path";
 import { fileURLToPath } from "url";
 
 export const URL_READER: URLReader = {
@@ -11,3 +12,13 @@ export const URL_READER: URLReader = {
       return fs.promises.readFile(path, "utf8");
    },
 };
+
+/**
+ * Skip dotfiles/dotdirs (.vscode, .git, .DS_Store, etc.) when walking a
+ * package tree. These come from editors/VCS, never contain Malloy models
+ * or databases, and have been a source of spurious ENOENTs when their
+ * contents disappear mid-scan.
+ */
+export function ignoreDotfiles(file: string): boolean {
+   return path.basename(file).startsWith(".");
+}

package/tests/harness/rest_e2e.ts

@@ -12,8 +12,8 @@ export interface RestE2EEnv {
  * reuses the cached Express app and binds on an OS-assigned port
  * to avoid collisions.
  *
- * Callers are responsible for creating any test-specific projects
- * via the REST API (POST /api/v0/projects) and cleaning them up.
+ * Callers are responsible for creating any test-specific environments
+ * via the REST API (POST /api/v0/environments) and cleaning them up.
  */
 export async function startRestE2E(): Promise<
    RestE2EEnv & { stop(): Promise<void> }