@malloy-publisher/server 0.0.205 → 0.0.207

package/src/storage/duckdb/ManifestRepository.ts

@@ -1,120 +0,0 @@
-import { ManifestEntry } from "../DatabaseInterface";
-import { DuckDBConnection } from "./DuckDBConnection";
-
-/**
- * Repository for build manifest entries stored in the `build_manifests` table.
- *
- * A build manifest records the materialized table produced by a specific build
- * of a package, linking it back to the Malloy source and connection that
- * generated it. Manifests are keyed by (environment, package, build) and enable
- * downstream consumers to discover which physical tables correspond to a given
- * package version.
- */
-export class ManifestRepository {
-   constructor(private db: DuckDBConnection) {}
-
-   private generateId(): string {
-      return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
-   }
-
-   /** Returns all manifest entries for a package, most recent first. */
-   async listEntries(
-      environmentId: string,
-      packageName: string,
-   ): Promise<ManifestEntry[]> {
-      const rows = await this.db.all<Record<string, unknown>>(
-         "SELECT * FROM build_manifests WHERE environment_id = ? AND package_name = ? ORDER BY created_at DESC",
-         [environmentId, packageName],
-      );
-      return rows.map(this.mapToEntry);
-   }
-
-   /** Looks up the manifest entry for a specific build, or null if none exists. */
-   async getEntryByBuildId(
-      environmentId: string,
-      packageName: string,
-      buildId: string,
-   ): Promise<ManifestEntry | null> {
-      const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM build_manifests WHERE environment_id = ? AND package_name = ? AND build_id = ?",
-         [environmentId, packageName, buildId],
-      );
-      return row ? this.mapToEntry(row) : null;
-   }
-
-   /**
-    * Inserts a new manifest entry, or updates the existing one if a row with
-    * the same (environment, package, build) triple already exists. Uses INSERT ON
-    * CONFLICT to avoid the TOCTOU race of SELECT-then-INSERT/UPDATE.
-    */
-   async upsertEntry(
-      entry: Omit<ManifestEntry, "id" | "createdAt" | "updatedAt">,
-   ): Promise<ManifestEntry> {
-      const id = this.generateId();
-      const now = new Date();
-      const iso = now.toISOString();
-
-      const rows = await this.db.all<Record<string, unknown>>(
-         `INSERT INTO build_manifests (id, environment_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
-         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-         ON CONFLICT (environment_id, package_name, build_id)
-         DO UPDATE SET table_name = EXCLUDED.table_name,
-                       source_name = EXCLUDED.source_name,
-                       connection_name = EXCLUDED.connection_name,
-                       updated_at = EXCLUDED.updated_at
-         RETURNING *`,
-         [
-            id,
-            entry.environmentId,
-            entry.packageName,
-            entry.buildId,
-            entry.tableName,
-            entry.sourceName,
-            entry.connectionName,
-            iso,
-            iso,
-         ],
-      );
-
-      return this.mapToEntry(rows[0]);
-   }
-
-   /** Deletes a single manifest entry by ID. */
-   async deleteEntry(id: string): Promise<void> {
-      await this.db.run("DELETE FROM build_manifests WHERE id = ?", [id]);
-   }
-
-   /** Removes all manifest entries belonging to an environment (used on environment deletion). */
-   async deleteEntriesByEnvironmentId(environmentId: string): Promise<void> {
-      await this.db.run(
-         "DELETE FROM build_manifests WHERE environment_id = ?",
-         [environmentId],
-      );
-   }
-
-   /** Removes all manifest entries for a specific package. */
-   async deleteEntriesByPackage(
-      environmentId: string,
-      packageName: string,
-   ): Promise<void> {
-      await this.db.run(
-         "DELETE FROM build_manifests WHERE environment_id = ? AND package_name = ?",
-         [environmentId, packageName],
-      );
-   }
-
-   /** Maps a raw DB row (snake_case columns) to a {@link ManifestEntry}. */
-   private mapToEntry(row: Record<string, unknown>): ManifestEntry {
-      return {
-         id: row.id as string,
-         environmentId: row.environment_id as string,
-         packageName: row.package_name as string,
-         buildId: row.build_id as string,
-         tableName: row.table_name as string,
-         sourceName: row.source_name as string,
-         connectionName: row.connection_name as string,
-         createdAt: new Date(row.created_at as string),
-         updatedAt: new Date(row.updated_at as string),
-      };
-   }
-}

package/src/storage/duckdb/manifest_store.spec.ts

@@ -1,133 +0,0 @@
-import { beforeEach, describe, expect, it } from "bun:test";
-import * as sinon from "sinon";
-import { ManifestEntry, ResourceRepository } from "../DatabaseInterface";
-import { DuckDBManifestStore } from "./DuckDBManifestStore";
-
-function makeEntry(overrides: Partial<ManifestEntry> = {}): ManifestEntry {
-   return {
-      id: "entry-1",
-      environmentId: "proj-1",
-      packageName: "pkg",
-      buildId: "build-abc",
-      tableName: "my_table",
-      sourceName: "my_source",
-      connectionName: "duckdb",
-      createdAt: new Date("2026-04-03"),
-      updatedAt: new Date("2026-04-03"),
-      ...overrides,
-   };
-}
-
-function createMocks() {
-   const sandbox = sinon.createSandbox();
-
-   const repository = {
-      listManifestEntries: sandbox.stub(),
-      upsertManifestEntry: sandbox.stub(),
-      deleteManifestEntry: sandbox.stub(),
-   };
-
-   const store = new DuckDBManifestStore(
-      repository as unknown as ResourceRepository,
-   );
-
-   return { sandbox, repository, store };
-}
-
-describe("DuckDBManifestStore", () => {
-   let ctx: ReturnType<typeof createMocks>;
-
-   beforeEach(() => {
-      ctx = createMocks();
-   });
-
-   describe("getManifest", () => {
-      it("should assemble a BuildManifest from repository entries", async () => {
-         ctx.repository.listManifestEntries.resolves([
-            makeEntry({ buildId: "b1", tableName: "tbl_a" }),
-            makeEntry({ buildId: "b2", tableName: "tbl_b" }),
-         ]);
-
-         const manifest = await ctx.store.getManifest("proj-1", "pkg");
-
-         expect(manifest.strict).toBe(false);
-         expect(manifest.entries).toEqual({
-            b1: { tableName: "tbl_a" },
-            b2: { tableName: "tbl_b" },
-         });
-      });
-
-      it("should return empty entries when no rows exist", async () => {
-         ctx.repository.listManifestEntries.resolves([]);
-
-         const manifest = await ctx.store.getManifest("proj-1", "pkg");
-
-         expect(manifest.strict).toBe(false);
-         expect(manifest.entries).toEqual({});
-      });
-   });
-
-   describe("writeEntry", () => {
-      it("should upsert an entry with all fields", async () => {
-         ctx.repository.upsertManifestEntry.resolves(makeEntry());
-
-         await ctx.store.writeEntry(
-            "proj-1",
-            "pkg",
-            "build-abc",
-            "tbl",
-            "src",
-            "conn",
-         );
-
-         expect(ctx.repository.upsertManifestEntry.calledOnce).toBe(true);
-         const arg = ctx.repository.upsertManifestEntry.firstCall.args[0];
-         expect(arg).toEqual({
-            environmentId: "proj-1",
-            packageName: "pkg",
-            buildId: "build-abc",
-            tableName: "tbl",
-            sourceName: "src",
-            connectionName: "conn",
-         });
-      });
-   });
-
-   describe("deleteEntry", () => {
-      it("should delegate to repository", async () => {
-         ctx.repository.deleteManifestEntry.resolves();
-
-         await ctx.store.deleteEntry("entry-1");
-
-         expect(ctx.repository.deleteManifestEntry.calledOnce).toBe(true);
-         expect(ctx.repository.deleteManifestEntry.firstCall.args[0]).toBe(
-            "entry-1",
-         );
-      });
-   });
-
-   describe("listEntries", () => {
-      it("should return entries from the repository", async () => {
-         const entries = [
-            makeEntry(),
-            makeEntry({ id: "entry-2", buildId: "build-def" }),
-         ];
-         ctx.repository.listManifestEntries.resolves(entries);
-
-         const result = await ctx.store.listEntries("proj-1", "pkg");
-
-         expect(result).toEqual(entries);
-         expect(
-            ctx.repository.listManifestEntries.calledWith("proj-1", "pkg"),
-         ).toBe(true);
-      });
-
-      it("should return empty array when no entries exist", async () => {
-         ctx.repository.listManifestEntries.resolves([]);
-
-         const result = await ctx.store.listEntries("proj-1", "pkg");
-
-         expect(result).toEqual([]);
-      });
-   });
-});

package/src/storage/ducklake/DuckLakeManifestStore.ts

@@ -1,155 +0,0 @@
-import { logger } from "../../logger";
-import {
-   BuildManifest,
-   ManifestEntry,
-   ManifestStore,
-} from "../DatabaseInterface";
-import { DuckDBConnection } from "../duckdb/DuckDBConnection";
-
-/**
- * DuckLake-backed ManifestStore used in orchestrated mode.
- *
- * Reads and writes manifest entries through a DuckLake catalog attached to
- * the publisher's internal DuckDB. All workers sharing the same DuckLake
- * catalog see the same manifest, enabling multi-worker coordination.
- *
- * The catalog is attached by {@link StorageManager} before this store is
- * instantiated. The first worker to call {@link bootstrapSchema} creates the
- * `build_manifests` table idempotently. Schema ownership lives in the
- * publisher so that DDL and code co-evolve in the same repo.
- *
- * **Scope: manifest sync only.** Orchestrated mode only shares manifest
- * state across workers. The active-materialization lock (the unique index
- * on `materializations.active_key`) still lives in each worker's local
- * DuckDB, so two workers can run builds concurrently and race on physical
- * table names, staging table names, and manifest writes. Until a shared
- * build lease lives in the DuckLake catalog, deployments running
- * orchestrated mode must ensure builds are externally single-writer (e.g.,
- * one designated build worker, or an external job scheduler). Other
- * workers should only call `manifest?action=reload` to pick up manifests produced
- * by the build worker.
- */
-export class DuckLakeManifestStore implements ManifestStore {
-   private readonly table: string;
-   private readonly environmentName: string;
-
-   constructor(
-      private db: DuckDBConnection,
-      catalogName: string,
-      environmentName: string,
-   ) {
-      this.table = `${catalogName}.build_manifests`;
-      this.environmentName = environmentName;
-   }
-
-   /**
-    * Idempotently creates the `build_manifests` table and indices in the
-    * DuckLake catalog. Safe to call from every worker on startup.
-    *
-    * Note: this table uses `project_name` for the partition column, unlike
-    * the local DuckDB `build_manifests` table (in `storage/duckdb/schema.ts`)
-    * which uses `project_id` and FK-references `projects.id`. The two stores
-    * partition by different identifiers — local random id vs cross-pod-stable
-    * project name — so they intentionally diverge here.
-    */
-   async bootstrapSchema(): Promise<void> {
-      await this.db.run(`
-         CREATE TABLE IF NOT EXISTS ${this.table} (
-            id VARCHAR,
-            environment_name VARCHAR NOT NULL,
-            package_name VARCHAR NOT NULL,
-            build_id VARCHAR NOT NULL,
-            table_name VARCHAR NOT NULL,
-            source_name VARCHAR NOT NULL,
-            connection_name VARCHAR NOT NULL,
-            created_at TIMESTAMP NOT NULL,
-            updated_at TIMESTAMP NOT NULL
-         )
-      `);
-      logger.info(`DuckLake manifest table bootstrapped: ${this.table}`);
-   }
-
-   async getManifest(
-      _environmentId: string,
-      packageName: string,
-   ): Promise<BuildManifest> {
-      const rows = await this.db.all<Record<string, unknown>>(
-         `SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`,
-         [this.environmentName, packageName],
-      );
-      const manifest: BuildManifest = { entries: {}, strict: false };
-      for (const row of rows) {
-         const buildId = row.build_id as string;
-         // Rows are ordered newest-first; keep only the latest per build_id
-         // to handle rare duplicates from cross-worker races.
-         if (!manifest.entries[buildId]) {
-            manifest.entries[buildId] = {
-               tableName: row.table_name as string,
-            };
-         }
-      }
-      return manifest;
-   }
-
-   /**
-    * Insert a manifest entry. If a row with the same `build_id` already
-    * exists (retry after crash), the duplicate is harmless:
-    * {@link getManifest} deduplicates by build_id keeping the newest row.
-    */
-   async writeEntry(
-      _environmentId: string,
-      packageName: string,
-      buildId: string,
-      tableName: string,
-      sourceName: string,
-      connectionName: string,
-   ): Promise<void> {
-      const now = new Date().toISOString();
-      const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
-
-      await this.db.run(
-         `INSERT INTO ${this.table} (id, environment_name, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
-          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-         [
-            id,
-            this.environmentName,
-            packageName,
-            buildId,
-            tableName,
-            sourceName,
-            connectionName,
-            now,
-            now,
-         ],
-      );
-   }
-
-   async deleteEntry(id: string): Promise<void> {
-      await this.db.run(`DELETE FROM ${this.table} WHERE id = ?`, [id]);
-   }
-
-   async listEntries(
-      _environmentId: string,
-      packageName: string,
-   ): Promise<ManifestEntry[]> {
-      const rows = await this.db.all<Record<string, unknown>>(
-         `SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`,
-         [this.environmentName, packageName],
-      );
-      return rows.map(this.mapToEntry);
-   }
-
-   private mapToEntry(row: Record<string, unknown>): ManifestEntry {
-      return {
-         id: row.id as string,
-         environmentId: row.environment_name as string,
-         packageName: row.package_name as string,
-         buildId: row.build_id as string,
-         tableName: row.table_name as string,
-         sourceName: row.source_name as string,
-         connectionName: row.connection_name as string,
-         createdAt: new Date(row.created_at as string),
-         updatedAt: new Date(row.updated_at as string),
-      };
-   }
-}

package/tests/unit/storage/StorageManager.test.ts

@@ -1,166 +0,0 @@
-// Unit tests for the catch-block wiring in
-// StorageManager.attachDuckLakeCatalog. The pure helpers from
-// `pg_helpers.ts` are tested directly in `pg_helpers.spec.ts`; this file
-// covers the integration — that the helpers are invoked in the right
-// order and the right places inside `initializeDuckLakeForEnvironment`.
-//
-// Stubs DuckDBConnection.run instead of using a real DuckDB so we can
-// inject libpq-style errors at the ATTACH boundary without standing up a
-// real Postgres.
-//
-// Lives under `tests/unit/` (not `src/`) on purpose: the `src/` unit-spec
-// process imports `service/environment_store.spec.ts`, which calls
-// `mock.module("../storage/StorageManager", ...)`. Bun's module mocks
-// persist process-wide across spec files, so a sibling spec in `src/` that
-// `import`s the real StorageManager would get the mock instead. Running
-// here puts us in the separate `test:integration` process with a clean
-// module cache.
-import { describe, expect, it } from "bun:test";
-import { ConnectionAuthError } from "../../../src/errors";
-import { StorageManager } from "../../../src/storage/StorageManager";
-
-interface PrivateStorageManager {
-   duckDbConnection: { run: (sql: string) => Promise<unknown> } | null;
-}
-
-const PG_CONFIG = {
-   catalogUrl: "postgres:host=h user=u password=hunter2 dbname=catalog",
-   dataPath: "gs://bucket/path",
-};
-
-function setupWithStubbedConn(runHandler: (sql: string) => Promise<unknown>): {
-   sm: StorageManager;
-   calls: string[];
-} {
-   const sm = new StorageManager({ type: "duckdb" });
-   const calls: string[] = [];
-   const stub = {
-      run: async (sql: string): Promise<unknown> => {
-         calls.push(sql);
-         return runHandler(sql);
-      },
-   };
-   (sm as unknown as PrivateStorageManager).duckDbConnection = stub;
-   return { sm, calls };
-}
-
-describe("StorageManager.attachDuckLakeCatalog wiring", () => {
-   it("classifies libpq auth failure on ATTACH as ConnectionAuthError", async () => {
-      const { sm } = setupWithStubbedConn(async (sql) => {
-         if (sql.startsWith("ATTACH")) {
-            throw new Error(
-               'IO Error: Unable to connect to Postgres at "host=h user=u password=hunter2 ...": FATAL:  password authentication failed for user "u"',
-            );
-         }
-         return undefined;
-      });
-
-      await expect(
-         sm.initializeDuckLakeForEnvironment("env-1", "env-name", PG_CONFIG),
-      ).rejects.toBeInstanceOf(ConnectionAuthError);
-   });
-
-   it("redacts the embedded password in the classified error", async () => {
-      const { sm } = setupWithStubbedConn(async (sql) => {
-         if (sql.startsWith("ATTACH")) {
-            throw new Error(
-               "password authentication failed: tried host=h password=hunter2",
-            );
-         }
-         return undefined;
-      });
-
-      try {
-         await sm.initializeDuckLakeForEnvironment(
-            "env-1",
-            "env-name",
-            PG_CONFIG,
-         );
-         throw new Error("expected ATTACH to throw");
-      } catch (e) {
-         expect(e).toBeInstanceOf(ConnectionAuthError);
-         expect((e as Error).message).toContain("password=***");
-         expect((e as Error).message).not.toContain("hunter2");
-      }
-   });
-
-   it("injects connect_timeout into PG catalogUrl before ATTACH", async () => {
-      const { sm, calls } = setupWithStubbedConn(async () => undefined);
-
-      await sm.initializeDuckLakeForEnvironment("env-1", "env-name", PG_CONFIG);
-
-      const attachSql = calls.find((s) => s.startsWith("ATTACH"));
-      expect(attachSql).toBeDefined();
-      expect(attachSql).toContain("connect_timeout=");
-   });
-
-   it("honors PG_CONNECT_TIMEOUT_SECONDS env override in the emitted SQL", async () => {
-      const original = process.env.PG_CONNECT_TIMEOUT_SECONDS;
-      process.env.PG_CONNECT_TIMEOUT_SECONDS = "17";
-      try {
-         const { sm, calls } = setupWithStubbedConn(async () => undefined);
-         await sm.initializeDuckLakeForEnvironment(
-            "env-1",
-            "env-name",
-            PG_CONFIG,
-         );
-         const attachSql = calls.find((s) => s.startsWith("ATTACH"));
-         expect(attachSql).toContain("connect_timeout=17");
-      } finally {
-         if (original === undefined) {
-            delete process.env.PG_CONNECT_TIMEOUT_SECONDS;
-         } else {
-            process.env.PG_CONNECT_TIMEOUT_SECONDS = original;
-         }
-      }
-   });
-
-   it("leaves a non-PG catalogUrl untouched (no connect_timeout)", async () => {
-      const { sm, calls } = setupWithStubbedConn(async () => undefined);
-      const sqliteConfig = {
-         catalogUrl: "sqlite:/tmp/x.db",
-         dataPath: "/tmp/data",
-      };
-
-      await sm.initializeDuckLakeForEnvironment(
-         "env-1",
-         "env-name",
-         sqliteConfig,
-      );
-
-      const attachSql = calls.find((s) => s.startsWith("ATTACH"));
-      expect(attachSql).toBeDefined();
-      expect(attachSql).not.toContain("connect_timeout");
-   });
-
-   it("rethrows non-auth errors from ATTACH unchanged (preserves cause)", async () => {
-      const original = new Error("disk I/O error: read failed");
-      const { sm } = setupWithStubbedConn(async (sql) => {
-         if (sql.startsWith("ATTACH")) throw original;
-         return undefined;
-      });
-
-      await expect(
-         sm.initializeDuckLakeForEnvironment("env-1", "env-name", PG_CONFIG),
-      ).rejects.toBe(original);
-   });
-
-   it("does not call connect_timeout injection when catalogUrl lacks postgres: prefix", async () => {
-      // Sanity: the isPostgres branch is detected purely by string prefix.
-      // A keyword-form string without the prefix shouldn't be misclassified.
-      const { sm, calls } = setupWithStubbedConn(async () => undefined);
-      const ambiguousConfig = {
-         catalogUrl: "host=h dbname=d", // no scheme prefix at all
-         dataPath: "/tmp/data",
-      };
-
-      await sm.initializeDuckLakeForEnvironment(
-         "env-1",
-         "env-name",
-         ambiguousConfig,
-      );
-
-      const attachSql = calls.find((s) => s.startsWith("ATTACH"));
-      expect(attachSql).not.toContain("connect_timeout");
-   });
-});