@malloy-publisher/server 0.0.382-dev → 0.2.0

package/src/storage/duckdb/MaterializationRepository.ts

@@ -1,249 +0,0 @@
-import { Materialization, MaterializationStatus } from "../DatabaseInterface";
-import { DuckDBConnection } from "./DuckDBConnection";
-
-const TERMINAL_STATUSES: ReadonlySet<MaterializationStatus> = new Set([
-   "SUCCESS",
-   "FAILED",
-   "CANCELLED",
-]);
-
-function activeKeyFor(projectId: string, packageName: string): string {
-   return `${projectId}|${packageName}`;
-}
-
-/**
- * Thrown when an atomic insert loses a race on (project, package) active
- * materialization. Surfaced separately from a generic DB error so the service
- * layer can translate to `MaterializationConflictError`.
- */
-export class DuplicateActiveMaterializationError extends Error {
-   constructor(projectId: string, packageName: string) {
-      super(
-         `Active materialization already exists for (${projectId}, ${packageName})`,
-      );
-   }
-}
-
-/**
- * DuckDB-backed repository for package materializations.
- *
- * A Materialization tracks a single build run for a (project, package) pair
- * through its lifecycle: PENDING -> RUNNING -> SUCCESS | FAILED | CANCELLED.
- */
-export class MaterializationRepository {
-   constructor(private db: DuckDBConnection) {}
-
-   private generateId(): string {
-      return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
-   }
-
-   private now(): Date {
-      return new Date();
-   }
-
-   async list(
-      projectId: string,
-      packageName: string,
-      options?: { limit?: number; offset?: number },
-   ): Promise<Materialization[]> {
-      let sql =
-         "SELECT * FROM materializations WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC";
-      const params: unknown[] = [projectId, packageName];
-      if (options?.limit !== undefined) {
-         sql += " LIMIT ?";
-         params.push(options.limit);
-      }
-      if (options?.offset !== undefined) {
-         sql += " OFFSET ?";
-         params.push(options.offset);
-      }
-      const rows = await this.db.all<Record<string, unknown>>(sql, params);
-      return rows.map(this.mapRow);
-   }
-
-   async getById(id: string): Promise<Materialization | null> {
-      const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM materializations WHERE id = ?",
-         [id],
-      );
-      return row ? this.mapRow(row) : null;
-   }
-
-   async getActive(
-      projectId: string,
-      packageName: string,
-   ): Promise<Materialization | null> {
-      const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM materializations WHERE project_id = ? AND package_name = ? AND status IN ('PENDING', 'RUNNING')",
-         [projectId, packageName],
-      );
-      return row ? this.mapRow(row) : null;
-   }
-
-   async create(
-      projectId: string,
-      packageName: string,
-      status: MaterializationStatus = "PENDING",
-      metadata: Record<string, unknown> | null = null,
-   ): Promise<Materialization> {
-      const id = this.generateId();
-      const now = this.now();
-      const iso = now.toISOString();
-      // Set active_key iff the row is in a non-terminal state. The unique
-      // index on active_key makes the race-free conditional insert: a second
-      // concurrent create on the same (project, package) fails here rather
-      // than in a check-then-write window.
-      const activeKey = TERMINAL_STATUSES.has(status)
-         ? null
-         : activeKeyFor(projectId, packageName);
-      const metadataJson = metadata ? JSON.stringify(metadata) : null;
-
-      try {
-         const rows = await this.db.all<Record<string, unknown>>(
-            `INSERT INTO materializations (id, project_id, package_name, status, active_key, metadata, created_at, updated_at)
-            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-            RETURNING *`,
-            [
-               id,
-               projectId,
-               packageName,
-               status,
-               activeKey,
-               metadataJson,
-               iso,
-               iso,
-            ],
-         );
-         return this.mapRow(rows[0]);
-      } catch (err) {
-         if (isUniqueViolation(err, "idx_materializations_active_key")) {
-            throw new DuplicateActiveMaterializationError(
-               projectId,
-               packageName,
-            );
-         }
-         throw err;
-      }
-   }
-
-   async update(
-      id: string,
-      updates: {
-         status?: MaterializationStatus;
-         startedAt?: Date;
-         completedAt?: Date;
-         error?: string | null;
-         metadata?: Record<string, unknown> | null;
-      },
-   ): Promise<Materialization> {
-      const now = this.now();
-      const setClauses: string[] = [];
-      const params: unknown[] = [];
-
-      if (updates.status !== undefined) {
-         setClauses.push(`status = ?`);
-         params.push(updates.status);
-         // Clear active_key on any transition to a terminal state; set it on
-         // any transition to a non-terminal state. The unique index
-         // guarantees we can never end up with two active rows for the same
-         // (project, package).
-         if (TERMINAL_STATUSES.has(updates.status)) {
-            setClauses.push(`active_key = NULL`);
-         } else {
-            setClauses.push(`active_key = project_id || '|' || package_name`);
-         }
-      }
-      if (updates.startedAt !== undefined) {
-         setClauses.push(`started_at = ?`);
-         params.push(updates.startedAt.toISOString());
-      }
-      if (updates.completedAt !== undefined) {
-         setClauses.push(`completed_at = ?`);
-         params.push(updates.completedAt.toISOString());
-      }
-      if (updates.error !== undefined) {
-         setClauses.push(`error = ?`);
-         params.push(updates.error);
-      }
-      if (updates.metadata !== undefined) {
-         setClauses.push(`metadata = ?`);
-         params.push(
-            updates.metadata ? JSON.stringify(updates.metadata) : null,
-         );
-      }
-
-      setClauses.push(`updated_at = ?`);
-      params.push(now.toISOString());
-      params.push(id);
-
-      await this.db.run(
-         `UPDATE materializations SET ${setClauses.join(", ")} WHERE id = ?`,
-         params,
-      );
-
-      const updated = await this.getById(id);
-      if (!updated) {
-         throw new Error(`Materialization ${id} not found after update`);
-      }
-      return updated;
-   }
-
-   async deleteByProjectId(projectId: string): Promise<void> {
-      await this.db.run("DELETE FROM materializations WHERE project_id = ?", [
-         projectId,
-      ]);
-   }
-
-   async deleteById(id: string): Promise<void> {
-      await this.db.run("DELETE FROM materializations WHERE id = ?", [id]);
-   }
-
-   async deleteByPackage(
-      projectId: string,
-      packageName: string,
-   ): Promise<void> {
-      await this.db.run(
-         "DELETE FROM materializations WHERE project_id = ? AND package_name = ?",
-         [projectId, packageName],
-      );
-   }
-
-   private mapRow(row: Record<string, unknown>): Materialization {
-      let metadata: Record<string, unknown> | null = null;
-      if (row.metadata) {
-         try {
-            metadata = JSON.parse(row.metadata as string);
-         } catch {
-            metadata = null;
-         }
-      }
-
-      return {
-         id: row.id as string,
-         projectId: row.project_id as string,
-         packageName: row.package_name as string,
-         status: row.status as MaterializationStatus,
-         startedAt: row.started_at ? new Date(row.started_at as string) : null,
-         completedAt: row.completed_at
-            ? new Date(row.completed_at as string)
-            : null,
-         error: row.error != null ? (row.error as string) : null,
-         metadata,
-         createdAt: new Date(row.created_at as string),
-         updatedAt: new Date(row.updated_at as string),
-      };
-   }
-}
-
-/**
- * DuckDB surfaces unique-constraint violations as plain Errors whose message
- * mentions the violated index. We match on the index name rather than a
- * generic substring so we don't misclassify unrelated constraint errors.
- */
-function isUniqueViolation(err: unknown, indexName: string): boolean {
-   if (!(err instanceof Error)) return false;
-   const msg = err.message;
-   return (
-      msg.includes(indexName) || /duplicate key|unique constraint/i.test(msg)
-   );
-}

package/src/storage/duckdb/PackageRepository.ts

@@ -1,135 +0,0 @@
-import { Package } from "../DatabaseInterface";
-import { DuckDBConnection } from "./DuckDBConnection";
-
-export class PackageRepository {
-   constructor(private db: DuckDBConnection) {}
-
-   private generateId(): string {
-      return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
-   }
-
-   private now(): Date {
-      return new Date();
-   }
-
-   async listPackages(projectId: string): Promise<Package[]> {
-      const rows = await this.db.all<Record<string, unknown>>(
-         "SELECT * FROM packages WHERE project_id = ? ORDER BY name",
-         [projectId],
-      );
-      return rows.map(this.mapToPackage);
-   }
-
-   async getPackageById(id: string): Promise<Package | null> {
-      const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM packages WHERE id = ?",
-         [id],
-      );
-      return row ? this.mapToPackage(row) : null;
-   }
-
-   async getPackageByName(
-      projectId: string,
-      name: string,
-   ): Promise<Package | null> {
-      const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM packages WHERE project_id = ? AND name = ?",
-         [projectId, name],
-      );
-      return row ? this.mapToPackage(row) : null;
-   }
-
-   async createPackage(
-      pkg: Omit<Package, "id" | "createdAt" | "updatedAt">,
-   ): Promise<Package> {
-      const id = this.generateId();
-      const now = this.now();
-
-      await this.db.run(
-         `INSERT INTO packages (id, project_id, name, description, manifest_path, metadata, created_at, updated_at)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
-         [
-            id,
-            pkg.projectId,
-            pkg.name,
-            pkg.description || null,
-            pkg.manifestPath,
-            pkg.metadata ? JSON.stringify(pkg.metadata) : null,
-            now.toISOString(),
-            now.toISOString(),
-         ],
-      );
-
-      return {
-         id,
-         ...pkg,
-         createdAt: now,
-         updatedAt: now,
-      };
-   }
-
-   async updatePackage(
-      id: string,
-      updates: Partial<Package>,
-   ): Promise<Package> {
-      const existing = await this.getPackageById(id);
-      if (!existing) {
-         throw new Error(`Package with id ${id} not found`);
-      }
-
-      const now = this.now();
-      const setClauses: string[] = [];
-      const params: unknown[] = [];
-
-      if (updates.name !== undefined) {
-         setClauses.push(`name = ?`);
-         params.push(updates.name);
-      }
-      if (updates.description !== undefined) {
-         setClauses.push(`description = ?`);
-         params.push(updates.description);
-      }
-      if (updates.manifestPath !== undefined) {
-         setClauses.push(`manifest_path = ?`);
-         params.push(updates.manifestPath);
-      }
-      if (updates.metadata !== undefined) {
-         setClauses.push(`metadata = ?`);
-         params.push(JSON.stringify(updates.metadata));
-      }
-
-      setClauses.push(`updated_at = ?`);
-      params.push(now.toISOString());
-      params.push(id);
-
-      await this.db.run(
-         `UPDATE packages SET ${setClauses.join(", ")} WHERE id = ?`,
-         params,
-      );
-
-      return this.getPackageById(id) as Promise<Package>;
-   }
-
-   async deletePackage(id: string): Promise<void> {
-      await this.db.run("DELETE FROM packages WHERE id = ?", [id]);
-   }
-
-   async deletePackagesByProjectId(id: string): Promise<void> {
-      await this.db.run("DELETE FROM packages WHERE project_id = ?", [id]);
-   }
-
-   private mapToPackage(row: Record<string, unknown>): Package {
-      return {
-         id: row.id as string,
-         projectId: row.project_id as string,
-         name: row.name as string,
-         description: row.description as string | undefined,
-         manifestPath: row.manifest_path as string,
-         metadata: row.metadata
-            ? JSON.parse(row.metadata as string)
-            : undefined,
-         createdAt: new Date(row.created_at as string),
-         updatedAt: new Date(row.updated_at as string),
-      };
-   }
-}

package/src/storage/duckdb/ProjectRepository.ts

@@ -1,148 +0,0 @@
-import { Project } from "../DatabaseInterface";
-import { DuckDBConnection } from "./DuckDBConnection";
-
-export class ProjectRepository {
-   constructor(private db: DuckDBConnection) {}
-
-   private generateId(): string {
-      return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
-   }
-
-   private now(): Date {
-      return new Date();
-   }
-
-   async listProjects(): Promise<Project[]> {
-      const rows = await this.db.all<Record<string, unknown>>(
-         "SELECT * FROM projects ORDER BY name",
-      );
-      return rows.map(this.mapToProject);
-   }
-
-   async getProjectById(id: string): Promise<Project | null> {
-      const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM projects WHERE id = ?",
-         [id],
-      );
-      return row ? this.mapToProject(row) : null;
-   }
-
-   async getProjectByName(name: string): Promise<Project | null> {
-      const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM projects WHERE name = ?",
-         [name],
-      );
-      return row ? this.mapToProject(row) : null;
-   }
-
-   async createProject(
-      project: Omit<Project, "id" | "createdAt" | "updatedAt">,
-   ): Promise<Project> {
-      const id = this.generateId();
-      const now = this.now();
-
-      const params = [
-         id,
-         project.name,
-         project.path,
-         project.description || null,
-         project.metadata ? JSON.stringify(project.metadata) : null,
-         now.toISOString(),
-         now.toISOString(),
-      ];
-
-      try {
-         await this.db.run(
-            `INSERT INTO projects (id, name, path, description, metadata, created_at, updated_at)
-        VALUES (?, ?, ?, ?, ?, ?, ?)`,
-            params,
-         );
-
-         return {
-            id,
-            ...project,
-            createdAt: now,
-            updatedAt: now,
-         };
-      } catch (err: unknown) {
-         const error = err as Error;
-         // If unique constraint violation, return existing project
-         if (
-            error.message?.includes("UNIQUE") ||
-            error.message?.includes("Constraint")
-         ) {
-            const existing = await this.db.get<Record<string, unknown>>(
-               "SELECT * FROM projects WHERE name = ?",
-               [project.name],
-            );
-            if (existing) {
-               console.log("Returning existing project");
-               return this.mapToProject(existing);
-            }
-         }
-         throw error;
-      }
-   }
-
-   async updateProject(
-      id: string,
-      updates: Partial<Project>,
-   ): Promise<Project> {
-      const existing = await this.getProjectById(id);
-      if (!existing) {
-         throw new Error(`Project with id ${id} not found`);
-      }
-
-      const now = this.now();
-      const setClauses: string[] = [];
-      const params: unknown[] = [];
-
-      if (updates.name !== undefined && updates.name !== existing.name) {
-         setClauses.push(`name = ?`);
-         params.push(updates.name);
-      }
-
-      if (updates.path !== undefined && updates.path !== existing.path) {
-         setClauses.push(`path = ?`);
-         params.push(updates.path);
-      }
-
-      if (updates.description !== undefined) {
-         setClauses.push(`description = ?`);
-         params.push(updates.description);
-      }
-      if (updates.metadata !== undefined) {
-         setClauses.push(`metadata = ?`);
-         params.push(JSON.stringify(updates.metadata));
-      }
-
-      setClauses.push(`updated_at = ?`);
-      params.push(now.toISOString());
-      params.push(id);
-
-      await this.db.run(
-         `UPDATE projects SET ${setClauses.join(", ")} WHERE id = ?`,
-         params,
-      );
-
-      return this.getProjectById(id) as Promise<Project>;
-   }
-
-   async deleteProject(id: string): Promise<void> {
-      await this.db.run("DELETE FROM projects WHERE id = ?", [id]);
-   }
-
-   private mapToProject(row: Record<string, unknown>): Project {
-      return {
-         id: row.id as string,
-         name: row.name as string,
-         path: row.path as string,
-         description: row.description as string | undefined,
-         metadata: row.metadata
-            ? JSON.parse(row.metadata as string)
-            : undefined,
-         createdAt: new Date(row.created_at as string),
-         updatedAt: new Date(row.updated_at as string),
-      };
-   }
-}

package/src/storage/duckdb/manifest_store.spec.ts

@@ -1,133 +0,0 @@
-import { beforeEach, describe, expect, it } from "bun:test";
-import * as sinon from "sinon";
-import { ManifestEntry, ResourceRepository } from "../DatabaseInterface";
-import { DuckDBManifestStore } from "./DuckDBManifestStore";
-
-function makeEntry(overrides: Partial<ManifestEntry> = {}): ManifestEntry {
-   return {
-      id: "entry-1",
-      projectId: "proj-1",
-      packageName: "pkg",
-      buildId: "build-abc",
-      tableName: "my_table",
-      sourceName: "my_source",
-      connectionName: "duckdb",
-      createdAt: new Date("2026-04-03"),
-      updatedAt: new Date("2026-04-03"),
-      ...overrides,
-   };
-}
-
-function createMocks() {
-   const sandbox = sinon.createSandbox();
-
-   const repository = {
-      listManifestEntries: sandbox.stub(),
-      upsertManifestEntry: sandbox.stub(),
-      deleteManifestEntry: sandbox.stub(),
-   };
-
-   const store = new DuckDBManifestStore(
-      repository as unknown as ResourceRepository,
-   );
-
-   return { sandbox, repository, store };
-}
-
-describe("DuckDBManifestStore", () => {
-   let ctx: ReturnType<typeof createMocks>;
-
-   beforeEach(() => {
-      ctx = createMocks();
-   });
-
-   describe("getManifest", () => {
-      it("should assemble a BuildManifest from repository entries", async () => {
-         ctx.repository.listManifestEntries.resolves([
-            makeEntry({ buildId: "b1", tableName: "tbl_a" }),
-            makeEntry({ buildId: "b2", tableName: "tbl_b" }),
-         ]);
-
-         const manifest = await ctx.store.getManifest("proj-1", "pkg");
-
-         expect(manifest.strict).toBe(false);
-         expect(manifest.entries).toEqual({
-            b1: { tableName: "tbl_a" },
-            b2: { tableName: "tbl_b" },
-         });
-      });
-
-      it("should return empty entries when no rows exist", async () => {
-         ctx.repository.listManifestEntries.resolves([]);
-
-         const manifest = await ctx.store.getManifest("proj-1", "pkg");
-
-         expect(manifest.strict).toBe(false);
-         expect(manifest.entries).toEqual({});
-      });
-   });
-
-   describe("writeEntry", () => {
-      it("should upsert an entry with all fields", async () => {
-         ctx.repository.upsertManifestEntry.resolves(makeEntry());
-
-         await ctx.store.writeEntry(
-            "proj-1",
-            "pkg",
-            "build-abc",
-            "tbl",
-            "src",
-            "conn",
-         );
-
-         expect(ctx.repository.upsertManifestEntry.calledOnce).toBe(true);
-         const arg = ctx.repository.upsertManifestEntry.firstCall.args[0];
-         expect(arg).toEqual({
-            projectId: "proj-1",
-            packageName: "pkg",
-            buildId: "build-abc",
-            tableName: "tbl",
-            sourceName: "src",
-            connectionName: "conn",
-         });
-      });
-   });
-
-   describe("deleteEntry", () => {
-      it("should delegate to repository", async () => {
-         ctx.repository.deleteManifestEntry.resolves();
-
-         await ctx.store.deleteEntry("entry-1");
-
-         expect(ctx.repository.deleteManifestEntry.calledOnce).toBe(true);
-         expect(ctx.repository.deleteManifestEntry.firstCall.args[0]).toBe(
-            "entry-1",
-         );
-      });
-   });
-
-   describe("listEntries", () => {
-      it("should return entries from the repository", async () => {
-         const entries = [
-            makeEntry(),
-            makeEntry({ id: "entry-2", buildId: "build-def" }),
-         ];
-         ctx.repository.listManifestEntries.resolves(entries);
-
-         const result = await ctx.store.listEntries("proj-1", "pkg");
-
-         expect(result).toEqual(entries);
-         expect(
-            ctx.repository.listManifestEntries.calledWith("proj-1", "pkg"),
-         ).toBe(true);
-      });
-
-      it("should return empty array when no entries exist", async () => {
-         ctx.repository.listManifestEntries.resolves([]);
-
-         const result = await ctx.store.listEntries("proj-1", "pkg");
-
-         expect(result).toEqual([]);
-      });
-   });
-});