@malloy-publisher/server 0.0.188 → 0.0.382-dev

package/src/service/manifest_service.spec.ts

@@ -0,0 +1,201 @@
+import { beforeEach, describe, expect, it } from "bun:test";
+import * as sinon from "sinon";
+import {
+   BuildManifest,
+   ManifestEntry,
+   ManifestStore,
+   ResourceRepository,
+} from "../storage/DatabaseInterface";
+import { ManifestService } from "./manifest_service";
+import { ProjectStore } from "./project_store";
+
+function makeEntry(overrides: Partial<ManifestEntry> = {}): ManifestEntry {
+   return {
+      id: "entry-1",
+      projectId: "proj-1",
+      packageName: "pkg",
+      buildId: "build-abc",
+      tableName: "my_table",
+      sourceName: "my_source",
+      connectionName: "duckdb",
+      createdAt: new Date("2025-01-01"),
+      updatedAt: new Date("2025-01-01"),
+      ...overrides,
+   };
+}
+
+function createMocks() {
+   const sandbox = sinon.createSandbox();
+
+   const manifestStore: sinon.SinonStubbedInstance<ManifestStore> = {
+      getManifest: sandbox.stub(),
+      writeEntry: sandbox.stub(),
+      deleteEntry: sandbox.stub(),
+      listEntries: sandbox.stub(),
+   };
+
+   const repository = {
+      listManifestEntries: sandbox.stub(),
+   } as unknown as sinon.SinonStubbedInstance<ResourceRepository>;
+
+   const reloadAllModels = sandbox.stub().resolves();
+
+   const pkg = {
+      reloadAllModels,
+   };
+
+   const project = {
+      getPackage: sandbox.stub().resolves(pkg),
+   };
+
+   const projectStore = {
+      storageManager: {
+         getManifestStore: (_projectId?: string) => manifestStore,
+         getRepository: () => repository,
+      },
+      getProject: sandbox.stub().resolves(project),
+   } as unknown as ProjectStore;
+
+   const service = new ManifestService(projectStore);
+
+   return {
+      sandbox,
+      manifestStore,
+      repository,
+      projectStore,
+      project,
+      pkg,
+      reloadAllModels,
+      service,
+   };
+}
+
+describe("ManifestService", () => {
+   let ctx: ReturnType<typeof createMocks>;
+
+   beforeEach(() => {
+      ctx = createMocks();
+   });
+
+   describe("getManifest", () => {
+      it("should delegate to the manifest store", async () => {
+         const manifest: BuildManifest = {
+            entries: { "build-abc": { tableName: "tbl" } },
+            strict: false,
+         };
+         ctx.manifestStore.getManifest.resolves(manifest);
+
+         const result = await ctx.service.getManifest("proj-1", "pkg");
+
+         expect(result).toEqual(manifest);
+         expect(ctx.manifestStore.getManifest.calledWith("proj-1", "pkg")).toBe(
+            true,
+         );
+      });
+   });
+
+   describe("writeEntry", () => {
+      it("should delegate to the manifest store", async () => {
+         ctx.manifestStore.writeEntry.resolves();
+
+         await ctx.service.writeEntry(
+            "proj-1",
+            "pkg",
+            "build-abc",
+            "tbl",
+            "src",
+            "duckdb",
+         );
+
+         expect(ctx.manifestStore.writeEntry.calledOnce).toBe(true);
+         const args = ctx.manifestStore.writeEntry.firstCall.args;
+         expect(args[0]).toBe("proj-1");
+         expect(args[1]).toBe("pkg");
+         expect(args[2]).toBe("build-abc");
+         expect(args[3]).toBe("tbl");
+         expect(args[4]).toBe("src");
+         expect(args[5]).toBe("duckdb");
+      });
+   });
+
+   describe("deleteEntry", () => {
+      it("should delegate to the manifest store", async () => {
+         ctx.manifestStore.deleteEntry.resolves();
+
+         await ctx.service.deleteEntry("proj-1", "entry-1");
+
+         expect(ctx.manifestStore.deleteEntry.calledOnce).toBe(true);
+         expect(ctx.manifestStore.deleteEntry.firstCall.args[0]).toBe(
+            "entry-1",
+         );
+      });
+   });
+
+   describe("reloadManifest", () => {
+      it("should get manifest, reload it onto the package, and return it", async () => {
+         const manifest: BuildManifest = {
+            entries: { "build-abc": { tableName: "tbl" } },
+            strict: false,
+         };
+         ctx.manifestStore.getManifest.resolves(manifest);
+
+         const result = await ctx.service.reloadManifest(
+            "proj-1",
+            "pkg",
+            "my-project",
+         );
+
+         expect(result).toEqual(manifest);
+         expect(
+            (ctx.projectStore.getProject as sinon.SinonStub).calledWith(
+               "my-project",
+               false,
+            ),
+         ).toBe(true);
+         expect(ctx.project.getPackage.calledWith("pkg", false)).toBe(true);
+         expect(ctx.reloadAllModels.calledWith(manifest.entries)).toBe(true);
+      });
+
+      it("should return an empty manifest when no entries exist", async () => {
+         const emptyManifest: BuildManifest = {
+            entries: {},
+            strict: false,
+         };
+         ctx.manifestStore.getManifest.resolves(emptyManifest);
+
+         const result = await ctx.service.reloadManifest(
+            "proj-1",
+            "pkg",
+            "my-project",
+         );
+
+         expect(result.entries).toEqual({});
+         expect(ctx.reloadAllModels.calledWith({})).toBe(true);
+      });
+   });
+
+   describe("listEntries", () => {
+      it("should return entries from the manifest store", async () => {
+         const entries = [
+            makeEntry(),
+            makeEntry({ id: "entry-2", buildId: "build-def" }),
+         ];
+         ctx.manifestStore.listEntries.resolves(entries);
+
+         const result = await ctx.service.listEntries("proj-1", "pkg");
+
+         expect(result).toEqual(entries);
+         expect(ctx.manifestStore.listEntries.calledWith("proj-1", "pkg")).toBe(
+            true,
+         );
+      });
+
+      it("should return empty array when no entries exist", async () => {
+         ctx.manifestStore.listEntries.resolves([]);
+
+         const result = await ctx.service.listEntries("proj-1", "pkg");
+
+         expect(result).toEqual([]);
+      });
+   });
+});

package/src/service/manifest_service.ts

@@ -0,0 +1,106 @@
+import { logger } from "../logger";
+import {
+   BuildManifest,
+   ManifestEntry,
+   ManifestStore,
+} from "../storage/DatabaseInterface";
+import { ProjectStore } from "./project_store";
+
+/**
+ * Manages build manifests that map source names to materialized table names.
+ *
+ * A build manifest is produced during materialization: each entry records which
+ * DuckDB table backs a given Malloy source.
+ *
+ * Two-phase lifecycle:
+ *  1. **Persist** – `writeEntry` stores individual entries to the DB as they
+ *     are produced during a build.
+ *  2. **Reload** – `reloadManifest` reads the manifest from the DB and
+ *     recompiles all models in the package so the Malloy Runtime resolves
+ *     persist references to the materialized tables.
+ *
+ * All manifest operations delegate to the active {@link ManifestStore}, which
+ * is either the local DuckDB store (standalone) or DuckLake (orchestrated).
+ */
+export class ManifestService {
+   constructor(private projectStore: ProjectStore) {}
+
+   private manifestStoreFor(projectId: string): ManifestStore {
+      return this.projectStore.storageManager.getManifestStore(projectId);
+   }
+
+   async getManifest(
+      projectId: string,
+      packageName: string,
+   ): Promise<BuildManifest> {
+      return this.manifestStoreFor(projectId).getManifest(
+         projectId,
+         packageName,
+      );
+   }
+
+   async writeEntry(
+      projectId: string,
+      packageName: string,
+      buildId: string,
+      tableName: string,
+      sourceName: string,
+      connectionName: string,
+   ): Promise<void> {
+      await this.manifestStoreFor(projectId).writeEntry(
+         projectId,
+         packageName,
+         buildId,
+         tableName,
+         sourceName,
+         connectionName,
+      );
+   }
+
+   async deleteEntry(projectId: string, entryId: string): Promise<void> {
+      await this.manifestStoreFor(projectId).deleteEntry(entryId);
+   }
+
+   /**
+    * Read the manifest from storage and reload it onto this worker's
+    * package so subsequent queries resolve persist references to
+    * materialized tables. This is what `POST .../manifest?action=reload` calls
+    * (used by orchestrated workers that need to pick up manifest state
+    * produced elsewhere). Despite the name, nothing is loaded *into*
+    * storage — the worker pulls the manifest down and recompiles.
+    */
+   async reloadManifest(
+      projectId: string,
+      packageName: string,
+      projectName: string,
+   ): Promise<BuildManifest> {
+      const manifest = await this.getManifest(projectId, packageName);
+
+      const project = await this.projectStore.getProject(projectName, false);
+      const pkg = await project.getPackage(packageName, false);
+      await pkg.reloadAllModels(manifest.entries);
+
+      logger.info("Reloaded manifest and recompiled models", {
+         projectId,
+         packageName,
+         entryCount: Object.keys(manifest.entries).length,
+      });
+
+      return manifest;
+   }
+
+   /**
+    * Get the manifest entries from the active store for inspection.
+    * Routes through ManifestStore so DuckLake mode reads the shared
+    * catalog instead of the local DuckDB table.
+    */
+   async listEntries(
+      projectId: string,
+      packageName: string,
+   ): Promise<ManifestEntry[]> {
+      return this.manifestStoreFor(projectId).listEntries(
+         projectId,
+         packageName,
+      );
+   }
+}