@malloy-publisher/server 0.0.87 → 0.0.89

package/src/service/project.ts

@@ -1,14 +1,15 @@
 import { BaseConnection } from "@malloydata/malloy/connection";
 import { Mutex } from "async-mutex";
-import * as fs from "fs/promises";
+import * as fs from "fs";
 import * as path from "path";
 import { components } from "../api";
-import { API_PREFIX, README_NAME } from "../constants";
+import { API_PREFIX, PACKAGE_MANIFEST_NAME, README_NAME } from "../constants";
 import {
    ConnectionNotFoundError,
    PackageNotFoundError,
    ProjectNotFoundError,
 } from "../errors";
+import { logger } from "../logger";
 import { createConnections, InternalConnection } from "./connection";
 import { ApiConnection } from "./model";
 import { Package } from "./package";
@@ -41,6 +42,7 @@ export class Project {
       this.metadata = {
          resource: `${API_PREFIX}/projects/${this.projectName}`,
          name: this.projectName,
+         location: this.projectPath,
       };
       void this.reloadProjectMetadata();
    }
@@ -58,7 +60,7 @@ export class Project {
             `${API_PREFIX}/projects/`,
             "",
          );
-         if (!(await fs.exists(this.projectPath))) {
+         if (!(await fs.promises.exists(this.projectPath))) {
             throw new ProjectNotFoundError(
                `Project path "${this.projectPath}" not found`,
             );
@@ -72,14 +74,24 @@ export class Project {
    static async create(
       projectName: string,
       projectPath: string,
+      defaultConnections: ApiConnection[],
    ): Promise<Project> {
-      if (!(await fs.stat(projectPath)).isDirectory()) {
+      if (!(await fs.promises.stat(projectPath)).isDirectory()) {
          throw new ProjectNotFoundError(
             `Project path ${projectPath} not found`,
          );
       }
-      const { malloyConnections, apiConnections } =
-         await createConnections(projectPath);
+      const { malloyConnections, apiConnections } = await createConnections(
+         projectPath,
+         defaultConnections,
+      );
+      logger.info(
+         `Loaded ${malloyConnections.size + apiConnections.length} connections for project ${projectName}`,
+         {
+            malloyConnections,
+            apiConnections,
+         },
+      );
       return new Project(
          projectName,
          projectPath,
@@ -103,7 +115,7 @@ export class Project {
       let readme = "";
       try {
          readme = (
-            await fs.readFile(path.join(this.projectPath, README_NAME))
+            await fs.promises.readFile(path.join(this.projectPath, README_NAME))
          ).toString();
       } catch {
          // Readme not found, so we'll just return an empty string
@@ -157,27 +169,33 @@ export class Project {
    }
 
    public async listPackages(): Promise<ApiPackage[]> {
+      logger.info("Listing packages", { projectPath: this.projectPath });
       try {
-         const files = await fs.readdir(this.projectPath, {
+         const files = await fs.promises.readdir(this.projectPath, {
             withFileTypes: true,
          });
+         const packageDirectories = files.filter(
+            (file) =>
+               file.isDirectory() &&
+               fs.existsSync(
+                  path.join(this.projectPath, file.name, PACKAGE_MANIFEST_NAME),
+               ),
+         );
          const packageMetadata = await Promise.all(
-            files
-               .filter((file) => file.isDirectory())
-               .map(async (directory) => {
-                  try {
-                     return (
-                        await this.getPackage(directory.name, false)
-                     ).getPackageMetadata();
-                  } catch (error) {
-                     console.log(
-                        `Failed to load package: ${directory.name} due to : ${error}`,
-                     );
-                     // Directory did not contain a valid package.json file -- therefore, it's not a package.
-                     // Or it timed out
-                     return undefined;
-                  }
-               }),
+            packageDirectories.map(async (directory) => {
+               try {
+                  return (
+                     await this.getPackage(directory.name, false)
+                  ).getPackageMetadata();
+               } catch (error) {
+                  logger.error(
+                     `Failed to load package: ${directory.name} due to : ${error}`,
+                  );
+                  // Directory did not contain a valid package.json file -- therefore, it's not a package.
+                  // Or it timed out
+                  return undefined;
+               }
+            }),
          );
          // Get rid of undefined entries (i.e, directories without publisher.json files).
          const filteredMetadata = packageMetadata.filter(
@@ -185,23 +203,27 @@ export class Project {
          ) as ApiPackage[];
          return filteredMetadata;
       } catch (error) {
-         throw new Error("Error listing packages: " + error);
+         logger.error("Error listing packages", { error });
+         console.error(error);
+         throw error;
       }
    }
 
    public async getPackage(
       packageName: string,
-      reload: boolean,
+      reload: boolean = false,
    ): Promise<Package> {
       // We need to acquire the mutex to prevent a thundering herd of requests from creating the
       // package multiple times.
       let packageMutex = this.packageMutexes.get(packageName);
-      if (!packageMutex) {
-         packageMutex = new Mutex();
-         this.packageMutexes.set(packageName, packageMutex);
+      if (packageMutex?.isLocked()) {
+         await packageMutex.waitForUnlock();
+         return this.packages.get(packageName)!;
       }
+      packageMutex = new Mutex();
+      this.packageMutexes.set(packageName, packageMutex);
 
-      return await packageMutex.runExclusive(async () => {
+      return packageMutex.runExclusive(async () => {
          const _package = this.packages.get(packageName);
          if (_package !== undefined && !reload) {
             return _package;
@@ -219,6 +241,9 @@ export class Project {
          } catch (error) {
             this.packages.delete(packageName);
             throw error;
+         } finally {
+            packageMutex.release();
+            this.packageMutexes.delete(packageName);
          }
       });
    }
@@ -226,11 +251,18 @@ export class Project {
    public async addPackage(packageName: string) {
       const packagePath = path.join(this.projectPath, packageName);
       if (
-         !(await fs.exists(packagePath)) ||
-         !(await fs.stat(packagePath)).isDirectory()
+         !(await fs.promises.exists(packagePath)) ||
+         !(await fs.promises.stat(packagePath)).isDirectory()
       ) {
          throw new PackageNotFoundError(`Package ${packageName} not found`);
       }
+      logger.info(
+         `Adding package ${packageName} to project ${this.projectName}`,
+         {
+            packagePath,
+            malloyConnections: this.malloyConnections,
+         },
+      );
       this.packages.set(
          packageName,
          await Package.create(
@@ -243,10 +275,7 @@ export class Project {
       return this.packages.get(packageName);
    }
 
-   public async updatePackage(
-      packageName: string,
-      body: { resource?: string; name?: string; description?: string },
-   ) {
+   public async updatePackage(packageName: string, body: ApiPackage) {
       const _package = this.packages.get(packageName);
       if (!_package) {
          throw new PackageNotFoundError(`Package ${packageName} not found`);
@@ -258,6 +287,7 @@ export class Project {
          name: body.name,
          description: body.description,
          resource: body.resource,
+         location: body.location,
       });
       return _package.getPackageMetadata();
    }
@@ -267,6 +297,17 @@ export class Project {
       if (!_package) {
          throw new PackageNotFoundError(`Package ${packageName} not found`);
       }
+      await fs.promises.rm(path.join(this.projectPath, packageName), {
+         recursive: true,
+      });
       this.packages.delete(packageName);
    }
+
+   public async serialize(): Promise<ApiProject> {
+      return {
+         ...this.metadata,
+         connections: this.listApiConnections(),
+         packages: await this.listPackages(),
+      };
+   }
 }

package/src/service/project_store.spec.ts

@@ -1,37 +1,52 @@
-import { describe, expect, it, mock, spyOn } from "bun:test";
+import {
+   afterAll,
+   beforeAll,
+   describe,
+   expect,
+   it,
+   mock,
+   spyOn,
+} from "bun:test";
+import { rmSync } from "fs";
 import * as fs from "fs/promises";
 import path from "path";
+import { isPublisherConfigFrozen } from "../config";
+import { publisherPath } from "../constants";
 import { FrozenConfigError, ProjectNotFoundError } from "../errors";
-import { isPublisherConfigFrozen } from "../utils";
+import { logger } from "../logger";
 import { ProjectStore } from "./project_store";
+import sinon from "sinon";
 
 describe("ProjectStore", () => {
    const serverRoot = path.resolve(
       process.cwd(),
       process.env.SERVER_ROOT || ".",
    );
+   let loggerStub: sinon.SinonStub;
 
-   async function waitForInitialization(projectStore: ProjectStore) {
-      const maxRetries = 100;
-      let retries = 0;
-      do {
-         await new Promise((resolve) => setTimeout(resolve, 10));
-         retries++;
-      } while (projectStore.listProjects().length === 0 && retries < maxRetries);
-      if (projectStore.listProjects().length === 0) {
-         throw new Error("Timed out initializing ProjectStore");
-      }
-   }
-
-   it("should load all projects from publisher.config.json within a second on initialization", async () => {
+   beforeAll(() => {
+      rmSync(path.resolve(publisherPath, "malloy-samples"), {
+         recursive: true,
+         force: true,
+      });
+      loggerStub = sinon.stub(logger, "info").returns(logger);
+   });
+   afterAll(() => {
+      loggerStub.restore();
+   });
+
+   it("should load all projects from publisher.config.json on initialization", async () => {
       mock(isPublisherConfigFrozen).mockReturnValue(true);
       const projectStore = new ProjectStore(serverRoot);
-      await waitForInitialization(projectStore);
-      expect(projectStore.listProjects()).toEqual([
+      mock(projectStore.downloadGitHubDirectory).mockResolvedValue(undefined);
+      await projectStore.finishedInitialization;
+      expect(await projectStore.listProjects()).toEqual([
          {
             name: "malloy-samples",
-            readme: expect.stringContaining("# Malloy Analysis Examples"),
+            readme: expect.any(String),
             resource: "/api/v0/projects/malloy-samples",
+            packages: expect.any(Array),
+            connections: expect.any(Array),
          },
       ]);
    });
@@ -39,14 +54,16 @@ describe("ProjectStore", () => {
    it("should list projects from memory by default", async () => {
       mock(isPublisherConfigFrozen).mockReturnValue(true);
       const projectStore = new ProjectStore(serverRoot);
-      // Mock fs.readFile & fs.readdir to track calls
-      const readFileSpy = spyOn(fs, "readFile");
-      const readdirSpy = spyOn(fs, "readdir");
-      // Call listProjects, which should use memory and not call fs.readFile
-      projectStore.listProjects();
-
-      expect(readFileSpy).not.toHaveBeenCalled();
-      expect(readdirSpy).not.toHaveBeenCalled();
+      const projects = await projectStore.listProjects();
+      expect(projects).toEqual([
+         {
+            name: "malloy-samples",
+            readme: expect.any(String),
+            resource: "/api/v0/projects/malloy-samples",
+            packages: expect.any(Array),
+            connections: expect.any(Array),
+         },
+      ]);
    });
 
    it("should list projects from disk if reload is true", async () => {
@@ -63,24 +80,31 @@ describe("ProjectStore", () => {
    });
 
    it("should allow modifying the in-memory hashmap if config is not frozen", async () => {
-      mock.module("../utils", () => ({
+      mock.module("../config", () => ({
          isPublisherConfigFrozen: () => false,
       }));
       const projectStore = new ProjectStore(serverRoot);
-      await waitForInitialization(projectStore);
+      mock(projectStore.downloadGitHubDirectory).mockResolvedValue(undefined);
+      await projectStore.finishedInitialization;
       await projectStore.updateProject({
          name: "malloy-samples",
          readme: "Updated README",
       });
-      expect(projectStore.listProjects()).toEqual([
-         {
+      let projects = await projectStore.listProjects();
+      projects = await projectStore.listProjects();
+      let malloySamplesProject = projects.find(
+         (p) => p.name === "malloy-samples",
+      );
+      expect(malloySamplesProject).toBeDefined();
+      expect(malloySamplesProject).toMatchObject(
+         expect.objectContaining({
             name: "malloy-samples",
             readme: "Updated README",
             resource: "/api/v0/projects/malloy-samples",
-         },
-      ]);
+         }),
+      );
       await projectStore.deleteProject("malloy-samples");
-      expect(projectStore.listProjects()).toEqual([]);
+      expect(await projectStore.listProjects()).toEqual([]);
       await projectStore.addProject({
          name: "malloy-samples",
       });
@@ -90,31 +114,32 @@ describe("ProjectStore", () => {
       ).toHaveProperty("metadata", {
          name: "malloy-samples",
          resource: "/api/v0/projects/malloy-samples",
+         location: expect.any(String),
       });
 
-      // After a while, it'll resolve the async promise where the readme gets loaded in memory
-      await waitForInitialization(projectStore);
-      expect(projectStore.listProjects()).toEqual([
-         {
-            name: "malloy-samples",
-            readme: expect.stringContaining("# Malloy Analysis Examples"),
-            resource: "/api/v0/projects/malloy-samples",
-         },
-      ]);
+      projects = await projectStore.listProjects();
+      malloySamplesProject = projects.find((p) => p.name === "malloy-samples");
+      expect(malloySamplesProject).toBeDefined();
+      expect(malloySamplesProject).toMatchObject({
+         name: "malloy-samples",
+         resource: "/api/v0/projects/malloy-samples",
+      });
    });
 
    it("should not allow modifying the in-memory hashmap if config is frozen", async () => {
-      mock.module("../utils", () => ({
+      mock.module("../config", () => ({
          isPublisherConfigFrozen: () => true,
       }));
       const projectStore = new ProjectStore(serverRoot);
       // Initialization should succeed
-      await waitForInitialization(projectStore);
-      expect(projectStore.listProjects()).toEqual([
+      await projectStore.finishedInitialization;
+      expect(await projectStore.listProjects()).toEqual([
          {
             name: "malloy-samples",
-            readme: expect.stringContaining("# Malloy Analysis Examples"),
+            readme: expect.any(String),
             resource: "/api/v0/projects/malloy-samples",
+            packages: expect.any(Array),
+            connections: expect.any(Array),
          },
       ]);
       // Adding a project should fail
@@ -138,36 +163,38 @@ describe("ProjectStore", () => {
       );
 
       // Failed methods should not modify the in-memory hashmap
-      expect(projectStore.listProjects()).toEqual([
+      expect(await projectStore.listProjects()).toEqual([
          {
             name: "malloy-samples",
-            readme: expect.stringContaining("# Malloy Analysis Examples"),
+            readme: expect.any(String),
             resource: "/api/v0/projects/malloy-samples",
+            packages: expect.any(Array),
+            connections: expect.any(Array),
          },
       ]);
    });
 
    it("should always try to reload a project if it's not in the hashmap", async () => {
-      mock.module("../utils", () => ({
+      mock.module("../config", () => ({
          isPublisherConfigFrozen: () => false,
       }));
       const projectStore = new ProjectStore(serverRoot);
-      await waitForInitialization(projectStore);
+      await projectStore.finishedInitialization;
       await projectStore.deleteProject("malloy-samples");
-      expect(projectStore.listProjects()).toEqual([]);
+      expect(await projectStore.listProjects()).toEqual([]);
       const readFileSpy = spyOn(fs, "readFile");
       await projectStore.getProject("malloy-samples", true);
       expect(readFileSpy).toHaveBeenCalled();
    });
 
    it("should throw a NotFound error when reloading a project that is not in disk", async () => {
-      mock.module("../utils", () => ({
+      mock.module("../config", () => ({
          isPublisherConfigFrozen: () => false,
       }));
       const projectStore = new ProjectStore(serverRoot);
-      await waitForInitialization(projectStore);
-      expect(projectStore.getProject("this-one-does-not-exist", true)).rejects.toThrow(
-         ProjectNotFoundError,
-      );
+      await projectStore.finishedInitialization;
+      expect(
+         projectStore.getProject("this-one-does-not-exist", true),
+      ).rejects.toThrow(ProjectNotFoundError);
    });
 });