@malloy-publisher/server 0.0.88 → 0.0.89

package/src/service/package.spec.ts

@@ -4,13 +4,18 @@ import { join } from "path";
 import sinon from "sinon";
 import { PackageNotFoundError } from "../errors";
 import { readConnectionConfig } from "./connection";
-import { Model } from "./model";
+import { ApiConnection, Model } from "./model";
 import { Package } from "./package";
 import { Scheduler } from "./scheduler";
 
 // Minimal partial types for mocking
 type PartialScheduler = Pick<Scheduler, "list">;
 
+const connectionMocks: ApiConnection[] = [
+   { name: "conn1", type: "postgres", postgresConnection: {} },
+   { name: "conn2", type: "bigquery", bigqueryConnection: {} },
+];
+
 describe("service/package", () => {
    const testPackageDirectory = "testPackage";
 
@@ -24,10 +29,7 @@ describe("service/package", () => {
          join(testPackageDirectory, "database.csv"),
          parquetBuffer,
       );
-      const content = JSON.stringify([
-         { name: "conn1", type: "database" },
-         { name: "conn2", type: "api" },
-      ]);
+      const content = JSON.stringify(connectionMocks);
       await fs.writeFile(
          join(testPackageDirectory, "publisher.connections.json"),
          content,
@@ -82,7 +84,7 @@ describe("service/package", () => {
          });
          it("should return a Package object if the package exists", async () => {
             sinon.stub(fs, "stat").resolves();
-            sinon
+            const readFileStub = sinon
                .stub(fs, "readFile")
                .resolves(
                   Buffer.from(JSON.stringify({ description: "Test package" })),
@@ -98,6 +100,9 @@ describe("service/package", () => {
                list: () => [],
             } as PartialScheduler);
 
+            readFileStub.restore();
+            readFileStub.resolves(Buffer.from(JSON.stringify([])));
+
             const packageInstance = await Package.create(
                "testProject",
                "testPackage",
@@ -221,10 +226,7 @@ describe("service/package", () => {
             sinon.stub(fs, "stat").resolves();
             const config = await readConnectionConfig(testPackageDirectory);
 
-            expect(config).toEqual([
-               { name: "conn1", type: "database" },
-               { name: "conn2", type: "api" },
-            ]);
+            expect(config).toEqual(connectionMocks);
          });
       });
    });

package/src/service/package.ts

@@ -101,7 +101,10 @@ export class Package {
             unit: "ms",
          });
          const connections = new Map<string, Connection>(projectConnections);
-
+         logger.info(`Project connections: ${connections.size}`, {
+            connections,
+            projectConnections,
+         });
          // Package connections override project connections.
          const { malloyConnections: packageConnections } =
             await createConnections(packagePath);
@@ -147,6 +150,11 @@ export class Package {
             malloy_package_name: packageName,
             status: "success",
          });
+         logger.info(`Successfully loaded package ${packageName}`, {
+            packageName,
+            duration: executionTime,
+            unit: "ms",
+         });
          return new Package(
             projectName,
             packageName,
@@ -158,15 +166,14 @@ export class Package {
          );
       } catch (error) {
          logger.error(`Error loading package ${packageName}`, { error });
+         console.error(error);
          const endTime = performance.now();
          const executionTime = endTime - startTime;
          this.packageLoadHistogram.record(executionTime, {
             malloy_package_name: packageName,
             status: "error",
          });
-         throw new Error(`Error loading package ${packageName}`, {
-            cause: error,
-         });
+         throw error;
       }
    }
 

package/src/service/project.ts

@@ -1,14 +1,15 @@
 import { BaseConnection } from "@malloydata/malloy/connection";
 import { Mutex } from "async-mutex";
-import * as fs from "fs/promises";
+import * as fs from "fs";
 import * as path from "path";
 import { components } from "../api";
-import { API_PREFIX, README_NAME } from "../constants";
+import { API_PREFIX, PACKAGE_MANIFEST_NAME, README_NAME } from "../constants";
 import {
    ConnectionNotFoundError,
    PackageNotFoundError,
    ProjectNotFoundError,
 } from "../errors";
+import { logger } from "../logger";
 import { createConnections, InternalConnection } from "./connection";
 import { ApiConnection } from "./model";
 import { Package } from "./package";
@@ -59,7 +60,7 @@ export class Project {
             `${API_PREFIX}/projects/`,
             "",
          );
-         if (!(await fs.exists(this.projectPath))) {
+         if (!(await fs.promises.exists(this.projectPath))) {
             throw new ProjectNotFoundError(
                `Project path "${this.projectPath}" not found`,
             );
@@ -73,14 +74,24 @@ export class Project {
    static async create(
       projectName: string,
       projectPath: string,
+      defaultConnections: ApiConnection[],
    ): Promise<Project> {
-      if (!(await fs.stat(projectPath)).isDirectory()) {
+      if (!(await fs.promises.stat(projectPath)).isDirectory()) {
          throw new ProjectNotFoundError(
             `Project path ${projectPath} not found`,
          );
       }
-      const { malloyConnections, apiConnections } =
-         await createConnections(projectPath);
+      const { malloyConnections, apiConnections } = await createConnections(
+         projectPath,
+         defaultConnections,
+      );
+      logger.info(
+         `Loaded ${malloyConnections.size + apiConnections.length} connections for project ${projectName}`,
+         {
+            malloyConnections,
+            apiConnections,
+         },
+      );
       return new Project(
          projectName,
          projectPath,
@@ -104,7 +115,7 @@ export class Project {
       let readme = "";
       try {
          readme = (
-            await fs.readFile(path.join(this.projectPath, README_NAME))
+            await fs.promises.readFile(path.join(this.projectPath, README_NAME))
          ).toString();
       } catch {
          // Readme not found, so we'll just return an empty string
@@ -158,27 +169,33 @@ export class Project {
    }
 
    public async listPackages(): Promise<ApiPackage[]> {
+      logger.info("Listing packages", { projectPath: this.projectPath });
       try {
-         const files = await fs.readdir(this.projectPath, {
+         const files = await fs.promises.readdir(this.projectPath, {
             withFileTypes: true,
          });
+         const packageDirectories = files.filter(
+            (file) =>
+               file.isDirectory() &&
+               fs.existsSync(
+                  path.join(this.projectPath, file.name, PACKAGE_MANIFEST_NAME),
+               ),
+         );
          const packageMetadata = await Promise.all(
-            files
-               .filter((file) => file.isDirectory())
-               .map(async (directory) => {
-                  try {
-                     return (
-                        await this.getPackage(directory.name, false)
-                     ).getPackageMetadata();
-                  } catch (error) {
-                     console.log(
-                        `Failed to load package: ${directory.name} due to : ${error}`,
-                     );
-                     // Directory did not contain a valid package.json file -- therefore, it's not a package.
-                     // Or it timed out
-                     return undefined;
-                  }
-               }),
+            packageDirectories.map(async (directory) => {
+               try {
+                  return (
+                     await this.getPackage(directory.name, false)
+                  ).getPackageMetadata();
+               } catch (error) {
+                  logger.error(
+                     `Failed to load package: ${directory.name} due to : ${error}`,
+                  );
+                  // Directory did not contain a valid package.json file -- therefore, it's not a package.
+                  // Or it timed out
+                  return undefined;
+               }
+            }),
          );
          // Get rid of undefined entries (i.e, directories without publisher.json files).
          const filteredMetadata = packageMetadata.filter(
@@ -186,23 +203,27 @@ export class Project {
          ) as ApiPackage[];
          return filteredMetadata;
       } catch (error) {
-         throw new Error("Error listing packages: " + error);
+         logger.error("Error listing packages", { error });
+         console.error(error);
+         throw error;
       }
    }
 
    public async getPackage(
       packageName: string,
-      reload: boolean,
+      reload: boolean = false,
    ): Promise<Package> {
       // We need to acquire the mutex to prevent a thundering herd of requests from creating the
       // package multiple times.
       let packageMutex = this.packageMutexes.get(packageName);
-      if (!packageMutex) {
-         packageMutex = new Mutex();
-         this.packageMutexes.set(packageName, packageMutex);
+      if (packageMutex?.isLocked()) {
+         await packageMutex.waitForUnlock();
+         return this.packages.get(packageName)!;
       }
+      packageMutex = new Mutex();
+      this.packageMutexes.set(packageName, packageMutex);
 
-      return await packageMutex.runExclusive(async () => {
+      return packageMutex.runExclusive(async () => {
          const _package = this.packages.get(packageName);
          if (_package !== undefined && !reload) {
             return _package;
@@ -220,6 +241,9 @@ export class Project {
          } catch (error) {
             this.packages.delete(packageName);
             throw error;
+         } finally {
+            packageMutex.release();
+            this.packageMutexes.delete(packageName);
          }
       });
    }
@@ -227,11 +251,18 @@ export class Project {
    public async addPackage(packageName: string) {
       const packagePath = path.join(this.projectPath, packageName);
       if (
-         !(await fs.exists(packagePath)) ||
-         !(await fs.stat(packagePath)).isDirectory()
+         !(await fs.promises.exists(packagePath)) ||
+         !(await fs.promises.stat(packagePath)).isDirectory()
       ) {
          throw new PackageNotFoundError(`Package ${packageName} not found`);
       }
+      logger.info(
+         `Adding package ${packageName} to project ${this.projectName}`,
+         {
+            packagePath,
+            malloyConnections: this.malloyConnections,
+         },
+      );
       this.packages.set(
          packageName,
          await Package.create(
@@ -266,9 +297,17 @@ export class Project {
       if (!_package) {
          throw new PackageNotFoundError(`Package ${packageName} not found`);
       }
-      await fs.rm(path.join(this.projectPath, packageName), {
+      await fs.promises.rm(path.join(this.projectPath, packageName), {
          recursive: true,
       });
       this.packages.delete(packageName);
    }
+
+   public async serialize(): Promise<ApiProject> {
+      return {
+         ...this.metadata,
+         connections: this.listApiConnections(),
+         packages: await this.listPackages(),
+      };
+   }
 }

package/src/service/project_store.spec.ts

@@ -1,40 +1,52 @@
-import { describe, expect, it, mock, spyOn } from "bun:test";
+import {
+   afterAll,
+   beforeAll,
+   describe,
+   expect,
+   it,
+   mock,
+   spyOn,
+} from "bun:test";
+import { rmSync } from "fs";
 import * as fs from "fs/promises";
 import path from "path";
 import { isPublisherConfigFrozen } from "../config";
+import { publisherPath } from "../constants";
 import { FrozenConfigError, ProjectNotFoundError } from "../errors";
+import { logger } from "../logger";
 import { ProjectStore } from "./project_store";
+import sinon from "sinon";
 
 describe("ProjectStore", () => {
    const serverRoot = path.resolve(
       process.cwd(),
       process.env.SERVER_ROOT || ".",
    );
+   let loggerStub: sinon.SinonStub;
 
-   async function waitForInitialization(projectStore: ProjectStore) {
-      const maxRetries = 100;
-      let retries = 0;
-      do {
-         await new Promise((resolve) => setTimeout(resolve, 10));
-         retries++;
-      } while (
-         (await projectStore.listProjects()).length === 0 &&
-         retries < maxRetries
-      );
-      if ((await projectStore.listProjects()).length === 0) {
-         throw new Error("Timed out initializing ProjectStore");
-      }
-   }
+   beforeAll(() => {
+      rmSync(path.resolve(publisherPath, "malloy-samples"), {
+         recursive: true,
+         force: true,
+      });
+      loggerStub = sinon.stub(logger, "info").returns(logger);
+   });
+   afterAll(() => {
+      loggerStub.restore();
+   });
 
-   it("should load all projects from publisher.config.json within a second on initialization", async () => {
+   it("should load all projects from publisher.config.json on initialization", async () => {
       mock(isPublisherConfigFrozen).mockReturnValue(true);
       const projectStore = new ProjectStore(serverRoot);
-      await waitForInitialization(projectStore);
+      mock(projectStore.downloadGitHubDirectory).mockResolvedValue(undefined);
+      await projectStore.finishedInitialization;
       expect(await projectStore.listProjects()).toEqual([
          {
             name: "malloy-samples",
-            readme: expect.stringContaining("# Malloy Analysis Examples"),
+            readme: expect.any(String),
             resource: "/api/v0/projects/malloy-samples",
+            packages: expect.any(Array),
+            connections: expect.any(Array),
          },
       ]);
    });
@@ -42,16 +54,16 @@ describe("ProjectStore", () => {
    it("should list projects from memory by default", async () => {
       mock(isPublisherConfigFrozen).mockReturnValue(true);
       const projectStore = new ProjectStore(serverRoot);
-      await waitForInitialization(projectStore);
-
-      // Mock fs.readFile & fs.readdir to track calls
-      const readFileSpy = spyOn(fs, "readFile");
-      const readdirSpy = spyOn(fs, "readdir");
-      // Call listProjects, which should use memory and not call fs.readFile
-      await projectStore.listProjects();
-
-      expect(readFileSpy).not.toHaveBeenCalled();
-      expect(readdirSpy).not.toHaveBeenCalled();
+      const projects = await projectStore.listProjects();
+      expect(projects).toEqual([
+         {
+            name: "malloy-samples",
+            readme: expect.any(String),
+            resource: "/api/v0/projects/malloy-samples",
+            packages: expect.any(Array),
+            connections: expect.any(Array),
+         },
+      ]);
    });
 
    it("should list projects from disk if reload is true", async () => {
@@ -68,22 +80,29 @@ describe("ProjectStore", () => {
    });
 
    it("should allow modifying the in-memory hashmap if config is not frozen", async () => {
-      mock.module("../utils", () => ({
+      mock.module("../config", () => ({
          isPublisherConfigFrozen: () => false,
       }));
       const projectStore = new ProjectStore(serverRoot);
-      await waitForInitialization(projectStore);
+      mock(projectStore.downloadGitHubDirectory).mockResolvedValue(undefined);
+      await projectStore.finishedInitialization;
       await projectStore.updateProject({
          name: "malloy-samples",
          readme: "Updated README",
       });
-      expect(await projectStore.listProjects()).toEqual([
-         {
+      let projects = await projectStore.listProjects();
+      projects = await projectStore.listProjects();
+      let malloySamplesProject = projects.find(
+         (p) => p.name === "malloy-samples",
+      );
+      expect(malloySamplesProject).toBeDefined();
+      expect(malloySamplesProject).toMatchObject(
+         expect.objectContaining({
             name: "malloy-samples",
             readme: "Updated README",
             resource: "/api/v0/projects/malloy-samples",
-         },
-      ]);
+         }),
+      );
       await projectStore.deleteProject("malloy-samples");
       expect(await projectStore.listProjects()).toEqual([]);
       await projectStore.addProject({
@@ -95,31 +114,32 @@ describe("ProjectStore", () => {
       ).toHaveProperty("metadata", {
          name: "malloy-samples",
          resource: "/api/v0/projects/malloy-samples",
+         location: expect.any(String),
       });
 
-      // After a while, it'll resolve the async promise where the readme gets loaded in memory
-      await waitForInitialization(projectStore);
-      expect(await projectStore.listProjects()).toEqual([
-         {
-            name: "malloy-samples",
-            readme: expect.stringContaining("# Malloy Analysis Examples"),
-            resource: "/api/v0/projects/malloy-samples",
-         },
-      ]);
+      projects = await projectStore.listProjects();
+      malloySamplesProject = projects.find((p) => p.name === "malloy-samples");
+      expect(malloySamplesProject).toBeDefined();
+      expect(malloySamplesProject).toMatchObject({
+         name: "malloy-samples",
+         resource: "/api/v0/projects/malloy-samples",
+      });
    });
 
    it("should not allow modifying the in-memory hashmap if config is frozen", async () => {
-      mock.module("../utils", () => ({
+      mock.module("../config", () => ({
          isPublisherConfigFrozen: () => true,
       }));
       const projectStore = new ProjectStore(serverRoot);
       // Initialization should succeed
-      await waitForInitialization(projectStore);
+      await projectStore.finishedInitialization;
       expect(await projectStore.listProjects()).toEqual([
          {
             name: "malloy-samples",
-            readme: expect.stringContaining("# Malloy Analysis Examples"),
+            readme: expect.any(String),
             resource: "/api/v0/projects/malloy-samples",
+            packages: expect.any(Array),
+            connections: expect.any(Array),
          },
       ]);
       // Adding a project should fail
@@ -146,18 +166,20 @@ describe("ProjectStore", () => {
       expect(await projectStore.listProjects()).toEqual([
          {
             name: "malloy-samples",
-            readme: expect.stringContaining("# Malloy Analysis Examples"),
+            readme: expect.any(String),
             resource: "/api/v0/projects/malloy-samples",
+            packages: expect.any(Array),
+            connections: expect.any(Array),
          },
       ]);
    });
 
    it("should always try to reload a project if it's not in the hashmap", async () => {
-      mock.module("../utils", () => ({
+      mock.module("../config", () => ({
          isPublisherConfigFrozen: () => false,
       }));
       const projectStore = new ProjectStore(serverRoot);
-      await waitForInitialization(projectStore);
+      await projectStore.finishedInitialization;
       await projectStore.deleteProject("malloy-samples");
       expect(await projectStore.listProjects()).toEqual([]);
       const readFileSpy = spyOn(fs, "readFile");
@@ -166,11 +188,11 @@ describe("ProjectStore", () => {
    });
 
    it("should throw a NotFound error when reloading a project that is not in disk", async () => {
-      mock.module("../utils", () => ({
+      mock.module("../config", () => ({
          isPublisherConfigFrozen: () => false,
       }));
       const projectStore = new ProjectStore(serverRoot);
-      await waitForInitialization(projectStore);
+      await projectStore.finishedInitialization;
       expect(
          projectStore.getProject("this-one-does-not-exist", true),
       ).rejects.toThrow(ProjectNotFoundError);

package/src/service/project_store.ts

@@ -2,11 +2,12 @@ import { GetObjectCommand, S3 } from "@aws-sdk/client-s3";
 import { Storage } from "@google-cloud/storage";
 import * as fs from "fs";
 import * as path from "path";
+import AdmZip from "adm-zip";
 import simpleGit from "simple-git";
 import { Writable } from "stream";
 import { components } from "../api";
 import { getPublisherConfig, isPublisherConfigFrozen } from "../config";
-import { API_PREFIX, PUBLISHER_CONFIG_NAME } from "../constants";
+import { API_PREFIX, PUBLISHER_CONFIG_NAME, publisherPath } from "../constants";
 import { FrozenConfigError, ProjectNotFoundError } from "../errors";
 import { logger } from "../logger";
 import { Project } from "./project";
@@ -56,20 +57,23 @@ export class ProjectStore {
          );
       } catch (error) {
          logger.error("Error initializing project store", { error });
+         console.error(error);
          process.exit(1);
       }
    }
 
    public async listProjects() {
       await this.finishedInitialization;
-      return Array.from(this.projects.values()).map(
-         (project) => project.metadata,
+      return Promise.all(
+         Array.from(this.projects.values()).map((project) =>
+            project.serialize(),
+         ),
       );
    }
 
    public async getProject(
       projectName: string,
-      reload: boolean,
+      reload: boolean = false,
    ): Promise<Project> {
       await this.finishedInitialization;
       let project = this.projects.get(projectName);
@@ -99,9 +103,10 @@ export class ProjectStore {
       if (!skipInitialization) {
          await this.finishedInitialization;
       }
-      if (this.publisherConfigIsFrozen) {
+      if (!skipInitialization && this.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
       }
+
       const projectName = project.name;
       if (!projectName) {
          throw new Error("Project name is required");
@@ -111,15 +116,44 @@ export class ProjectStore {
       );
       const projectPath =
          project.location || projectManifest.projects[projectName];
-      const absoluteProjectPath = await this.loadProjectIntoDisk(
+      let absoluteProjectPath: string;
+      if (projectPath) {
+         absoluteProjectPath = await this.loadProjectIntoDisk(
+            projectName,
+            projectPath,
+         );
+         if (absoluteProjectPath.endsWith(".zip")) {
+            absoluteProjectPath = await this.unzipProject(absoluteProjectPath);
+         }
+      } else {
+         absoluteProjectPath = await this.scaffoldProject(project);
+      }
+      const newProject = await Project.create(
          projectName,
-         projectPath,
+         absoluteProjectPath,
+         project.connections || [],
       );
-      const newProject = await Project.create(projectName, absoluteProjectPath);
       this.projects.set(projectName, newProject);
       return newProject;
    }
 
+   public async unzipProject(absoluteProjectPath: string) {
+      logger.info(
+         `Detected zip file at "${absoluteProjectPath}". Unzipping...`,
+      );
+      const unzippedProjectPath = absoluteProjectPath.replace(".zip", "");
+      await fs.promises.rm(unzippedProjectPath, {
+         recursive: true,
+         force: true,
+      });
+      await fs.promises.mkdir(unzippedProjectPath, { recursive: true });
+
+      const zip = new AdmZip(absoluteProjectPath);
+      zip.extractAllTo(unzippedProjectPath, true);
+
+      return unzippedProjectPath;
+   }
+
    public async updateProject(project: ApiProject) {
       await this.finishedInitialization;
       if (this.publisherConfigIsFrozen) {
@@ -184,8 +218,24 @@ export class ProjectStore {
       }
    }
 
+   private async scaffoldProject(project: ApiProject) {
+      const projectName = project.name;
+      if (!projectName) {
+         throw new Error("Project name is required");
+      }
+      const absoluteProjectPath = `${publisherPath}/${projectName}`;
+      await fs.promises.mkdir(absoluteProjectPath, { recursive: true });
+      if (project.readme) {
+         await fs.promises.writeFile(
+            path.join(absoluteProjectPath, "README.md"),
+            project.readme,
+         );
+      }
+      return absoluteProjectPath;
+   }
+
    private async loadProjectIntoDisk(projectName: string, projectPath: string) {
-      const absoluteTargetPath = `/etc/publisher/${projectName}`;
+      const absoluteTargetPath = `${publisherPath}/${projectName}`;
       // Handle absolute paths
       if (projectPath.startsWith("/")) {
          try {
@@ -263,7 +313,7 @@ export class ProjectStore {
       throw new ProjectNotFoundError(errorMsg);
    }
 
-   private async mountLocalDirectory(
+   public async mountLocalDirectory(
       projectPath: string,
       absoluteTargetPath: string,
       projectName: string,
@@ -384,6 +434,21 @@ export class ProjectStore {
    async downloadGitHubDirectory(githubUrl: string, absoluteDirPath: string) {
       await fs.promises.rm(absoluteDirPath, { recursive: true, force: true });
       await fs.promises.mkdir(absoluteDirPath, { recursive: true });
-      await simpleGit().clone(githubUrl, absoluteDirPath);
+
+      await new Promise<void>((resolve, reject) => {
+         simpleGit().clone(githubUrl, absoluteDirPath, {}, (err) => {
+            if (err) {
+               console.error(err);
+               logger.error(
+                  `Failed to clone GitHub repository "${githubUrl}"`,
+                  {
+                     error: err,
+                  },
+               );
+               reject(err);
+            }
+            resolve();
+         });
+      });
    }
 }