@malloy-publisher/server 0.0.87 → 0.0.88

package/src/service/project_store.ts

@@ -1,24 +1,34 @@
-import * as fs from "fs/promises";
+import { GetObjectCommand, S3 } from "@aws-sdk/client-s3";
+import { Storage } from "@google-cloud/storage";
+import * as fs from "fs";
 import * as path from "path";
+import simpleGit from "simple-git";
+import { Writable } from "stream";
 import { components } from "../api";
-import { API_PREFIX } from "../constants";
+import { getPublisherConfig, isPublisherConfigFrozen } from "../config";
+import { API_PREFIX, PUBLISHER_CONFIG_NAME } from "../constants";
 import { FrozenConfigError, ProjectNotFoundError } from "../errors";
 import { logger } from "../logger";
-import { isPublisherConfigFrozen } from "../utils";
 import { Project } from "./project";
 type ApiProject = components["schemas"]["Project"];
 
 export class ProjectStore {
-   private serverRootPath: string;
+   public serverRootPath: string;
    private projects: Map<string, Project> = new Map();
    public publisherConfigIsFrozen: boolean;
+   public finishedInitialization: Promise<void>;
+   private s3Client = new S3({
+      followRegionRedirects: true,
+   });
+   private gcsClient = new Storage();
 
    constructor(serverRootPath: string) {
       this.serverRootPath = serverRootPath;
-      void this.initialize();
+      this.finishedInitialization = this.initialize();
    }
 
    private async initialize() {
+      const initialTime = performance.now();
       try {
          this.publisherConfigIsFrozen = isPublisherConfigFrozen(
             this.serverRootPath,
@@ -26,26 +36,32 @@ export class ProjectStore {
          const projectManifest = await ProjectStore.reloadProjectManifest(
             this.serverRootPath,
          );
-         for (const projectName of Object.keys(projectManifest.projects)) {
-            const projectPath = projectManifest.projects[projectName];
-            const absoluteProjectPath = path.join(
-               this.serverRootPath,
-               projectPath,
-            );
-            const project = await Project.create(
-               projectName,
-               absoluteProjectPath,
-            );
-            this.projects.set(projectName, project);
-         }
-         logger.info("Project store successfully initialized");
+         logger.info(`Initializing project store.`);
+         await Promise.all(
+            Object.keys(projectManifest.projects).map(async (projectName) => {
+               logger.info(`Adding project "${projectName}"`);
+               const project = await this.addProject(
+                  {
+                     name: projectName,
+                     resource: `${API_PREFIX}/projects/${projectName}`,
+                     location: projectManifest.projects[projectName],
+                  },
+                  true,
+               );
+               return project.listPackages();
+            }),
+         );
+         logger.info(
+            `Project store successfully initialized in ${performance.now() - initialTime}ms`,
+         );
       } catch (error) {
          logger.error("Error initializing project store", { error });
          process.exit(1);
       }
    }
 
-   public listProjects() {
+   public async listProjects() {
+      await this.finishedInitialization;
       return Array.from(this.projects.values()).map(
          (project) => project.metadata,
       );
@@ -55,17 +71,17 @@ export class ProjectStore {
       projectName: string,
       reload: boolean,
    ): Promise<Project> {
+      await this.finishedInitialization;
       let project = this.projects.get(projectName);
       if (project === undefined || reload) {
          const projectManifest = await ProjectStore.reloadProjectManifest(
             this.serverRootPath,
          );
-         if (
-            !projectManifest.projects ||
-            !projectManifest.projects[projectName]
-         ) {
+         const projectPath =
+            project?.metadata.location || projectManifest.projects[projectName];
+         if (!projectPath) {
             throw new ProjectNotFoundError(
-               `Project "${projectName}" not found in publisher`,
+               `Project "${projectName}" could not be resolved to a path.`,
             );
          }
          project = await this.addProject({
@@ -76,7 +92,13 @@ export class ProjectStore {
       return project;
    }
 
-   public async addProject(project: ApiProject) {
+   public async addProject(
+      project: ApiProject,
+      skipInitialization: boolean = false,
+   ) {
+      if (!skipInitialization) {
+         await this.finishedInitialization;
+      }
       if (this.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
       }
@@ -87,24 +109,19 @@ export class ProjectStore {
       const projectManifest = await ProjectStore.reloadProjectManifest(
          this.serverRootPath,
       );
-      const projectPath = projectManifest.projects[projectName];
-      if (!projectPath) {
-         throw new ProjectNotFoundError(
-            `Project "${projectName}" not found in publisher.config.json`,
-         );
-      }
-      const absoluteProjectPath = path.join(this.serverRootPath, projectPath);
-      if (!(await fs.stat(absoluteProjectPath)).isDirectory()) {
-         throw new ProjectNotFoundError(
-            `Project ${projectName} not found in ${absoluteProjectPath}`,
-         );
-      }
+      const projectPath =
+         project.location || projectManifest.projects[projectName];
+      const absoluteProjectPath = await this.loadProjectIntoDisk(
+         projectName,
+         projectPath,
+      );
       const newProject = await Project.create(projectName, absoluteProjectPath);
       this.projects.set(projectName, newProject);
       return newProject;
    }
 
    public async updateProject(project: ApiProject) {
+      await this.finishedInitialization;
       if (this.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
       }
@@ -122,6 +139,7 @@ export class ProjectStore {
    }
 
    public async deleteProject(projectName: string) {
+      await this.finishedInitialization;
       if (this.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
       }
@@ -133,26 +151,20 @@ export class ProjectStore {
       return project;
    }
 
-   private static async reloadProjectManifest(
-      serverRootPath: string,
-   ): Promise<{ projects: { [key: string]: string } }> {
+   public static async reloadProjectManifest(serverRootPath: string) {
       try {
-         const projectManifestContent = await fs.readFile(
-            path.join(serverRootPath, "publisher.config.json"),
-            "utf8",
-         );
-         return JSON.parse(projectManifestContent);
+         return getPublisherConfig(serverRootPath);
       } catch (error) {
          if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
             logger.error(
-               `Error reading publisher.config.json. Generating from directory`,
+               `Error reading ${PUBLISHER_CONFIG_NAME}. Generating from directory`,
                { error },
             );
             return { projects: {} };
          } else {
             // If publisher.config.json is missing, generate the manifest from directories
             try {
-               const entries = await fs.readdir(serverRootPath, {
+               const entries = await fs.promises.readdir(serverRootPath, {
                   withFileTypes: true,
                });
                const projects: { [key: string]: string } = {};
@@ -171,4 +183,207 @@ export class ProjectStore {
          }
       }
    }
+
+   private async loadProjectIntoDisk(projectName: string, projectPath: string) {
+      const absoluteTargetPath = `/etc/publisher/${projectName}`;
+      // Handle absolute paths
+      if (projectPath.startsWith("/")) {
+         try {
+            logger.info(`Mounting local directory at "${projectPath}"`);
+            await this.mountLocalDirectory(
+               projectPath,
+               absoluteTargetPath,
+               projectName,
+            );
+            return absoluteTargetPath;
+         } catch (error) {
+            logger.error(`Failed to mount local directory "${projectPath}"`, {
+               error,
+            });
+            throw error;
+         }
+      }
+
+      // Handle GCS URIs
+      if (projectPath.startsWith("gs://")) {
+         // Download from GCS
+         try {
+            logger.info(
+               `Downloading GCS path "${projectPath}" to "${absoluteTargetPath}"`,
+            );
+            await this.downloadGcsDirectory(
+               projectPath,
+               projectName,
+               absoluteTargetPath,
+            );
+         } catch (error) {
+            logger.error(`Failed to download GCS path "${projectPath}"`, {
+               error,
+            });
+            throw error;
+         }
+         return absoluteTargetPath;
+      }
+
+      // Handle S3 URIs
+      if (projectPath.startsWith("s3://")) {
+         try {
+            logger.info(`Mounting S3 path "${projectPath}"`);
+            await this.downloadS3Directory(
+               projectPath,
+               projectName,
+               absoluteTargetPath,
+            );
+            return absoluteTargetPath;
+         } catch (error) {
+            logger.error(`Failed to mount S3 path "${projectPath}"`, { error });
+            throw error;
+         }
+      }
+
+      // Handle GitHub URIs
+      if (
+         projectPath.startsWith("https://github.com/") ||
+         projectPath.startsWith("git@")
+      ) {
+         try {
+            logger.info(`Mounting GitHub path "${projectPath}"`);
+            await this.downloadGitHubDirectory(projectPath, absoluteTargetPath);
+            return absoluteTargetPath;
+         } catch (error) {
+            logger.error(`Failed to mount GitHub path "${projectPath}"`, {
+               error,
+            });
+            throw error;
+         }
+      }
+
+      const errorMsg = `Invalid project path: "${projectPath}". Must be an absolute mounted path or a GCS/S3/GitHub URI.`;
+      logger.error(errorMsg, { projectName, projectPath });
+      throw new ProjectNotFoundError(errorMsg);
+   }
+
+   private async mountLocalDirectory(
+      projectPath: string,
+      absoluteTargetPath: string,
+      projectName: string,
+   ) {
+      const projectDirExists = (
+         await fs.promises.stat(projectPath)
+      ).isDirectory();
+      if (projectDirExists) {
+         await fs.promises.rm(absoluteTargetPath, {
+            recursive: true,
+            force: true,
+         });
+         await fs.promises.mkdir(absoluteTargetPath, { recursive: true });
+         await fs.promises.cp(projectPath, absoluteTargetPath, {
+            recursive: true,
+         });
+      } else {
+         throw new ProjectNotFoundError(
+            `Project ${projectName} not found in "${projectPath}"`,
+         );
+      }
+   }
+
+   async downloadGcsDirectory(
+      gcsPath: string,
+      projectName: string,
+      absoluteDirPath: string,
+   ) {
+      const trimmedPath = gcsPath.slice(5);
+      const [bucketName, ...prefixParts] = trimmedPath.split("/");
+      const prefix = prefixParts.join("/");
+      const [files] = await this.gcsClient.bucket(bucketName).getFiles({
+         prefix,
+      });
+      if (files.length === 0) {
+         throw new ProjectNotFoundError(
+            `Project ${projectName} not found in ${gcsPath}`,
+         );
+      }
+      await fs.promises.rm(absoluteDirPath, { recursive: true, force: true });
+      await fs.promises.mkdir(absoluteDirPath, { recursive: true });
+      await Promise.all(
+         files.map(async (file) => {
+            const relativeFilePath = file.name.replace(prefix, "");
+            const absoluteFilePath = path.join(
+               absoluteDirPath,
+               relativeFilePath,
+            );
+            if (file.name.endsWith("/")) {
+               return;
+            }
+            await fs.promises.mkdir(path.dirname(absoluteFilePath), {
+               recursive: true,
+            });
+            return fs.promises.writeFile(
+               absoluteFilePath,
+               await file.download(),
+            );
+         }),
+      );
+   }
+
+   async downloadS3Directory(
+      s3Path: string,
+      projectName: string,
+      absoluteDirPath: string,
+   ) {
+      const trimmedPath = s3Path.slice(5);
+      const [bucketName, ...prefixParts] = trimmedPath.split("/");
+      const prefix = prefixParts.join("/");
+      const objects = await this.s3Client.listObjectsV2({
+         Bucket: bucketName,
+         Prefix: prefix,
+      });
+      await fs.promises.rm(absoluteDirPath, { recursive: true, force: true });
+      await fs.promises.mkdir(absoluteDirPath, { recursive: true });
+
+      if (!objects.Contents || objects.Contents.length === 0) {
+         throw new ProjectNotFoundError(
+            `Project ${projectName} not found in ${s3Path}`,
+         );
+      }
+      await Promise.all(
+         objects.Contents?.map(async (object) => {
+            const key = object.Key;
+            if (!key) {
+               return;
+            }
+            const relativeFilePath = key.replace(prefix, "");
+            if (!relativeFilePath || relativeFilePath.endsWith("/")) {
+               return;
+            }
+            const absoluteFilePath = path.join(
+               absoluteDirPath,
+               relativeFilePath,
+            );
+            await fs.promises.mkdir(path.dirname(absoluteFilePath), {
+               recursive: true,
+            });
+            const command = new GetObjectCommand({
+               Bucket: bucketName,
+               Key: key,
+            });
+            const item = await this.s3Client.send(command);
+            if (!item.Body) {
+               return;
+            }
+            const file = fs.createWriteStream(absoluteFilePath);
+            item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
+            await new Promise<void>((resolve, reject) => {
+               file.on("error", reject);
+               file.on("finish", resolve);
+            });
+         }),
+      );
+   }
+
+   async downloadGitHubDirectory(githubUrl: string, absoluteDirPath: string) {
+      await fs.promises.rm(absoluteDirPath, { recursive: true, force: true });
+      await fs.promises.mkdir(absoluteDirPath, { recursive: true });
+      await simpleGit().clone(githubUrl, absoluteDirPath);
+   }
 }

package/src/utils.ts

@@ -1,15 +1,7 @@
 import { URLReader } from "@malloydata/malloy";
 import * as fs from "fs";
-import path from "path";
 import { fileURLToPath } from "url";
 
-export const PACKAGE_MANIFEST_NAME = "publisher.json";
-export const CONNECTIONS_MANIFEST_NAME = "publisher.connections.json";
-export const MODEL_FILE_SUFFIX = ".malloy";
-export const NOTEBOOK_FILE_SUFFIX = ".malloynb";
-// TODO: Move this to server config.
-export const ROW_LIMIT = 1000;
-
 export const URL_READER: URLReader = {
    readURL: (url: URL) => {
       let path = url.toString();
@@ -20,13 +12,3 @@ export const URL_READER: URLReader = {
    },
 };
 
-export const isPublisherConfigFrozen = (serverRoot: string) => {
-   const publisherConfigPath = path.join(serverRoot, "publisher.config.json");
-   if (!fs.existsSync(publisherConfigPath)) {
-      return false;
-   }
-   const publisherConfig = JSON.parse(
-      fs.readFileSync(publisherConfigPath, "utf8"),
-   );
-   return Boolean(publisherConfig.frozenConfig);
-};