@malloy-publisher/server 0.0.87 → 0.0.89

package/src/service/project_store.ts

@@ -1,24 +1,35 @@
-import * as fs from "fs/promises";
+import { GetObjectCommand, S3 } from "@aws-sdk/client-s3";
+import { Storage } from "@google-cloud/storage";
+import * as fs from "fs";
 import * as path from "path";
+import AdmZip from "adm-zip";
+import simpleGit from "simple-git";
+import { Writable } from "stream";
 import { components } from "../api";
-import { API_PREFIX } from "../constants";
+import { getPublisherConfig, isPublisherConfigFrozen } from "../config";
+import { API_PREFIX, PUBLISHER_CONFIG_NAME, publisherPath } from "../constants";
 import { FrozenConfigError, ProjectNotFoundError } from "../errors";
 import { logger } from "../logger";
-import { isPublisherConfigFrozen } from "../utils";
 import { Project } from "./project";
 type ApiProject = components["schemas"]["Project"];
 
 export class ProjectStore {
-   private serverRootPath: string;
+   public serverRootPath: string;
    private projects: Map<string, Project> = new Map();
    public publisherConfigIsFrozen: boolean;
+   public finishedInitialization: Promise<void>;
+   private s3Client = new S3({
+      followRegionRedirects: true,
+   });
+   private gcsClient = new Storage();
 
    constructor(serverRootPath: string) {
       this.serverRootPath = serverRootPath;
-      void this.initialize();
+      this.finishedInitialization = this.initialize();
    }
 
    private async initialize() {
+      const initialTime = performance.now();
       try {
          this.publisherConfigIsFrozen = isPublisherConfigFrozen(
             this.serverRootPath,
@@ -26,46 +37,55 @@ export class ProjectStore {
          const projectManifest = await ProjectStore.reloadProjectManifest(
             this.serverRootPath,
          );
-         for (const projectName of Object.keys(projectManifest.projects)) {
-            const projectPath = projectManifest.projects[projectName];
-            const absoluteProjectPath = path.join(
-               this.serverRootPath,
-               projectPath,
-            );
-            const project = await Project.create(
-               projectName,
-               absoluteProjectPath,
-            );
-            this.projects.set(projectName, project);
-         }
-         logger.info("Project store successfully initialized");
+         logger.info(`Initializing project store.`);
+         await Promise.all(
+            Object.keys(projectManifest.projects).map(async (projectName) => {
+               logger.info(`Adding project "${projectName}"`);
+               const project = await this.addProject(
+                  {
+                     name: projectName,
+                     resource: `${API_PREFIX}/projects/${projectName}`,
+                     location: projectManifest.projects[projectName],
+                  },
+                  true,
+               );
+               return project.listPackages();
+            }),
+         );
+         logger.info(
+            `Project store successfully initialized in ${performance.now() - initialTime}ms`,
+         );
       } catch (error) {
          logger.error("Error initializing project store", { error });
+         console.error(error);
          process.exit(1);
       }
    }
 
-   public listProjects() {
-      return Array.from(this.projects.values()).map(
-         (project) => project.metadata,
+   public async listProjects() {
+      await this.finishedInitialization;
+      return Promise.all(
+         Array.from(this.projects.values()).map((project) =>
+            project.serialize(),
+         ),
       );
    }
 
    public async getProject(
       projectName: string,
-      reload: boolean,
+      reload: boolean = false,
    ): Promise<Project> {
+      await this.finishedInitialization;
       let project = this.projects.get(projectName);
       if (project === undefined || reload) {
          const projectManifest = await ProjectStore.reloadProjectManifest(
             this.serverRootPath,
          );
-         if (
-            !projectManifest.projects ||
-            !projectManifest.projects[projectName]
-         ) {
+         const projectPath =
+            project?.metadata.location || projectManifest.projects[projectName];
+         if (!projectPath) {
             throw new ProjectNotFoundError(
-               `Project "${projectName}" not found in publisher`,
+               `Project "${projectName}" could not be resolved to a path.`,
             );
          }
          project = await this.addProject({
@@ -76,10 +96,17 @@ export class ProjectStore {
       return project;
    }
 
-   public async addProject(project: ApiProject) {
-      if (this.publisherConfigIsFrozen) {
+   public async addProject(
+      project: ApiProject,
+      skipInitialization: boolean = false,
+   ) {
+      if (!skipInitialization) {
+         await this.finishedInitialization;
+      }
+      if (!skipInitialization && this.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
       }
+
       const projectName = project.name;
       if (!projectName) {
          throw new Error("Project name is required");
@@ -87,24 +114,48 @@ export class ProjectStore {
       const projectManifest = await ProjectStore.reloadProjectManifest(
          this.serverRootPath,
       );
-      const projectPath = projectManifest.projects[projectName];
-      if (!projectPath) {
-         throw new ProjectNotFoundError(
-            `Project "${projectName}" not found in publisher.config.json`,
-         );
-      }
-      const absoluteProjectPath = path.join(this.serverRootPath, projectPath);
-      if (!(await fs.stat(absoluteProjectPath)).isDirectory()) {
-         throw new ProjectNotFoundError(
-            `Project ${projectName} not found in ${absoluteProjectPath}`,
+      const projectPath =
+         project.location || projectManifest.projects[projectName];
+      let absoluteProjectPath: string;
+      if (projectPath) {
+         absoluteProjectPath = await this.loadProjectIntoDisk(
+            projectName,
+            projectPath,
          );
+         if (absoluteProjectPath.endsWith(".zip")) {
+            absoluteProjectPath = await this.unzipProject(absoluteProjectPath);
+         }
+      } else {
+         absoluteProjectPath = await this.scaffoldProject(project);
       }
-      const newProject = await Project.create(projectName, absoluteProjectPath);
+      const newProject = await Project.create(
+         projectName,
+         absoluteProjectPath,
+         project.connections || [],
+      );
       this.projects.set(projectName, newProject);
       return newProject;
    }
 
+   public async unzipProject(absoluteProjectPath: string) {
+      logger.info(
+         `Detected zip file at "${absoluteProjectPath}". Unzipping...`,
+      );
+      const unzippedProjectPath = absoluteProjectPath.replace(".zip", "");
+      await fs.promises.rm(unzippedProjectPath, {
+         recursive: true,
+         force: true,
+      });
+      await fs.promises.mkdir(unzippedProjectPath, { recursive: true });
+
+      const zip = new AdmZip(absoluteProjectPath);
+      zip.extractAllTo(unzippedProjectPath, true);
+
+      return unzippedProjectPath;
+   }
+
    public async updateProject(project: ApiProject) {
+      await this.finishedInitialization;
       if (this.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
       }
@@ -122,6 +173,7 @@ export class ProjectStore {
    }
 
    public async deleteProject(projectName: string) {
+      await this.finishedInitialization;
       if (this.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
       }
@@ -133,26 +185,20 @@ export class ProjectStore {
       return project;
    }
 
-   private static async reloadProjectManifest(
-      serverRootPath: string,
-   ): Promise<{ projects: { [key: string]: string } }> {
+   public static async reloadProjectManifest(serverRootPath: string) {
       try {
-         const projectManifestContent = await fs.readFile(
-            path.join(serverRootPath, "publisher.config.json"),
-            "utf8",
-         );
-         return JSON.parse(projectManifestContent);
+         return getPublisherConfig(serverRootPath);
       } catch (error) {
          if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
             logger.error(
-               `Error reading publisher.config.json. Generating from directory`,
+               `Error reading ${PUBLISHER_CONFIG_NAME}. Generating from directory`,
                { error },
             );
             return { projects: {} };
          } else {
             // If publisher.config.json is missing, generate the manifest from directories
             try {
-               const entries = await fs.readdir(serverRootPath, {
+               const entries = await fs.promises.readdir(serverRootPath, {
                   withFileTypes: true,
                });
                const projects: { [key: string]: string } = {};
@@ -171,4 +217,238 @@ export class ProjectStore {
          }
       }
    }
+
+   private async scaffoldProject(project: ApiProject) {
+      const projectName = project.name;
+      if (!projectName) {
+         throw new Error("Project name is required");
+      }
+      const absoluteProjectPath = `${publisherPath}/${projectName}`;
+      await fs.promises.mkdir(absoluteProjectPath, { recursive: true });
+      if (project.readme) {
+         await fs.promises.writeFile(
+            path.join(absoluteProjectPath, "README.md"),
+            project.readme,
+         );
+      }
+      return absoluteProjectPath;
+   }
+
+   private async loadProjectIntoDisk(projectName: string, projectPath: string) {
+      const absoluteTargetPath = `${publisherPath}/${projectName}`;
+      // Handle absolute paths
+      if (projectPath.startsWith("/")) {
+         try {
+            logger.info(`Mounting local directory at "${projectPath}"`);
+            await this.mountLocalDirectory(
+               projectPath,
+               absoluteTargetPath,
+               projectName,
+            );
+            return absoluteTargetPath;
+         } catch (error) {
+            logger.error(`Failed to mount local directory "${projectPath}"`, {
+               error,
+            });
+            throw error;
+         }
+      }
+
+      // Handle GCS URIs
+      if (projectPath.startsWith("gs://")) {
+         // Download from GCS
+         try {
+            logger.info(
+               `Downloading GCS path "${projectPath}" to "${absoluteTargetPath}"`,
+            );
+            await this.downloadGcsDirectory(
+               projectPath,
+               projectName,
+               absoluteTargetPath,
+            );
+         } catch (error) {
+            logger.error(`Failed to download GCS path "${projectPath}"`, {
+               error,
+            });
+            throw error;
+         }
+         return absoluteTargetPath;
+      }
+
+      // Handle S3 URIs
+      if (projectPath.startsWith("s3://")) {
+         try {
+            logger.info(`Mounting S3 path "${projectPath}"`);
+            await this.downloadS3Directory(
+               projectPath,
+               projectName,
+               absoluteTargetPath,
+            );
+            return absoluteTargetPath;
+         } catch (error) {
+            logger.error(`Failed to mount S3 path "${projectPath}"`, { error });
+            throw error;
+         }
+      }
+
+      // Handle GitHub URIs
+      if (
+         projectPath.startsWith("https://github.com/") ||
+         projectPath.startsWith("git@")
+      ) {
+         try {
+            logger.info(`Mounting GitHub path "${projectPath}"`);
+            await this.downloadGitHubDirectory(projectPath, absoluteTargetPath);
+            return absoluteTargetPath;
+         } catch (error) {
+            logger.error(`Failed to mount GitHub path "${projectPath}"`, {
+               error,
+            });
+            throw error;
+         }
+      }
+
+      const errorMsg = `Invalid project path: "${projectPath}". Must be an absolute mounted path or a GCS/S3/GitHub URI.`;
+      logger.error(errorMsg, { projectName, projectPath });
+      throw new ProjectNotFoundError(errorMsg);
+   }
+
+   public async mountLocalDirectory(
+      projectPath: string,
+      absoluteTargetPath: string,
+      projectName: string,
+   ) {
+      const projectDirExists = (
+         await fs.promises.stat(projectPath)
+      ).isDirectory();
+      if (projectDirExists) {
+         await fs.promises.rm(absoluteTargetPath, {
+            recursive: true,
+            force: true,
+         });
+         await fs.promises.mkdir(absoluteTargetPath, { recursive: true });
+         await fs.promises.cp(projectPath, absoluteTargetPath, {
+            recursive: true,
+         });
+      } else {
+         throw new ProjectNotFoundError(
+            `Project ${projectName} not found in "${projectPath}"`,
+         );
+      }
+   }
+
+   async downloadGcsDirectory(
+      gcsPath: string,
+      projectName: string,
+      absoluteDirPath: string,
+   ) {
+      const trimmedPath = gcsPath.slice(5);
+      const [bucketName, ...prefixParts] = trimmedPath.split("/");
+      const prefix = prefixParts.join("/");
+      const [files] = await this.gcsClient.bucket(bucketName).getFiles({
+         prefix,
+      });
+      if (files.length === 0) {
+         throw new ProjectNotFoundError(
+            `Project ${projectName} not found in ${gcsPath}`,
+         );
+      }
+      await fs.promises.rm(absoluteDirPath, { recursive: true, force: true });
+      await fs.promises.mkdir(absoluteDirPath, { recursive: true });
+      await Promise.all(
+         files.map(async (file) => {
+            const relativeFilePath = file.name.replace(prefix, "");
+            const absoluteFilePath = path.join(
+               absoluteDirPath,
+               relativeFilePath,
+            );
+            if (file.name.endsWith("/")) {
+               return;
+            }
+            await fs.promises.mkdir(path.dirname(absoluteFilePath), {
+               recursive: true,
+            });
+            return fs.promises.writeFile(
+               absoluteFilePath,
+               await file.download(),
+            );
+         }),
+      );
+   }
+
+   async downloadS3Directory(
+      s3Path: string,
+      projectName: string,
+      absoluteDirPath: string,
+   ) {
+      const trimmedPath = s3Path.slice(5);
+      const [bucketName, ...prefixParts] = trimmedPath.split("/");
+      const prefix = prefixParts.join("/");
+      const objects = await this.s3Client.listObjectsV2({
+         Bucket: bucketName,
+         Prefix: prefix,
+      });
+      await fs.promises.rm(absoluteDirPath, { recursive: true, force: true });
+      await fs.promises.mkdir(absoluteDirPath, { recursive: true });
+
+      if (!objects.Contents || objects.Contents.length === 0) {
+         throw new ProjectNotFoundError(
+            `Project ${projectName} not found in ${s3Path}`,
+         );
+      }
+      await Promise.all(
+         objects.Contents?.map(async (object) => {
+            const key = object.Key;
+            if (!key) {
+               return;
+            }
+            const relativeFilePath = key.replace(prefix, "");
+            if (!relativeFilePath || relativeFilePath.endsWith("/")) {
+               return;
+            }
+            const absoluteFilePath = path.join(
+               absoluteDirPath,
+               relativeFilePath,
+            );
+            await fs.promises.mkdir(path.dirname(absoluteFilePath), {
+               recursive: true,
+            });
+            const command = new GetObjectCommand({
+               Bucket: bucketName,
+               Key: key,
+            });
+            const item = await this.s3Client.send(command);
+            if (!item.Body) {
+               return;
+            }
+            const file = fs.createWriteStream(absoluteFilePath);
+            item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
+            await new Promise<void>((resolve, reject) => {
+               file.on("error", reject);
+               file.on("finish", resolve);
+            });
+         }),
+      );
+   }
+
+   async downloadGitHubDirectory(githubUrl: string, absoluteDirPath: string) {
+      await fs.promises.rm(absoluteDirPath, { recursive: true, force: true });
+      await fs.promises.mkdir(absoluteDirPath, { recursive: true });
+
+      await new Promise<void>((resolve, reject) => {
+         simpleGit().clone(githubUrl, absoluteDirPath, {}, (err) => {
+            if (err) {
+               console.error(err);
+               logger.error(
+                  `Failed to clone GitHub repository "${githubUrl}"`,
+                  {
+                     error: err,
+                  },
+               );
+               reject(err);
+            }
+            resolve();
+         });
+      });
+   }
 }

package/src/utils.ts

@@ -1,15 +1,7 @@
 import { URLReader } from "@malloydata/malloy";
 import * as fs from "fs";
-import path from "path";
 import { fileURLToPath } from "url";
 
-export const PACKAGE_MANIFEST_NAME = "publisher.json";
-export const CONNECTIONS_MANIFEST_NAME = "publisher.connections.json";
-export const MODEL_FILE_SUFFIX = ".malloy";
-export const NOTEBOOK_FILE_SUFFIX = ".malloynb";
-// TODO: Move this to server config.
-export const ROW_LIMIT = 1000;
-
 export const URL_READER: URLReader = {
    readURL: (url: URL) => {
       let path = url.toString();
@@ -20,13 +12,3 @@ export const URL_READER: URLReader = {
    },
 };
 
-export const isPublisherConfigFrozen = (serverRoot: string) => {
-   const publisherConfigPath = path.join(serverRoot, "publisher.config.json");
-   if (!fs.existsSync(publisherConfigPath)) {
-      return false;
-   }
-   const publisherConfig = JSON.parse(
-      fs.readFileSync(publisherConfigPath, "utf8"),
-   );
-   return Boolean(publisherConfig.frozenConfig);
-};

package/tests/harness/mcp_test_setup.ts

@@ -1,14 +1,14 @@
-import http from "http";
-import { AddressInfo } from "net";
-import { URL } from "url";
-import path from "path";
 import { Client } from "@modelcontextprotocol/sdk/client/index.js";
 import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
 import {
-   Request,
    Notification,
+   Request,
    Result,
 } from "@modelcontextprotocol/sdk/types.js";
+import http from "http";
+import { AddressInfo } from "net";
+import path from "path";
+import { URL } from "url";
 
 // --- Real Server Import ---
 // Import the actual configured Express app instance

package/dist/app/assets/RenderedResult-BAZuT25g-BMU632YI.js

@@ -1,2 +0,0 @@
-const __vite__mapDeps=(i,m=__vite__mapDeps,d=(m.f||(m.f=["assets/index.umd-x-naS8R7.js","assets/index-C7whj6wK.js","assets/index-2xWCh-ya.css"])))=>i.map(i=>d[i]);
-import{j as a,r as c,_ as w}from"./index-C7whj6wK.js";const x=async e=>{if(typeof window>"u")throw new Error("MalloyRenderer can only be used in browser environment");const{MalloyRenderer:d}=await w(async()=>{const{MalloyRenderer:i}=await import("./index.umd-x-naS8R7.js").then(l=>l.i);return{MalloyRenderer:i}},__vite__mapDeps([0,1,2]));return new d({onClick:e}).createViz()};function m({result:e,height:d,isFillElement:i,onSizeChange:l,onDrill:y}){const u=c.useRef(null),[g,f]=c.useState(!1);return c.useLayoutEffect(()=>{if(!u.current||!e)return;let t=!0;const r=u.current;for(;r.firstChild;)r.removeChild(r.firstChild);return x(y).then(o=>{if(!t)return;const n=new MutationObserver(s=>{for(const h of s)if(h.type==="childList"&&h.addedNodes.length>0&&Array.from(h.addedNodes).some(p=>p.nodeType===Node.ELEMENT_NODE)){n.disconnect(),setTimeout(()=>{t&&f(!0)},50);break}});n.observe(r,{childList:!0,subtree:!0,characterData:!0});try{o.setResult(JSON.parse(e)),o.render(r)}catch(s){console.error("Error rendering visualization:",s),n.disconnect(),t&&f(!0)}}).catch(o=>{console.error("Failed to create renderer:",o),t&&f(!0)}),()=>{t=!1}},[e,y]),c.useEffect(()=>{if(!u.current||!g)return;const t=u.current,r=()=>{if(t){const s=t.offsetHeight;s>0?l&&l(s):i&&t.firstChild&&(t.firstChild.offsetHeight==0?i(!0):i(!1))}},o=setTimeout(r,100);let n=null;return n=new MutationObserver(r),n.observe(t,{childList:!0,subtree:!0,attributes:!0}),()=>{clearTimeout(o),n?.disconnect()}},[l,e,i,g]),a.jsx("div",{ref:u,style:{width:"100%",height:d?`${d}px`:"100%"}})}function E(e){return typeof window>"u"?a.jsx("div",{style:{width:"100%",height:e.height?`${e.height}px`:"100%",display:"flex",alignItems:"center",justifyContent:"center",color:"#666"},children:"Loading..."}):a.jsx(c.Suspense,{fallback:a.jsx("div",{style:{width:"100%",height:e.height?`${e.height}px`:"100%",display:"flex",alignItems:"center",justifyContent:"center",color:"#666"},children:"Loading visualization..."}),children:a.jsx(m,{...e})})}export{E as default};