@malloy-publisher/server 0.0.106 → 0.0.108

package/src/service/project.ts

@@ -35,23 +35,22 @@ export class Project {
    private packageStatuses: Map<string, PackageInfo> = new Map();
    private malloyConnections: Map<string, BaseConnection>;
    private apiConnections: ApiConnection[];
-   private internalConnections: InternalConnection[];
    private projectPath: string;
    private projectName: string;
+   private serverRootPath: string;
    public metadata: ApiProject;
 
    constructor(
       projectName: string,
       projectPath: string,
       malloyConnections: Map<string, BaseConnection>,
-      internalConnections: InternalConnection[],
       apiConnections: InternalConnection[],
+      serverRootPath: string,
    ) {
       this.projectName = projectName;
       this.projectPath = projectPath;
+      this.serverRootPath = serverRootPath;
       this.malloyConnections = malloyConnections;
-      // InternalConnections have full connection details for doing schema inspection
-      this.internalConnections = internalConnections;
       this.apiConnections = apiConnections;
       this.metadata = {
          resource: `${API_PREFIX}/projects/${this.projectName}`,
@@ -82,7 +81,6 @@ export class Project {
          // Update the project's connection maps
          this.malloyConnections = malloyConnections;
          this.apiConnections = apiConnections;
-         this.internalConnections = apiConnections;
 
          logger.info(
             `Successfully updated connections for project ${this.projectName}`,
@@ -101,6 +99,7 @@ export class Project {
       projectName: string,
       projectPath: string,
       defaultConnections: ApiConnection[],
+      serverRootPath?: string,
    ): Promise<Project> {
       if (!(await fs.promises.stat(projectPath)).isDirectory()) {
          throw new ProjectNotFoundError(
@@ -112,7 +111,12 @@ export class Project {
       let apiConnections: InternalConnection[] = [];
 
       logger.info(`Creating project with connection configuration`);
-      const result = await createConnections(projectPath, defaultConnections);
+      const result = await createConnections(
+         projectPath,
+         defaultConnections,
+         projectName,
+         serverRootPath,
+      );
       malloyConnections = result.malloyConnections;
       apiConnections = result.apiConnections;
 
@@ -129,17 +133,7 @@ export class Project {
          projectPath,
          malloyConnections,
          apiConnections,
-         apiConnections.map((internalConnection) => {
-            // Create a new ApiConnection object from each InternalConnection
-            // by excluding the internal connection details
-            // We don't want to send passwords and connection strings to the client
-            return {
-               name: internalConnection.name,
-               type: internalConnection.type,
-               attributes: internalConnection.attributes,
-               resource: internalConnection.resource,
-            };
-         }),
+         serverRootPath || "",
       );
    }
 
@@ -176,20 +170,6 @@ export class Project {
       return connection;
    }
 
-   // Returns a connection with full connection details for doing schema inspection
-   // Don't send this to the client as it contains sensitive information
-   public getInternalConnection(connectionName: string): InternalConnection {
-      const connection = this.internalConnections.find(
-         (connection) => connection.name === connectionName,
-      );
-      if (!connection) {
-         throw new ConnectionNotFoundError(
-            `Connection ${connectionName} not found`,
-         );
-      }
-      return connection;
-   }
-
    public getMalloyConnection(connectionName: string): BaseConnection {
       const connection = this.malloyConnections.get(connectionName);
       if (!connection) {
@@ -290,6 +270,7 @@ export class Project {
                packageName,
                path.join(this.projectPath, packageName),
                this.malloyConnections,
+               this.serverRootPath,
             );
             this.packages.set(packageName, _package);
 
@@ -334,6 +315,7 @@ export class Project {
                packageName,
                packagePath,
                this.malloyConnections,
+               this.serverRootPath,
             ),
          );
       } catch (error) {

package/src/service/project_store.spec.ts

@@ -15,19 +15,6 @@ const serverRootPath = path.join(
    "pathways-worker-publisher-project-store-test",
 );
 const projectName = "organizationName-projectName";
-const testConnections: Connection[] = [
-   {
-      name: "testConnection",
-      type: "postgres",
-      postgresConnection: {
-         host: "host",
-         port: 1234,
-         databaseName: "databaseName",
-         userName: "userName",
-         password: "password",
-      },
-   },
-];
 
 let sandbox: sinon.SinonSandbox;
 
@@ -72,13 +59,6 @@ describe("ProjectStore Service", () => {
       const projectPath = path.join(serverRootPath, projectName);
       mkdirSync(projectPath, { recursive: true });
 
-      // Create connections file
-      const connectionsPath = path.join(
-         projectPath,
-         "publisher.connections.json",
-      );
-      writeFileSync(connectionsPath, JSON.stringify(testConnections));
-
       // Create publisher config
       const publisherConfigPath = path.join(
          serverRootPath,
@@ -87,6 +67,7 @@ describe("ProjectStore Service", () => {
       writeFileSync(
          publisherConfigPath,
          JSON.stringify({
+            frozenConfig: false,
             projects: [
                {
                   name: projectName,
@@ -96,6 +77,12 @@ describe("ProjectStore Service", () => {
                         location: projectPath,
                      },
                   ],
+                  connections: [
+                     {
+                        name: "testConnection",
+                        type: "postgres",
+                     },
+                  ],
                },
             ],
          }),
@@ -117,16 +104,6 @@ describe("ProjectStore Service", () => {
       mkdirSync(projectPath1, { recursive: true });
       mkdirSync(projectPath2, { recursive: true });
 
-      // Create connections files
-      writeFileSync(
-         path.join(projectPath1, "publisher.connections.json"),
-         JSON.stringify(testConnections),
-      );
-      writeFileSync(
-         path.join(projectPath2, "publisher.connections.json"),
-         JSON.stringify(testConnections),
-      );
-
       // Create publisher config
       const publisherConfigPath = path.join(
          serverRootPath,
@@ -135,6 +112,7 @@ describe("ProjectStore Service", () => {
       writeFileSync(
          publisherConfigPath,
          JSON.stringify({
+            frozenConfig: false,
             projects: [
                {
                   name: projectName1,
@@ -144,6 +122,12 @@ describe("ProjectStore Service", () => {
                         location: projectPath1,
                      },
                   ],
+                  connections: [
+                     {
+                        name: "testConnection",
+                        type: "postgres",
+                     },
+                  ],
                },
                {
                   name: projectName2,
@@ -153,6 +137,13 @@ describe("ProjectStore Service", () => {
                         location: projectPath2,
                      },
                   ],
+                  connections: [
+                     {
+                        name: "testConnection2",
+                        type: "bigquery",
+                        bigqueryConnection: {},
+                     },
+                  ],
                },
             ],
          }),
@@ -174,14 +165,6 @@ describe("ProjectStore Service", () => {
       // Create a project directory
       const projectPath = path.join(serverRootPath, projectName);
       mkdirSync(projectPath, { recursive: true });
-
-      // Create connections file
-      const connectionsPath = path.join(
-         projectPath,
-         "publisher.connections.json",
-      );
-      writeFileSync(connectionsPath, JSON.stringify(testConnections));
-
       // Create publisher config
       const publisherConfigPath = path.join(
          serverRootPath,
@@ -221,13 +204,6 @@ describe("ProjectStore Service", () => {
       const projectPath = path.join(serverRootPath, projectName);
       mkdirSync(projectPath, { recursive: true });
 
-      // Create connections file
-      const connectionsPath = path.join(
-         projectPath,
-         "publisher.connections.json",
-      );
-      writeFileSync(connectionsPath, JSON.stringify(testConnections));
-
       // Create publisher config
       const publisherConfigPath = path.join(
          serverRootPath,
@@ -310,14 +286,6 @@ describe("ProjectStore Service", () => {
       const projectPath = path.join(serverRootPath, projectName);
       mkdirSync(projectPath, { recursive: true });
 
-      // Create connections file
-      const connectionsPath = path.join(
-         projectPath,
-         "publisher.connections.json",
-      );
-      writeFileSync(connectionsPath, JSON.stringify(testConnections));
-
-      // Create publisher config
       const publisherConfigPath = path.join(
          serverRootPath,
          "publisher.config.json",
@@ -325,6 +293,7 @@ describe("ProjectStore Service", () => {
       writeFileSync(
          publisherConfigPath,
          JSON.stringify({
+            frozenConfig: false,
             projects: [
                {
                   name: projectName,
@@ -334,11 +303,19 @@ describe("ProjectStore Service", () => {
                         location: projectPath,
                      },
                   ],
+                  connections: [
+                     {
+                        name: "testConnection",
+                        type: "postgres",
+                     },
+                  ],
                },
             ],
          }),
       );
 
+      await projectStore.finishedInitialization;
+
       // Test concurrent access to the same project
       const promises = Array.from({ length: 5 }, () =>
          projectStore.getProject(projectName),

package/src/service/project_store.ts

@@ -6,13 +6,13 @@ import * as path from "path";
 import simpleGit from "simple-git";
 import { Writable } from "stream";
 import { components } from "../api";
-import { getPublisherConfig, isPublisherConfigFrozen } from "../config";
 import {
-   API_PREFIX,
-   CONNECTIONS_MANIFEST_NAME,
-   PUBLISHER_CONFIG_NAME,
-   publisherPath,
-} from "../constants";
+   getProcessedPublisherConfig,
+   isPublisherConfigFrozen,
+   ProcessedPublisherConfig,
+   ProcessedProject,
+} from "../config";
+import { API_PREFIX, PUBLISHER_CONFIG_NAME, publisherPath } from "../constants";
 import {
    FrozenConfigError,
    PackageNotFoundError,
@@ -57,6 +57,7 @@ export class ProjectStore {
                   {
                      name: project.name,
                      resource: `${API_PREFIX}/projects/${project.name}`,
+                     connections: project.connections,
                   },
                   true,
                );
@@ -154,6 +155,7 @@ export class ProjectStore {
          project = await this.addProject({
             name: projectName,
             resource: `${API_PREFIX}/projects/${projectName}`,
+            connections: projectConfig?.connections || [],
          });
       }
       return project;
@@ -213,6 +215,7 @@ export class ProjectStore {
          projectName,
          absoluteProjectPath,
          project.connections || [],
+         this.serverRootPath,
       );
       this.projects.set(projectName, newProject);
       return newProject;
@@ -266,29 +269,25 @@ export class ProjectStore {
       return project;
    }
 
-   public static async reloadProjectManifest(serverRootPath: string) {
+   public static async reloadProjectManifest(
+      serverRootPath: string,
+   ): Promise<ProcessedPublisherConfig> {
       try {
-         return getPublisherConfig(serverRootPath);
+         return getProcessedPublisherConfig(serverRootPath);
       } catch (error) {
          if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
             logger.error(
                `Error reading ${PUBLISHER_CONFIG_NAME}. Generating from directory`,
                { error },
             );
-            return { projects: [] };
+            return { frozenConfig: false, projects: [] };
          } else {
             // If publisher.config.json is missing, generate the manifest from directories
             try {
                const entries = await fs.promises.readdir(serverRootPath, {
                   withFileTypes: true,
                });
-               const projects: {
-                  name: string;
-                  packages: {
-                     name: string;
-                     location: string;
-                  }[];
-               }[] = [];
+               const projects: ProcessedProject[] = [];
                for (const entry of entries) {
                   if (entry.isDirectory()) {
                      projects.push({
@@ -296,18 +295,19 @@ export class ProjectStore {
                         packages: [
                            {
                               name: entry.name,
-                              location: entry.name,
+                              location: `./${entry.name}` as const,
                            },
                         ],
+                        connections: [],
                      });
                   }
                }
-               return { projects };
+               return { frozenConfig: false, projects };
             } catch (lsError) {
                logger.error(`Error listing directories in ${serverRootPath}`, {
                   error: lsError,
                });
-               return { projects: [] };
+               return { frozenConfig: false, projects: [] };
             }
          }
       }
@@ -429,52 +429,27 @@ export class ProjectStore {
                   );
                }
             }
-
-            const connectionsFileInDownload = path.join(
-               tempDownloadPath,
-               CONNECTIONS_MANIFEST_NAME,
-            );
-            const connectionsFileInServerRoot = path.join(
-               this.serverRootPath,
-               CONNECTIONS_MANIFEST_NAME,
-            );
-            const connectionsFileInProject = path.join(
-               absoluteTargetPath,
-               CONNECTIONS_MANIFEST_NAME,
+         } catch (error) {
+            logger.error(`Failed to download or mount location "${location}"`, {
+               error,
+            });
+            throw new PackageNotFoundError(
+               `Failed to download or mount location: ${location}`,
             );
-
-            const sources = [
-               {
-                  path: connectionsFileInDownload,
-                  description: "projectLocation",
-               },
-               { path: connectionsFileInServerRoot, description: "serverRoot" },
-            ];
-
-            for (const source of sources) {
-               try {
-                  await fs.promises.access(source.path, fs.constants.F_OK);
-                  await fs.promises.cp(source.path, connectionsFileInProject);
-                  logger.info(
-                     `Copied ${CONNECTIONS_MANIFEST_NAME} from ${source.description} to project directory`,
-                  );
-                  break;
-               } catch (_error) {
-                  if (source.description === "projectLocation") {
-                     logger.info(
-                        `No ${CONNECTIONS_MANIFEST_NAME} found in downloaded location, checking in server root for ${CONNECTIONS_MANIFEST_NAME}`,
-                     );
-                  } else {
-                     logger.info(`No ${CONNECTIONS_MANIFEST_NAME} found`);
-                  }
-               }
-            }
-         } finally {
+         }
+         try {
             // Clean up temporary download directory
             await fs.promises.rm(tempDownloadPath, {
                recursive: true,
                force: true,
             });
+         } catch (error) {
+            logger.warn(
+               `Failed to clean up temporary download directory "${tempDownloadPath}"`,
+               {
+                  error,
+               },
+            );
          }
       }
 
@@ -716,23 +691,80 @@ export class ProjectStore {
    }
 
    async downloadGitHubDirectory(githubUrl: string, absoluteDirPath: string) {
-      await fs.promises.rm(absoluteDirPath, { recursive: true, force: true });
+      // First we'll clone the repo without the additional path
+      // E.g. we're removing `/tree/main/imdb` from https://github.com/credibledata/malloy-samples/tree/main/imdb
+      const githubRepoRegex =
+         /github\.com\/(?<owner>[^/]+)\/(?<repoName>[^/]+)(?<packagePath>\/[^/]+)*/;
+      const match = githubUrl.match(githubRepoRegex);
+      if (!match) {
+         throw new Error(`Invalid GitHub URL: ${githubUrl}`);
+      }
+      const { owner, repoName, packagePath } = match.groups!;
+      const cleanPackagePath = packagePath.replace("/tree/main", "");
+      // We'll make sure whatever was in absoluteDirPath is removed,
+      // so we have a nice a clean directory where we can clone the repo
+      await fs.promises.rm(absoluteDirPath, {
+         recursive: true,
+         force: true,
+      });
       await fs.promises.mkdir(absoluteDirPath, { recursive: true });
-
+      const repoUrl = `https://github.com/${owner}/${repoName}`;
+      // We'll clone the repo into absoluteDirPath
       await new Promise<void>((resolve, reject) => {
-         simpleGit().clone(githubUrl, absoluteDirPath, {}, (err) => {
+         simpleGit().clone(repoUrl, absoluteDirPath, {}, (err) => {
             if (err) {
                console.error(err);
-               logger.error(
-                  `Failed to clone GitHub repository "${githubUrl}"`,
-                  {
-                     error: err,
-                  },
-               );
+               logger.error(`Failed to clone GitHub repository "${repoUrl}"`, {
+                  error: err,
+               });
                reject(err);
             }
             resolve();
          });
       });
+      // After cloning, we'll replace all contents of absoluteDirPath with the contents of absoluteDirPath/cleanPackagePath
+      // E.g. we're moving /var/publisher/asd123/imdb/publisher.json into /var/publisher/asd123/publisher.json
+
+      // Remove all contents of absoluteDirPath (/var/publisher/asd123)
+      // except for the cleanPackagePath directory (/var/publisher/asd123/imdb)
+      const packageFullPath = path.join(absoluteDirPath, cleanPackagePath);
+
+      // Check if the cleanPackagePath (/var/publisher/asd123/imdb) exists
+      const packageExists = await fs.promises
+         .access(packageFullPath)
+         .then(() => true)
+         .catch(() => false);
+
+      if (!packageExists) {
+         throw new Error(
+            `Package path "${cleanPackagePath}" does not exist in the cloned repository.`,
+         );
+      }
+
+      // Remove everything in absoluteDirPath (/var/publisher/asd123)
+      const dirContents = await fs.promises.readdir(absoluteDirPath);
+      for (const entry of dirContents) {
+         // Don't remove the cleanPackagePath directory itself (/var/publisher/asd123/imdb)
+         if (entry !== cleanPackagePath.replace(/^\/+/, "").split("/")[0]) {
+            await fs.promises.rm(path.join(absoluteDirPath, entry), {
+               recursive: true,
+               force: true,
+            });
+         }
+      }
+
+      // Now, move the contents of packageFullPath (/var/publisher/asd123/imdb) up to absoluteDirPath (/var/publisher/asd123)
+      const packageContents = await fs.promises.readdir(packageFullPath);
+      for (const entry of packageContents) {
+         await fs.promises.rename(
+            path.join(packageFullPath, entry),
+            path.join(absoluteDirPath, entry),
+         );
+      }
+
+      // Remove the now-empty cleanPackagePath directory (/var/publisher/asd123/imdb)
+      await fs.promises.rm(packageFullPath, { recursive: true, force: true });
+
+      // https://github.com/credibledata/malloy-samples/imdb/publisher.json -> ${absoluteDirPath}/publisher.json
    }
 }