@malloy-publisher/server 0.0.104 → 0.0.108

package/src/service/package.spec.ts

@@ -4,7 +4,7 @@ import { join } from "path";
 import sinon from "sinon";
 import { PackageNotFoundError } from "../errors";
 import { readConnectionConfig } from "./connection";
-import { ApiConnection, Model } from "./model";
+import { Model } from "./model";
 import { Package } from "./package";
 import { Scheduler } from "./scheduler";
 import { Stats } from "fs";
@@ -12,11 +12,6 @@ import { Stats } from "fs";
 // Minimal partial types for mocking
 type PartialScheduler = Pick<Scheduler, "list">;
 
-const connectionMocks: ApiConnection[] = [
-   { name: "conn1", type: "postgres", postgresConnection: {} },
-   { name: "conn2", type: "bigquery", bigqueryConnection: {} },
-];
-
 describe("service/package", () => {
    const testPackageDirectory = "testPackage";
 
@@ -30,11 +25,6 @@ describe("service/package", () => {
          join(testPackageDirectory, "database.csv"),
          parquetBuffer,
       );
-      const content = JSON.stringify(connectionMocks);
-      await fs.writeFile(
-         join(testPackageDirectory, "publisher.connections.json"),
-         content,
-      );
       const publisherContent = JSON.stringify({ description: "Test package" });
       await fs.writeFile(
          join(testPackageDirectory, "publisher.json"),
@@ -85,6 +75,7 @@ describe("service/package", () => {
                   "testPackage",
                   testPackageDirectory,
                   new Map(),
+                  "/server/root",
                ),
             ).rejects.toThrowError(
                new PackageNotFoundError(
@@ -124,6 +115,7 @@ describe("service/package", () => {
                   "testPackage",
                   testPackageDirectory,
                   new Map(),
+                  "/server/root",
                );
 
                expect(packageInstance).toBeInstanceOf(Package);
@@ -232,23 +224,76 @@ describe("service/package", () => {
       });
 
       describe("readConnectionConfig", () => {
-         it("should return an empty array if the connection manifest does not exist", async () => {
-            await fs.rm(
-               join(testPackageDirectory, "publisher.connections.json"),
-            );
-
-            sinon.stub(fs, "stat").rejects(new Error("File not found"));
-
+         it("should return an empty array if no project name or server root path is provided", async () => {
             const config = await readConnectionConfig(testPackageDirectory);
             expect(Array.isArray(config)).toBe(true);
             expect(config).toHaveLength(0);
          });
 
-         it("should return the parsed connection config if it exists", async () => {
-            sinon.stub(fs, "stat").resolves();
-            const config = await readConnectionConfig(testPackageDirectory);
+         it("should return an empty array if project name or server root path is missing", async () => {
+            const config1 = await readConnectionConfig(
+               testPackageDirectory,
+               "testProject",
+            );
+            expect(Array.isArray(config1)).toBe(true);
+            expect(config1).toHaveLength(0);
+
+            const config2 = await readConnectionConfig(
+               testPackageDirectory,
+               undefined,
+               "/server/root",
+            );
+            expect(Array.isArray(config2)).toBe(true);
+            expect(config2).toHaveLength(0);
+         });
+
+         it("should return connections from publisher config when project name and server root are provided", async () => {
+            const tempServerRoot = "/tmp/test-server-root";
+            const tempConfigPath = join(
+               tempServerRoot,
+               "publisher.config.json",
+            );
+
+            // Create temp directory and config file
+            await fs.mkdir(tempServerRoot, { recursive: true });
+            await fs.writeFile(
+               tempConfigPath,
+               JSON.stringify({
+                  frozenConfig: false,
+                  projects: [
+                     {
+                        name: "testProject",
+                        packages: [],
+                        connections: [
+                           { name: "test-conn", type: "postgres" },
+                           { name: "test-conn2", type: "bigquery" },
+                        ],
+                     },
+                  ],
+               }),
+            );
+
+            const config = await readConnectionConfig(
+               testPackageDirectory,
+               "testProject",
+               tempServerRoot,
+            );
+
+            expect(Array.isArray(config)).toBe(true);
+            expect(config).toHaveLength(2);
+            expect(config[0]).toMatchObject({
+               name: "test-conn",
+               type: "postgres",
+               resource: "/api/v0/connections/test-conn",
+            });
+            expect(config[1]).toMatchObject({
+               name: "test-conn2",
+               type: "bigquery",
+               resource: "/api/v0/connections/test-conn2",
+            });
 
-            expect(config).toEqual(connectionMocks);
+            // Clean up
+            await fs.rm(tempServerRoot, { recursive: true, force: true });
          });
       });
    });

package/src/service/package.ts

@@ -72,6 +72,7 @@ export class Package {
       packageName: string,
       packagePath: string,
       projectConnections: Map<string, Connection>,
+      serverRootPath: string,
    ): Promise<Package> {
       const startTime = performance.now();
       await Package.validatePackageManifestExistsOrThrowError(packagePath);
@@ -107,7 +108,12 @@ export class Package {
          });
          // Package connections override project connections.
          const { malloyConnections: packageConnections } =
-            await createConnections(packagePath);
+            await createConnections(
+               packagePath,
+               [],
+               projectName,
+               serverRootPath,
+            );
          const connectionsTime = performance.now();
          logger.info("Package connections created", {
             packageName,
@@ -297,6 +303,7 @@ export class Package {
       try {
          await fs.stat(packageConfigPath);
       } catch {
+         logger.error(`Can't find ${packageConfigPath}`);
          throw new PackageNotFoundError(
             `Package manifest for ${packagePath} does not exist.`,
          );
@@ -365,8 +372,13 @@ export class Package {
          urlReader: new EmptyURLReader(),
          connections: [new DuckDBConnection("duckdb")],
       });
+      // Escape backslashes for Windows paths in the Malloy model string
+      const escapedPath =
+         process.platform === "win32"
+            ? fullPath.replace(/\\/g, "\\\\")
+            : fullPath;
       const model = runtime.loadModel(
-         `source: temp is duckdb.table('${fullPath}')`,
+         `source: temp is duckdb.table('${escapedPath}')`,
       );
       const modelDef = await model.getModel();
       const fields = (modelDef._modelDef.contents["temp"] as SourceDef).fields;

package/src/service/project.ts

@@ -35,23 +35,22 @@ export class Project {
    private packageStatuses: Map<string, PackageInfo> = new Map();
    private malloyConnections: Map<string, BaseConnection>;
    private apiConnections: ApiConnection[];
-   private internalConnections: InternalConnection[];
    private projectPath: string;
    private projectName: string;
+   private serverRootPath: string;
    public metadata: ApiProject;
 
    constructor(
       projectName: string,
       projectPath: string,
       malloyConnections: Map<string, BaseConnection>,
-      internalConnections: InternalConnection[],
       apiConnections: InternalConnection[],
+      serverRootPath: string,
    ) {
       this.projectName = projectName;
       this.projectPath = projectPath;
+      this.serverRootPath = serverRootPath;
       this.malloyConnections = malloyConnections;
-      // InternalConnections have full connection details for doing schema inspection
-      this.internalConnections = internalConnections;
       this.apiConnections = apiConnections;
       this.metadata = {
          resource: `${API_PREFIX}/projects/${this.projectName}`,
@@ -82,7 +81,6 @@ export class Project {
          // Update the project's connection maps
          this.malloyConnections = malloyConnections;
          this.apiConnections = apiConnections;
-         this.internalConnections = apiConnections;
 
          logger.info(
             `Successfully updated connections for project ${this.projectName}`,
@@ -101,6 +99,7 @@ export class Project {
       projectName: string,
       projectPath: string,
       defaultConnections: ApiConnection[],
+      serverRootPath?: string,
    ): Promise<Project> {
       if (!(await fs.promises.stat(projectPath)).isDirectory()) {
          throw new ProjectNotFoundError(
@@ -112,7 +111,12 @@ export class Project {
       let apiConnections: InternalConnection[] = [];
 
       logger.info(`Creating project with connection configuration`);
-      const result = await createConnections(projectPath, defaultConnections);
+      const result = await createConnections(
+         projectPath,
+         defaultConnections,
+         projectName,
+         serverRootPath,
+      );
       malloyConnections = result.malloyConnections;
       apiConnections = result.apiConnections;
 
@@ -129,17 +133,7 @@ export class Project {
          projectPath,
          malloyConnections,
          apiConnections,
-         apiConnections.map((internalConnection) => {
-            // Create a new ApiConnection object from each InternalConnection
-            // by excluding the internal connection details
-            // We don't want to send passwords and connection strings to the client
-            return {
-               name: internalConnection.name,
-               type: internalConnection.type,
-               attributes: internalConnection.attributes,
-               resource: internalConnection.resource,
-            };
-         }),
+         serverRootPath || "",
       );
    }
 
@@ -176,20 +170,6 @@ export class Project {
       return connection;
    }
 
-   // Returns a connection with full connection details for doing schema inspection
-   // Don't send this to the client as it contains sensitive information
-   public getInternalConnection(connectionName: string): InternalConnection {
-      const connection = this.internalConnections.find(
-         (connection) => connection.name === connectionName,
-      );
-      if (!connection) {
-         throw new ConnectionNotFoundError(
-            `Connection ${connectionName} not found`,
-         );
-      }
-      return connection;
-   }
-
    public getMalloyConnection(connectionName: string): BaseConnection {
       const connection = this.malloyConnections.get(connectionName);
       if (!connection) {
@@ -290,6 +270,7 @@ export class Project {
                packageName,
                path.join(this.projectPath, packageName),
                this.malloyConnections,
+               this.serverRootPath,
             );
             this.packages.set(packageName, _package);
 
@@ -334,6 +315,7 @@ export class Project {
                packageName,
                packagePath,
                this.malloyConnections,
+               this.serverRootPath,
             ),
          );
       } catch (error) {

package/src/service/project_store.spec.ts

@@ -15,19 +15,6 @@ const serverRootPath = path.join(
    "pathways-worker-publisher-project-store-test",
 );
 const projectName = "organizationName-projectName";
-const testConnections: Connection[] = [
-   {
-      name: "testConnection",
-      type: "postgres",
-      postgresConnection: {
-         host: "host",
-         port: 1234,
-         databaseName: "databaseName",
-         userName: "userName",
-         password: "password",
-      },
-   },
-];
 
 let sandbox: sinon.SinonSandbox;
 
@@ -72,13 +59,6 @@ describe("ProjectStore Service", () => {
       const projectPath = path.join(serverRootPath, projectName);
       mkdirSync(projectPath, { recursive: true });
 
-      // Create connections file
-      const connectionsPath = path.join(
-         projectPath,
-         "publisher.connections.json",
-      );
-      writeFileSync(connectionsPath, JSON.stringify(testConnections));
-
       // Create publisher config
       const publisherConfigPath = path.join(
          serverRootPath,
@@ -87,6 +67,7 @@ describe("ProjectStore Service", () => {
       writeFileSync(
          publisherConfigPath,
          JSON.stringify({
+            frozenConfig: false,
             projects: [
                {
                   name: projectName,
@@ -96,6 +77,12 @@ describe("ProjectStore Service", () => {
                         location: projectPath,
                      },
                   ],
+                  connections: [
+                     {
+                        name: "testConnection",
+                        type: "postgres",
+                     },
+                  ],
                },
             ],
          }),
@@ -117,16 +104,6 @@ describe("ProjectStore Service", () => {
       mkdirSync(projectPath1, { recursive: true });
       mkdirSync(projectPath2, { recursive: true });
 
-      // Create connections files
-      writeFileSync(
-         path.join(projectPath1, "publisher.connections.json"),
-         JSON.stringify(testConnections),
-      );
-      writeFileSync(
-         path.join(projectPath2, "publisher.connections.json"),
-         JSON.stringify(testConnections),
-      );
-
       // Create publisher config
       const publisherConfigPath = path.join(
          serverRootPath,
@@ -135,6 +112,7 @@ describe("ProjectStore Service", () => {
       writeFileSync(
          publisherConfigPath,
          JSON.stringify({
+            frozenConfig: false,
             projects: [
                {
                   name: projectName1,
@@ -144,6 +122,12 @@ describe("ProjectStore Service", () => {
                         location: projectPath1,
                      },
                   ],
+                  connections: [
+                     {
+                        name: "testConnection",
+                        type: "postgres",
+                     },
+                  ],
                },
                {
                   name: projectName2,
@@ -153,6 +137,13 @@ describe("ProjectStore Service", () => {
                         location: projectPath2,
                      },
                   ],
+                  connections: [
+                     {
+                        name: "testConnection2",
+                        type: "bigquery",
+                        bigqueryConnection: {},
+                     },
+                  ],
                },
             ],
          }),
@@ -174,14 +165,6 @@ describe("ProjectStore Service", () => {
       // Create a project directory
       const projectPath = path.join(serverRootPath, projectName);
       mkdirSync(projectPath, { recursive: true });
-
-      // Create connections file
-      const connectionsPath = path.join(
-         projectPath,
-         "publisher.connections.json",
-      );
-      writeFileSync(connectionsPath, JSON.stringify(testConnections));
-
       // Create publisher config
       const publisherConfigPath = path.join(
          serverRootPath,
@@ -221,13 +204,6 @@ describe("ProjectStore Service", () => {
       const projectPath = path.join(serverRootPath, projectName);
       mkdirSync(projectPath, { recursive: true });
 
-      // Create connections file
-      const connectionsPath = path.join(
-         projectPath,
-         "publisher.connections.json",
-      );
-      writeFileSync(connectionsPath, JSON.stringify(testConnections));
-
       // Create publisher config
       const publisherConfigPath = path.join(
          serverRootPath,
@@ -310,14 +286,6 @@ describe("ProjectStore Service", () => {
       const projectPath = path.join(serverRootPath, projectName);
       mkdirSync(projectPath, { recursive: true });
 
-      // Create connections file
-      const connectionsPath = path.join(
-         projectPath,
-         "publisher.connections.json",
-      );
-      writeFileSync(connectionsPath, JSON.stringify(testConnections));
-
-      // Create publisher config
       const publisherConfigPath = path.join(
          serverRootPath,
          "publisher.config.json",
@@ -325,6 +293,7 @@ describe("ProjectStore Service", () => {
       writeFileSync(
          publisherConfigPath,
          JSON.stringify({
+            frozenConfig: false,
             projects: [
                {
                   name: projectName,
@@ -334,11 +303,19 @@ describe("ProjectStore Service", () => {
                         location: projectPath,
                      },
                   ],
+                  connections: [
+                     {
+                        name: "testConnection",
+                        type: "postgres",
+                     },
+                  ],
                },
             ],
          }),
       );
 
+      await projectStore.finishedInitialization;
+
       // Test concurrent access to the same project
       const promises = Array.from({ length: 5 }, () =>
          projectStore.getProject(projectName),