@malloy-publisher/server 0.0.106 → 0.0.108

package/eslint.config.mjs

@@ -1,11 +1,11 @@
 import { fixupConfigRules, fixupPluginRules } from "@eslint/compat";
+import { FlatCompat } from "@eslint/eslintrc";
+import js from "@eslint/js";
 import typescriptEslint from "@typescript-eslint/eslint-plugin";
-import globals from "globals";
 import tsParser from "@typescript-eslint/parser";
+import globals from "globals";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
-import js from "@eslint/js";
-import { FlatCompat } from "@eslint/eslintrc";
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -16,6 +16,19 @@ const compat = new FlatCompat({
 });
 
 export default [
+   {
+      ignores: [
+         "**/node_modules/**",
+         "**/dist/**",
+         "**/build/**",
+         "**/.DS_Store",
+         "**/__pycache__/**",
+         "**/*.py[cod]",
+         "**/*.egg-info/**",
+         "**/.env",
+         "**/*.log",
+      ],
+   },
    ...fixupConfigRules(
       compat.extends(
          "eslint:recommended",

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@malloy-publisher/server",
   "description": "Malloy Publisher Server",
-  "version": "0.0.106",
+  "version": "0.0.108",
   "main": "dist/server.js",
   "bin": {
     "malloy-publisher": "dist/server.js"
@@ -23,15 +23,15 @@
   },
   "dependencies": {
     "@google-cloud/storage": "^7.16.0",
-    "@malloydata/db-bigquery": "^0.0.307",
-    "@malloydata/db-duckdb": "^0.0.307",
-    "@malloydata/db-mysql": "^0.0.307",
-    "@malloydata/db-postgres": "^0.0.307",
-    "@malloydata/db-snowflake": "^0.0.307",
-    "@malloydata/db-trino": "^0.0.307",
-    "@malloydata/malloy": "^0.0.307",
-    "@malloydata/malloy-sql": "^0.0.307",
-    "@malloydata/render": "^0.0.307",
+    "@malloydata/db-bigquery": "^0.0.310",
+    "@malloydata/db-duckdb": "^0.0.310",
+    "@malloydata/db-mysql": "^0.0.310",
+    "@malloydata/db-postgres": "^0.0.310",
+    "@malloydata/db-snowflake": "^0.0.310",
+    "@malloydata/db-trino": "^0.0.310",
+    "@malloydata/malloy": "^0.0.310",
+    "@malloydata/malloy-sql": "^0.0.310",
+    "@malloydata/render": "^0.0.310",
     "@modelcontextprotocol/sdk": "^1.13.2",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/auto-instrumentations-node": "^0.57.0",

package/publisher.config.json

@@ -1,26 +1,33 @@
 {
-    "frozenConfig": false,
-    "projects": [
-      {
-        "name": "malloy-samples",
-        "packages": [
-            {
-            "name": "ecommerce",
-            "location": "https://github.com/credibledata/malloy-samples"
-          },
-          {
-            "name": "imdb",
-            "location": "https://github.com/credibledata/malloy-samples"
-          },
-          {
-            "name": "faa",
-            "location": "https://github.com/credibledata/malloy-samples"
-          },
-          {
-            "name": "bigquery-hackernews",
-            "location": "https://github.com/credibledata/malloy-samples"
-          }
-        ]
-      }
-    ]
-  }
+  "frozenConfig": false,
+  "projects": [
+    {
+      "name": "malloy-samples",
+      "packages": [
+        {
+          "name": "ecommerce",
+          "location": "https://github.com/credibledata/malloy-samples/tree/main/ecommerce"
+        },
+        {
+          "name": "imdb",
+          "location": "https://github.com/credibledata/malloy-samples/tree/main/imdb"
+        },
+        {
+          "name": "faa",
+          "location": "https://github.com/credibledata/malloy-samples/tree/main/faa"
+        },
+        {
+          "name": "bigquery-hackernews",
+          "location": "https://github.com/credibledata/malloy-samples/tree/main/bigquery-hackernews"
+        }
+      ],
+      "connections": [
+        {
+          "name": "bigquery",
+          "type": "bigquery",
+          "bigqueryConnection": {}
+        }
+      ]
+    }
+  ]
+}

package/src/config.ts

@@ -1,18 +1,26 @@
 import fs from "fs";
 import path from "path";
-import { PUBLISHER_CONFIG_NAME } from "./constants";
+import { PUBLISHER_CONFIG_NAME, API_PREFIX } from "./constants";
+import { components } from "./api";
 
 type FilesystemPath = `./${string}` | `../${string}` | `/${string}`;
 type GcsPath = `gs://${string}`;
+type ApiConnection = components["schemas"]["Connection"];
 
 export type Package = {
    name: string;
    location: FilesystemPath | GcsPath;
 };
 
+export type Connection = {
+   name: string;
+   type: string;
+};
+
 export type Project = {
    name: string;
    packages: Package[];
+   connections?: Connection[];
 };
 
 export type PublisherConfig = {
@@ -20,6 +28,17 @@ export type PublisherConfig = {
    projects: Project[];
 };
 
+export type ProcessedProject = {
+   name: string;
+   packages: Package[];
+   connections: ApiConnection[];
+};
+
+export type ProcessedPublisherConfig = {
+   frozenConfig: boolean;
+   projects: ProcessedProject[];
+};
+
 export const getPublisherConfig = (serverRoot: string): PublisherConfig => {
    const publisherConfigPath = path.join(serverRoot, PUBLISHER_CONFIG_NAME);
    if (!fs.existsSync(publisherConfigPath)) {
@@ -47,3 +66,44 @@ export const isPublisherConfigFrozen = (serverRoot: string) => {
    const publisherConfig = getPublisherConfig(serverRoot);
    return Boolean(publisherConfig.frozenConfig);
 };
+
+export const getConnectionsFromPublisherConfig = (
+   serverRoot: string,
+   projectName: string,
+): Connection[] => {
+   const publisherConfig = getPublisherConfig(serverRoot);
+   const project = publisherConfig.projects.find((p) => p.name === projectName);
+   return project?.connections || [];
+};
+
+export const convertConnectionsToApiConnections = (
+   connections: Connection[],
+): ApiConnection[] => {
+   return connections.map((conn) => ({
+      ...conn,
+      name: conn.name,
+      type: conn.type as
+         | "postgres"
+         | "bigquery"
+         | "snowflake"
+         | "trino"
+         | "mysql",
+      resource: `${API_PREFIX}/connections/${conn.name}`,
+   }));
+};
+
+export const getProcessedPublisherConfig = (
+   serverRoot: string,
+): ProcessedPublisherConfig => {
+   const rawConfig = getPublisherConfig(serverRoot);
+   return {
+      frozenConfig: rawConfig.frozenConfig,
+      projects: rawConfig.projects.map((project) => ({
+         name: project.name,
+         packages: project.packages,
+         connections: convertConnectionsToApiConnections(
+            project.connections || [],
+         ),
+      })),
+   };
+};

package/src/constants.ts

@@ -5,7 +5,6 @@ export const API_PREFIX = "/api/v0";
 export const README_NAME = "README.md";
 export const PUBLISHER_CONFIG_NAME = "publisher.config.json";
 export const PACKAGE_MANIFEST_NAME = "publisher.json";
-export const CONNECTIONS_MANIFEST_NAME = "publisher.connections.json";
 export const MODEL_FILE_SUFFIX = ".malloy";
 export const NOTEBOOK_FILE_SUFFIX = ".malloynb";
 export const ROW_LIMIT = 1000;

package/src/controller/connection.controller.ts

@@ -46,7 +46,7 @@ export class ConnectionController {
       connectionName: string,
    ): Promise<ApiSchemaName[]> {
       const project = await this.projectStore.getProject(projectName, false);
-      const connection = project.getInternalConnection(connectionName);
+      const connection = project.getApiConnection(connectionName);
       return getSchemasForConnection(connection);
    }
 
@@ -57,7 +57,7 @@ export class ConnectionController {
       schemaName: string,
    ): Promise<string[]> {
       const project = await this.projectStore.getProject(projectName, false);
-      const connection = project.getInternalConnection(connectionName);
+      const connection = project.getApiConnection(connectionName);
       return getTablesForSchema(connection, schemaName);
    }
 

package/src/server.ts

@@ -3,7 +3,6 @@ import * as bodyParser from "body-parser";
 import cors from "cors";
 import express from "express";
 import * as http from "http";
-import { createProxyMiddleware } from "http-proxy-middleware";
 import { AddressInfo } from "net";
 import * as path from "path";
 import { ConnectionController } from "./controller/connection.controller";
@@ -81,7 +80,6 @@ const isDevelopment = process.env["NODE_ENV"] === "development";
 
 const app = express();
 app.use(loggerMiddleware);
-app.use(cors());
 
 const projectStore = new ProjectStore(SERVER_ROOT);
 const watchModeController = new WatchModeController(projectStore);
@@ -186,16 +184,6 @@ if (!isDevelopment) {
    // In development mode, proxy requests to React dev server
    // Handle API routes first
    app.use(`${API_PREFIX}`, loggerMiddleware);
-
-   // Proxy everything else to Vite
-   app.use(
-      createProxyMiddleware({
-         target: "http://localhost:5173",
-         changeOrigin: true,
-         ws: true,
-         pathFilter: (path) => !path.startsWith("/api/"),
-      }),
-   );
 }
 
 const setVersionIdError = (res: express.Response) => {
@@ -205,7 +193,12 @@ const setVersionIdError = (res: express.Response) => {
    res.status(status).json(json);
 };
 
-app.use(cors());
+app.use(
+   cors({
+      origin: "http://localhost:5173",
+      credentials: true,
+   }),
+);
 app.use(bodyParser.json());
 
 app.get(`${API_PREFIX}/status`, async (_req, res) => {
@@ -378,6 +371,9 @@ app.get(
    },
 );
 
+/**
+ * @deprecated Use /projects/:projectName/connections/:connectionName/sqlSource POST method instead
+ */
 app.get(
    `${API_PREFIX}/projects/:projectName/connections/:connectionName/sqlSource`,
    async (req, res) => {
@@ -397,6 +393,25 @@ app.get(
    },
 );
 
+app.post(
+   `${API_PREFIX}/projects/:projectName/connections/:connectionName/sqlSource`,
+   async (req, res) => {
+      try {
+         res.status(200).json(
+            await connectionController.getConnectionSqlSource(
+               req.params.projectName,
+               req.params.connectionName,
+               req.body.sqlStatement as string,
+            ),
+         );
+      } catch (error) {
+         logger.error(error);
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
 app.get(
    `${API_PREFIX}/projects/:projectName/connections/:connectionName/tableSource`,
    async (req, res) => {
@@ -417,6 +432,9 @@ app.get(
    },
 );
 
+/**
+ * @deprecated Use /projects/:projectName/connections/:connectionName/queryData POST method instead
+ */
 app.get(
    `${API_PREFIX}/projects/:projectName/connections/:connectionName/queryData`,
    async (req, res) => {
@@ -437,6 +455,29 @@ app.get(
    },
 );
 
+app.post(
+   `${API_PREFIX}/projects/:projectName/connections/:connectionName/queryData`,
+   async (req, res) => {
+      try {
+         res.status(200).json(
+            await connectionController.getConnectionQueryData(
+               req.params.projectName,
+               req.params.connectionName,
+               req.body.sqlStatement as string,
+               req.query.options as string,
+            ),
+         );
+      } catch (error) {
+         logger.error(error);
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+/**
+ * @deprecated Use /projects/:projectName/connections/:connectionName/temporaryTable POST method instead
+ */
 app.get(
    `${API_PREFIX}/projects/:projectName/connections/:connectionName/temporaryTable`,
    async (req, res) => {
@@ -456,6 +497,25 @@ app.get(
    },
 );
 
+app.post(
+   `${API_PREFIX}/projects/:projectName/connections/:connectionName/temporaryTable`,
+   async (req, res) => {
+      try {
+         res.status(200).json(
+            await connectionController.getConnectionTemporaryTable(
+               req.params.projectName,
+               req.params.connectionName,
+               req.body.sqlStatement as string,
+            ),
+         );
+      } catch (error) {
+         logger.error(error);
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
 app.get(`${API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
    if (req.query.versionId) {
       setVersionIdError(res);

package/src/service/connection.ts

@@ -11,7 +11,11 @@ import fs from "fs/promises";
 import path from "path";
 import { v4 as uuidv4 } from "uuid";
 import { components } from "../api";
-import { CONNECTIONS_MANIFEST_NAME, TEMP_DIR_PATH } from "../constants";
+import { TEMP_DIR_PATH } from "../constants";
+import {
+   convertConnectionsToApiConnections,
+   getConnectionsFromPublisherConfig,
+} from "../config";
 import { AxiosError } from "axios";
 
 type ApiConnection = components["schemas"]["Connection"];
@@ -29,32 +33,38 @@ export type InternalConnection = ApiConnection & {
 };
 
 export async function readConnectionConfig(
-   basePath: string,
-): Promise<InternalConnection[]> {
-   const fullPath = path.join(basePath, CONNECTIONS_MANIFEST_NAME);
-
-   try {
-      await fs.stat(fullPath);
-   } catch {
-      // If there's no connection manifest, it's no problem.  Just return an
-      // empty array.
-      return new Array<InternalConnection>();
+   _basePath: string,
+   projectName?: string,
+   serverRootPath?: string,
+): Promise<ApiConnection[]> {
+   // If no project name is provided, return empty array for backward compatibility
+   if (!projectName || !serverRootPath) {
+      return new Array<ApiConnection>();
    }
 
-   const connectionFileContents = await fs.readFile(fullPath);
-   // TODO: Validate connection manifest.  Define manifest type in public API.
-   return JSON.parse(connectionFileContents.toString()) as ApiConnection[];
+   // Get connections from publisher config
+   const connections = getConnectionsFromPublisherConfig(
+      serverRootPath,
+      projectName,
+   );
+   return convertConnectionsToApiConnections(connections);
 }
 
 export async function createConnections(
    basePath: string,
    defaultConnections: ApiConnection[] = [],
+   projectName?: string,
+   serverRootPath?: string,
 ): Promise<{
    malloyConnections: Map<string, BaseConnection>;
    apiConnections: InternalConnection[];
 }> {
    const connectionMap = new Map<string, BaseConnection>();
-   const connectionConfig = await readConnectionConfig(basePath);
+   const connectionConfig = await readConnectionConfig(
+      basePath,
+      projectName,
+      serverRootPath,
+   );
 
    const allConnections = [...defaultConnections, ...connectionConfig];
 

package/src/service/package.spec.ts

@@ -4,7 +4,7 @@ import { join } from "path";
 import sinon from "sinon";
 import { PackageNotFoundError } from "../errors";
 import { readConnectionConfig } from "./connection";
-import { ApiConnection, Model } from "./model";
+import { Model } from "./model";
 import { Package } from "./package";
 import { Scheduler } from "./scheduler";
 import { Stats } from "fs";
@@ -12,11 +12,6 @@ import { Stats } from "fs";
 // Minimal partial types for mocking
 type PartialScheduler = Pick<Scheduler, "list">;
 
-const connectionMocks: ApiConnection[] = [
-   { name: "conn1", type: "postgres", postgresConnection: {} },
-   { name: "conn2", type: "bigquery", bigqueryConnection: {} },
-];
-
 describe("service/package", () => {
    const testPackageDirectory = "testPackage";
 
@@ -30,11 +25,6 @@ describe("service/package", () => {
          join(testPackageDirectory, "database.csv"),
          parquetBuffer,
       );
-      const content = JSON.stringify(connectionMocks);
-      await fs.writeFile(
-         join(testPackageDirectory, "publisher.connections.json"),
-         content,
-      );
       const publisherContent = JSON.stringify({ description: "Test package" });
       await fs.writeFile(
          join(testPackageDirectory, "publisher.json"),
@@ -85,6 +75,7 @@ describe("service/package", () => {
                   "testPackage",
                   testPackageDirectory,
                   new Map(),
+                  "/server/root",
                ),
             ).rejects.toThrowError(
                new PackageNotFoundError(
@@ -124,6 +115,7 @@ describe("service/package", () => {
                   "testPackage",
                   testPackageDirectory,
                   new Map(),
+                  "/server/root",
                );
 
                expect(packageInstance).toBeInstanceOf(Package);
@@ -232,23 +224,76 @@ describe("service/package", () => {
       });
 
       describe("readConnectionConfig", () => {
-         it("should return an empty array if the connection manifest does not exist", async () => {
-            await fs.rm(
-               join(testPackageDirectory, "publisher.connections.json"),
-            );
-
-            sinon.stub(fs, "stat").rejects(new Error("File not found"));
-
+         it("should return an empty array if no project name or server root path is provided", async () => {
             const config = await readConnectionConfig(testPackageDirectory);
             expect(Array.isArray(config)).toBe(true);
             expect(config).toHaveLength(0);
          });
 
-         it("should return the parsed connection config if it exists", async () => {
-            sinon.stub(fs, "stat").resolves();
-            const config = await readConnectionConfig(testPackageDirectory);
+         it("should return an empty array if project name or server root path is missing", async () => {
+            const config1 = await readConnectionConfig(
+               testPackageDirectory,
+               "testProject",
+            );
+            expect(Array.isArray(config1)).toBe(true);
+            expect(config1).toHaveLength(0);
+
+            const config2 = await readConnectionConfig(
+               testPackageDirectory,
+               undefined,
+               "/server/root",
+            );
+            expect(Array.isArray(config2)).toBe(true);
+            expect(config2).toHaveLength(0);
+         });
+
+         it("should return connections from publisher config when project name and server root are provided", async () => {
+            const tempServerRoot = "/tmp/test-server-root";
+            const tempConfigPath = join(
+               tempServerRoot,
+               "publisher.config.json",
+            );
+
+            // Create temp directory and config file
+            await fs.mkdir(tempServerRoot, { recursive: true });
+            await fs.writeFile(
+               tempConfigPath,
+               JSON.stringify({
+                  frozenConfig: false,
+                  projects: [
+                     {
+                        name: "testProject",
+                        packages: [],
+                        connections: [
+                           { name: "test-conn", type: "postgres" },
+                           { name: "test-conn2", type: "bigquery" },
+                        ],
+                     },
+                  ],
+               }),
+            );
+
+            const config = await readConnectionConfig(
+               testPackageDirectory,
+               "testProject",
+               tempServerRoot,
+            );
+
+            expect(Array.isArray(config)).toBe(true);
+            expect(config).toHaveLength(2);
+            expect(config[0]).toMatchObject({
+               name: "test-conn",
+               type: "postgres",
+               resource: "/api/v0/connections/test-conn",
+            });
+            expect(config[1]).toMatchObject({
+               name: "test-conn2",
+               type: "bigquery",
+               resource: "/api/v0/connections/test-conn2",
+            });
 
-            expect(config).toEqual(connectionMocks);
+            // Clean up
+            await fs.rm(tempServerRoot, { recursive: true, force: true });
          });
       });
    });

package/src/service/package.ts

@@ -72,6 +72,7 @@ export class Package {
       packageName: string,
       packagePath: string,
       projectConnections: Map<string, Connection>,
+      serverRootPath: string,
    ): Promise<Package> {
       const startTime = performance.now();
       await Package.validatePackageManifestExistsOrThrowError(packagePath);
@@ -107,7 +108,12 @@ export class Package {
          });
          // Package connections override project connections.
          const { malloyConnections: packageConnections } =
-            await createConnections(packagePath);
+            await createConnections(
+               packagePath,
+               [],
+               projectName,
+               serverRootPath,
+            );
          const connectionsTime = performance.now();
          logger.info("Package connections created", {
             packageName,
@@ -297,6 +303,7 @@ export class Package {
       try {
          await fs.stat(packageConfigPath);
       } catch {
+         logger.error(`Can't find ${packageConfigPath}`);
          throw new PackageNotFoundError(
             `Package manifest for ${packagePath} does not exist.`,
          );