@malloy-publisher/server 0.0.104 → 0.0.108

package/eslint.config.mjs

@@ -1,11 +1,11 @@
 import { fixupConfigRules, fixupPluginRules } from "@eslint/compat";
+import { FlatCompat } from "@eslint/eslintrc";
+import js from "@eslint/js";
 import typescriptEslint from "@typescript-eslint/eslint-plugin";
-import globals from "globals";
 import tsParser from "@typescript-eslint/parser";
+import globals from "globals";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
-import js from "@eslint/js";
-import { FlatCompat } from "@eslint/eslintrc";
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -16,6 +16,19 @@ const compat = new FlatCompat({
 });
 
 export default [
+   {
+      ignores: [
+         "**/node_modules/**",
+         "**/dist/**",
+         "**/build/**",
+         "**/.DS_Store",
+         "**/__pycache__/**",
+         "**/*.py[cod]",
+         "**/*.egg-info/**",
+         "**/.env",
+         "**/*.log",
+      ],
+   },
    ...fixupConfigRules(
       compat.extends(
          "eslint:recommended",

package/package.json

@@ -1,83 +1,83 @@
 {
-   "name": "@malloy-publisher/server",
-   "description": "Malloy Publisher Server",
-   "version": "0.0.104",
-   "main": "dist/server.js",
-   "bin": {
-      "malloy-publisher": "dist/server.js"
-   },
-   "publishConfig": {
-      "access": "public"
-   },
-   "scripts": {
-      "test": "bun test --timeout 100000",
-      "build": "bun generate-api-types && bun build:app && NODE_ENV=production bun run build.ts",
-      "build:server-only": "bun generate-api-types && NODE_ENV=production bun run build.ts",
-      "start": "NODE_ENV=production node ./dist/server.js",
-      "start:dev": "NODE_ENV=development bun --watch src/server.ts",
-      "start:instrumented": "node --require ./dist/instrumentation.js ./dist/server.js",
-      "lint": "bunx eslint \"./src/**/*.ts\"",
-      "format": "bunx prettier --write '**/*.{ts,tsx}'",
-      "build:app": "cd ../app && NODE_ENV=production bunx vite build",
-      "generate-api-types": "bunx openapi-typescript ../../api-doc.yaml --output src/api.ts"
-   },
-   "dependencies": {
-      "@google-cloud/storage": "^7.16.0",
-      "@malloydata/db-bigquery": "^0.0.307",
-      "@malloydata/db-duckdb": "^0.0.307",
-      "@malloydata/db-mysql": "^0.0.307",
-      "@malloydata/db-postgres": "^0.0.307",
-      "@malloydata/db-snowflake": "^0.0.307",
-      "@malloydata/db-trino": "^0.0.307",
-      "@malloydata/malloy": "^0.0.307",
-      "@malloydata/malloy-sql": "^0.0.307",
-      "@malloydata/render": "^0.0.307",
-      "@modelcontextprotocol/sdk": "^1.13.2",
-      "@opentelemetry/api": "^1.9.0",
-      "@opentelemetry/auto-instrumentations-node": "^0.57.0",
-      "@opentelemetry/sdk-metrics": "^2.0.0",
-      "@opentelemetry/sdk-node": "^0.200.0",
-      "@opentelemetry/sdk-trace-node": "^2.0.0",
-      "adm-zip": "^0.5.16",
-      "async-mutex": "^0.5.0",
-      "aws-sdk": "^2.1692.0",
-      "body-parser": "^1.20.2",
-      "chokidar": "^4.0.3",
-      "class-transformer": "^0.5.1",
-      "class-validator": "^0.14.1",
-      "cors": "^2.8.5",
-      "express": "^4.21.0",
-      "globals": "^15.9.0",
-      "handlebars": "^4.7.8",
-      "http-proxy-middleware": "^3.0.5",
-      "morgan": "^1.10.0",
-      "node-cron": "^3.0.3",
-      "recursive-readdir": "^2.2.3",
-      "simple-git": "^3.28.0",
-      "uuid": "^11.0.3"
-   },
-   "devDependencies": {
-      "@anthropic-ai/dxt": "^0.1.0",
-      "@eslint/compat": "^1.2.7",
-      "@eslint/eslintrc": "^3.3.1",
-      "@eslint/js": "^9.23.0",
-      "@faker-js/faker": "^9.4.0",
-      "@types/adm-zip": "^0.5.7",
-      "@types/bun": "^1.2.20",
-      "@types/cors": "^2.8.12",
-      "@types/express": "^4.17.14",
-      "@types/morgan": "^1.9.9",
-      "@types/node-cron": "^3.0.11",
-      "@types/recursive-readdir": "^2.2.4",
-      "@types/sinon": "^17.0.4",
-      "@types/supertest": "^6.0.2",
-      "@typescript-eslint/eslint-plugin": "^8.6.0",
-      "@typescript-eslint/parser": "^8.6.0",
-      "eslint-config-prettier": "^9.1.0",
-      "eslint-plugin-prettier": "^5.2.1",
-      "mock-aws-s3": "^4.0.2",
-      "nock": "^14.0.1",
-      "openapi-typescript": "^6.7.6",
-      "supertest": "^7.0.0"
-   }
+  "name": "@malloy-publisher/server",
+  "description": "Malloy Publisher Server",
+  "version": "0.0.108",
+  "main": "dist/server.js",
+  "bin": {
+    "malloy-publisher": "dist/server.js"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "test": "bun test --timeout 100000",
+    "build": "bun generate-api-types && bun build:app && NODE_ENV=production bun run build.ts",
+    "build:server-only": "bun generate-api-types && NODE_ENV=production bun run build.ts",
+    "start": "NODE_ENV=production node ./dist/server.js",
+    "start:dev": "NODE_ENV=development bun --watch src/server.ts",
+    "start:instrumented": "node --require ./dist/instrumentation.js ./dist/server.js",
+    "lint": "bunx eslint \"./src/**/*.ts\"",
+    "format": "bunx prettier --write '**/*.{ts,tsx}'",
+    "build:app": "cd ../app && NODE_ENV=production bunx vite build",
+    "generate-api-types": "bunx openapi-typescript ../../api-doc.yaml --output src/api.ts"
+  },
+  "dependencies": {
+    "@google-cloud/storage": "^7.16.0",
+    "@malloydata/db-bigquery": "^0.0.310",
+    "@malloydata/db-duckdb": "^0.0.310",
+    "@malloydata/db-mysql": "^0.0.310",
+    "@malloydata/db-postgres": "^0.0.310",
+    "@malloydata/db-snowflake": "^0.0.310",
+    "@malloydata/db-trino": "^0.0.310",
+    "@malloydata/malloy": "^0.0.310",
+    "@malloydata/malloy-sql": "^0.0.310",
+    "@malloydata/render": "^0.0.310",
+    "@modelcontextprotocol/sdk": "^1.13.2",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/auto-instrumentations-node": "^0.57.0",
+    "@opentelemetry/sdk-metrics": "^2.0.0",
+    "@opentelemetry/sdk-node": "^0.200.0",
+    "@opentelemetry/sdk-trace-node": "^2.0.0",
+    "adm-zip": "^0.5.16",
+    "async-mutex": "^0.5.0",
+    "aws-sdk": "^2.1692.0",
+    "body-parser": "^1.20.2",
+    "chokidar": "^4.0.3",
+    "class-transformer": "^0.5.1",
+    "class-validator": "^0.14.1",
+    "cors": "^2.8.5",
+    "express": "^4.21.0",
+    "globals": "^15.9.0",
+    "handlebars": "^4.7.8",
+    "http-proxy-middleware": "^3.0.5",
+    "morgan": "^1.10.0",
+    "node-cron": "^3.0.3",
+    "recursive-readdir": "^2.2.3",
+    "simple-git": "^3.28.0",
+    "uuid": "^11.0.3"
+  },
+  "devDependencies": {
+    "@anthropic-ai/dxt": "^0.1.0",
+    "@eslint/compat": "^1.2.7",
+    "@eslint/eslintrc": "^3.3.1",
+    "@eslint/js": "^9.23.0",
+    "@faker-js/faker": "^9.4.0",
+    "@types/adm-zip": "^0.5.7",
+    "@types/bun": "^1.2.20",
+    "@types/cors": "^2.8.12",
+    "@types/express": "^4.17.14",
+    "@types/morgan": "^1.9.9",
+    "@types/node-cron": "^3.0.11",
+    "@types/recursive-readdir": "^2.2.4",
+    "@types/sinon": "^17.0.4",
+    "@types/supertest": "^6.0.2",
+    "@typescript-eslint/eslint-plugin": "^8.6.0",
+    "@typescript-eslint/parser": "^8.6.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-plugin-prettier": "^5.2.1",
+    "mock-aws-s3": "^4.0.2",
+    "nock": "^14.0.1",
+    "openapi-typescript": "^6.7.6",
+    "supertest": "^7.0.0"
+  }
 }

package/publisher.config.json

@@ -1,26 +1,33 @@
 {
-    "frozenConfig": false,
-    "projects": [
-      {
-        "name": "malloy-samples",
-        "packages": [
-            {
-            "name": "ecommerce",
-            "location": "https://github.com/credibledata/malloy-samples"
-          },
-          {
-            "name": "imdb",
-            "location": "https://github.com/credibledata/malloy-samples"
-          },
-          {
-            "name": "faa",
-            "location": "https://github.com/credibledata/malloy-samples"
-          },
-          {
-            "name": "bigquery-hackernews",
-            "location": "https://github.com/credibledata/malloy-samples"
-          }
-        ]
-      }
-    ]
-  }
+  "frozenConfig": false,
+  "projects": [
+    {
+      "name": "malloy-samples",
+      "packages": [
+        {
+          "name": "ecommerce",
+          "location": "https://github.com/credibledata/malloy-samples/tree/main/ecommerce"
+        },
+        {
+          "name": "imdb",
+          "location": "https://github.com/credibledata/malloy-samples/tree/main/imdb"
+        },
+        {
+          "name": "faa",
+          "location": "https://github.com/credibledata/malloy-samples/tree/main/faa"
+        },
+        {
+          "name": "bigquery-hackernews",
+          "location": "https://github.com/credibledata/malloy-samples/tree/main/bigquery-hackernews"
+        }
+      ],
+      "connections": [
+        {
+          "name": "bigquery",
+          "type": "bigquery",
+          "bigqueryConnection": {}
+        }
+      ]
+    }
+  ]
+}

package/src/config.ts

@@ -1,18 +1,26 @@
 import fs from "fs";
 import path from "path";
-import { PUBLISHER_CONFIG_NAME } from "./constants";
+import { PUBLISHER_CONFIG_NAME, API_PREFIX } from "./constants";
+import { components } from "./api";
 
 type FilesystemPath = `./${string}` | `../${string}` | `/${string}`;
 type GcsPath = `gs://${string}`;
+type ApiConnection = components["schemas"]["Connection"];
 
 export type Package = {
    name: string;
    location: FilesystemPath | GcsPath;
 };
 
+export type Connection = {
+   name: string;
+   type: string;
+};
+
 export type Project = {
    name: string;
    packages: Package[];
+   connections?: Connection[];
 };
 
 export type PublisherConfig = {
@@ -20,6 +28,17 @@ export type PublisherConfig = {
    projects: Project[];
 };
 
+export type ProcessedProject = {
+   name: string;
+   packages: Package[];
+   connections: ApiConnection[];
+};
+
+export type ProcessedPublisherConfig = {
+   frozenConfig: boolean;
+   projects: ProcessedProject[];
+};
+
 export const getPublisherConfig = (serverRoot: string): PublisherConfig => {
    const publisherConfigPath = path.join(serverRoot, PUBLISHER_CONFIG_NAME);
    if (!fs.existsSync(publisherConfigPath)) {
@@ -47,3 +66,44 @@ export const isPublisherConfigFrozen = (serverRoot: string) => {
    const publisherConfig = getPublisherConfig(serverRoot);
    return Boolean(publisherConfig.frozenConfig);
 };
+
+export const getConnectionsFromPublisherConfig = (
+   serverRoot: string,
+   projectName: string,
+): Connection[] => {
+   const publisherConfig = getPublisherConfig(serverRoot);
+   const project = publisherConfig.projects.find((p) => p.name === projectName);
+   return project?.connections || [];
+};
+
+export const convertConnectionsToApiConnections = (
+   connections: Connection[],
+): ApiConnection[] => {
+   return connections.map((conn) => ({
+      ...conn,
+      name: conn.name,
+      type: conn.type as
+         | "postgres"
+         | "bigquery"
+         | "snowflake"
+         | "trino"
+         | "mysql",
+      resource: `${API_PREFIX}/connections/${conn.name}`,
+   }));
+};
+
+export const getProcessedPublisherConfig = (
+   serverRoot: string,
+): ProcessedPublisherConfig => {
+   const rawConfig = getPublisherConfig(serverRoot);
+   return {
+      frozenConfig: rawConfig.frozenConfig,
+      projects: rawConfig.projects.map((project) => ({
+         name: project.name,
+         packages: project.packages,
+         connections: convertConnectionsToApiConnections(
+            project.connections || [],
+         ),
+      })),
+   };
+};

package/src/constants.ts

@@ -5,7 +5,6 @@ export const API_PREFIX = "/api/v0";
 export const README_NAME = "README.md";
 export const PUBLISHER_CONFIG_NAME = "publisher.config.json";
 export const PACKAGE_MANIFEST_NAME = "publisher.json";
-export const CONNECTIONS_MANIFEST_NAME = "publisher.connections.json";
 export const MODEL_FILE_SUFFIX = ".malloy";
 export const NOTEBOOK_FILE_SUFFIX = ".malloynb";
 export const ROW_LIMIT = 1000;

package/src/controller/connection.controller.ts

@@ -46,7 +46,7 @@ export class ConnectionController {
       connectionName: string,
    ): Promise<ApiSchemaName[]> {
       const project = await this.projectStore.getProject(projectName, false);
-      const connection = project.getInternalConnection(connectionName);
+      const connection = project.getApiConnection(connectionName);
       return getSchemasForConnection(connection);
    }
 
@@ -57,7 +57,7 @@ export class ConnectionController {
       schemaName: string,
    ): Promise<string[]> {
       const project = await this.projectStore.getProject(projectName, false);
-      const connection = project.getInternalConnection(connectionName);
+      const connection = project.getApiConnection(connectionName);
       return getTablesForSchema(connection, schemaName);
    }
 

package/src/server.ts

@@ -3,7 +3,6 @@ import * as bodyParser from "body-parser";
 import cors from "cors";
 import express from "express";
 import * as http from "http";
-import { createProxyMiddleware } from "http-proxy-middleware";
 import { AddressInfo } from "net";
 import * as path from "path";
 import { ConnectionController } from "./controller/connection.controller";
@@ -81,7 +80,6 @@ const isDevelopment = process.env["NODE_ENV"] === "development";
 
 const app = express();
 app.use(loggerMiddleware);
-app.use(cors());
 
 const projectStore = new ProjectStore(SERVER_ROOT);
 const watchModeController = new WatchModeController(projectStore);
@@ -186,16 +184,6 @@ if (!isDevelopment) {
    // In development mode, proxy requests to React dev server
    // Handle API routes first
    app.use(`${API_PREFIX}`, loggerMiddleware);
-
-   // Proxy everything else to Vite
-   app.use(
-      createProxyMiddleware({
-         target: "http://localhost:5173",
-         changeOrigin: true,
-         ws: true,
-         pathFilter: (path) => !path.startsWith("/api/"),
-      }),
-   );
 }
 
 const setVersionIdError = (res: express.Response) => {
@@ -205,7 +193,12 @@ const setVersionIdError = (res: express.Response) => {
    res.status(status).json(json);
 };
 
-app.use(cors());
+app.use(
+   cors({
+      origin: "http://localhost:5173",
+      credentials: true,
+   }),
+);
 app.use(bodyParser.json());
 
 app.get(`${API_PREFIX}/status`, async (_req, res) => {
@@ -378,6 +371,9 @@ app.get(
    },
 );
 
+/**
+ * @deprecated Use /projects/:projectName/connections/:connectionName/sqlSource POST method instead
+ */
 app.get(
    `${API_PREFIX}/projects/:projectName/connections/:connectionName/sqlSource`,
    async (req, res) => {
@@ -397,6 +393,25 @@ app.get(
    },
 );
 
+app.post(
+   `${API_PREFIX}/projects/:projectName/connections/:connectionName/sqlSource`,
+   async (req, res) => {
+      try {
+         res.status(200).json(
+            await connectionController.getConnectionSqlSource(
+               req.params.projectName,
+               req.params.connectionName,
+               req.body.sqlStatement as string,
+            ),
+         );
+      } catch (error) {
+         logger.error(error);
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
 app.get(
    `${API_PREFIX}/projects/:projectName/connections/:connectionName/tableSource`,
    async (req, res) => {
@@ -417,6 +432,9 @@ app.get(
    },
 );
 
+/**
+ * @deprecated Use /projects/:projectName/connections/:connectionName/queryData POST method instead
+ */
 app.get(
    `${API_PREFIX}/projects/:projectName/connections/:connectionName/queryData`,
    async (req, res) => {
@@ -437,6 +455,29 @@ app.get(
    },
 );
 
+app.post(
+   `${API_PREFIX}/projects/:projectName/connections/:connectionName/queryData`,
+   async (req, res) => {
+      try {
+         res.status(200).json(
+            await connectionController.getConnectionQueryData(
+               req.params.projectName,
+               req.params.connectionName,
+               req.body.sqlStatement as string,
+               req.query.options as string,
+            ),
+         );
+      } catch (error) {
+         logger.error(error);
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+/**
+ * @deprecated Use /projects/:projectName/connections/:connectionName/temporaryTable POST method instead
+ */
 app.get(
    `${API_PREFIX}/projects/:projectName/connections/:connectionName/temporaryTable`,
    async (req, res) => {
@@ -456,6 +497,25 @@ app.get(
    },
 );
 
+app.post(
+   `${API_PREFIX}/projects/:projectName/connections/:connectionName/temporaryTable`,
+   async (req, res) => {
+      try {
+         res.status(200).json(
+            await connectionController.getConnectionTemporaryTable(
+               req.params.projectName,
+               req.params.connectionName,
+               req.body.sqlStatement as string,
+            ),
+         );
+      } catch (error) {
+         logger.error(error);
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
 app.get(`${API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
    if (req.query.versionId) {
       setVersionIdError(res);

package/src/service/connection.ts

@@ -11,7 +11,11 @@ import fs from "fs/promises";
 import path from "path";
 import { v4 as uuidv4 } from "uuid";
 import { components } from "../api";
-import { CONNECTIONS_MANIFEST_NAME, TEMP_DIR_PATH } from "../constants";
+import { TEMP_DIR_PATH } from "../constants";
+import {
+   convertConnectionsToApiConnections,
+   getConnectionsFromPublisherConfig,
+} from "../config";
 import { AxiosError } from "axios";
 
 type ApiConnection = components["schemas"]["Connection"];
@@ -29,32 +33,38 @@ export type InternalConnection = ApiConnection & {
 };
 
 export async function readConnectionConfig(
-   basePath: string,
-): Promise<InternalConnection[]> {
-   const fullPath = path.join(basePath, CONNECTIONS_MANIFEST_NAME);
-
-   try {
-      await fs.stat(fullPath);
-   } catch {
-      // If there's no connection manifest, it's no problem.  Just return an
-      // empty array.
-      return new Array<InternalConnection>();
+   _basePath: string,
+   projectName?: string,
+   serverRootPath?: string,
+): Promise<ApiConnection[]> {
+   // If no project name is provided, return empty array for backward compatibility
+   if (!projectName || !serverRootPath) {
+      return new Array<ApiConnection>();
    }
 
-   const connectionFileContents = await fs.readFile(fullPath);
-   // TODO: Validate connection manifest.  Define manifest type in public API.
-   return JSON.parse(connectionFileContents.toString()) as ApiConnection[];
+   // Get connections from publisher config
+   const connections = getConnectionsFromPublisherConfig(
+      serverRootPath,
+      projectName,
+   );
+   return convertConnectionsToApiConnections(connections);
 }
 
 export async function createConnections(
    basePath: string,
    defaultConnections: ApiConnection[] = [],
+   projectName?: string,
+   serverRootPath?: string,
 ): Promise<{
    malloyConnections: Map<string, BaseConnection>;
    apiConnections: InternalConnection[];
 }> {
    const connectionMap = new Map<string, BaseConnection>();
-   const connectionConfig = await readConnectionConfig(basePath);
+   const connectionConfig = await readConnectionConfig(
+      basePath,
+      projectName,
+      serverRootPath,
+   );
 
    const allConnections = [...defaultConnections, ...connectionConfig];