@malloy-publisher/server 0.0.191 → 0.0.193

package/src/server.ts

@@ -1,10 +1,4 @@
 // Pre-load the instrumentation module; the instrumentation module must be loaded before the other imports.
-import "./instrumentation";
-import {
-   getPrometheusMetricsHandler,
-   httpMetricsMiddleware,
-} from "./instrumentation";
-
 import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
 import bodyParser from "body-parser";
 import cors from "cors";
@@ -13,6 +7,7 @@ import * as http from "http";
 import { createProxyMiddleware } from "http-proxy-middleware";
 import { AddressInfo } from "net";
 import * as path from "path";
+import { ParsedQs } from "qs";
 import { fileURLToPath } from "url";
 import { CompileController } from "./controller/compile.controller";
 import { ConnectionController } from "./controller/connection.controller";
@@ -31,6 +26,11 @@ import {
    registerHealthEndpoints,
    registerSignalHandlers,
 } from "./health";
+import "./instrumentation";
+import {
+   getPrometheusMetricsHandler,
+   httpMetricsMiddleware,
+} from "./instrumentation";
 import { logger, loggerMiddleware } from "./logger";
 
 import { ManifestController } from "./controller/manifest.controller";
@@ -524,6 +524,70 @@ app.get(
    },
 );
 
+// ── Per-package connection data routes ─────────────────────────────
+// `duckdb` is per-package; non-`duckdb` names fall through to the
+// project's connection registry via the package's MalloyConfig wrapper.
+app.get(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/schemas`,
+   async (req, res) => {
+      try {
+         res.status(200).json(
+            await connectionController.listSchemas(
+               req.params.projectName,
+               req.params.connectionName,
+               req.params.packageName,
+            ),
+         );
+      } catch (error) {
+         logger.error(error);
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.get(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/schemas/:schemaName/tables`,
+   async (req, res) => {
+      try {
+         res.status(200).json(
+            await connectionController.listTables(
+               req.params.projectName,
+               req.params.connectionName,
+               req.params.schemaName,
+               normalizeQueryArray(req.query.tableNames),
+               req.params.packageName,
+            ),
+         );
+      } catch (error) {
+         logger.error(error);
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.get(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/schemas/:schemaName/tables/:tablePath`,
+   async (req, res) => {
+      try {
+         res.status(200).json(
+            await connectionController.getTable(
+               req.params.projectName,
+               req.params.connectionName,
+               req.params.schemaName,
+               req.params.tablePath,
+               req.params.packageName,
+            ),
+         );
+      } catch (error) {
+         logger.error(error);
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
 /**
  * @deprecated Use /projects/:projectName/connections/:connectionName/sqlSource POST method instead
  */
@@ -565,8 +629,49 @@ app.post(
    },
 );
 
+// Per-package versions
+app.get(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/sqlSource`,
+   async (req, res) => {
+      try {
+         res.status(200).json(
+            await connectionController.getConnectionSqlSource(
+               req.params.projectName,
+               req.params.connectionName,
+               req.query.sqlStatement as string,
+               req.params.packageName,
+            ),
+         );
+      } catch (error) {
+         logger.error(error);
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/sqlSource`,
+   async (req, res) => {
+      try {
+         res.status(200).json(
+            await connectionController.getConnectionSqlSource(
+               req.params.projectName,
+               req.params.connectionName,
+               req.body.sqlStatement as string,
+               req.params.packageName,
+            ),
+         );
+      } catch (error) {
+         logger.error(error);
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
 /**
- * @deprecated Use /projects/:projectName/connections/:connectionName/queryData POST method instead
+ * @deprecated Use /projects/:projectName/connections/:connectionName/sqlQuery POST method instead
  */
 app.get(
    `${API_PREFIX}/projects/:projectName/connections/:connectionName/queryData`,
@@ -588,16 +693,76 @@ app.get(
    },
 );
 
+/**
+ * @deprecated Use /projects/:projectName/packages/:packageName/connections/:connectionName/sqlQuery
+ */
+app.get(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/queryData`,
+   async (req, res) => {
+      try {
+         res.status(200).json(
+            await connectionController.getConnectionQueryData(
+               req.params.projectName,
+               req.params.connectionName,
+               req.query.sqlStatement as string,
+               req.query.options as string,
+               req.params.packageName,
+            ),
+         );
+      } catch (error) {
+         logger.error(error);
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
 app.post(
    `${API_PREFIX}/projects/:projectName/connections/:connectionName/sqlQuery`,
    async (req, res) => {
       try {
+         let options: string | ParsedQs | (string | ParsedQs)[] | undefined;
+
+         // Support both body and query parameters for options for backwards compatibility
+         // TODO: To be removed in the future
+         if (req.body?.options) {
+            options = req.body.options;
+         } else {
+            options = req.query.options;
+         }
          res.status(200).json(
             await connectionController.getConnectionQueryData(
                req.params.projectName,
                req.params.connectionName,
                req.body.sqlStatement as string,
-               req.body.options as string,
+               options as string,
+            ),
+         );
+      } catch (error) {
+         logger.error(error);
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/sqlQuery`,
+   async (req, res) => {
+      try {
+         let options: string | ParsedQs | (string | ParsedQs)[] | undefined;
+         if (req.body?.options) {
+            options = req.body.options;
+         } else {
+            options = req.query.options;
+         }
+         res.status(200).json(
+            await connectionController.getConnectionQueryData(
+               req.params.projectName,
+               req.params.connectionName,
+               req.body.sqlStatement as string,
+               options as string,
+               req.params.packageName,
             ),
          );
       } catch (error) {
@@ -609,7 +774,7 @@ app.post(
 );
 
 /**
- * @deprecated Use /projects/:projectName/connections/:connectionName/temporaryTable POST method instead
+ * @deprecated Use /projects/:projectName/connections/:connectionName/sqlTemporaryTable POST method instead
  */
 app.get(
    `${API_PREFIX}/projects/:projectName/connections/:connectionName/temporaryTable`,
@@ -630,6 +795,29 @@ app.get(
    },
 );
 
+/**
+ * @deprecated Use /projects/:projectName/packages/:packageName/connections/:connectionName/sqlTemporaryTable
+ */
+app.get(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/temporaryTable`,
+   async (req, res) => {
+      try {
+         res.status(200).json(
+            await connectionController.getConnectionTemporaryTable(
+               req.params.projectName,
+               req.params.connectionName,
+               req.query.sqlStatement as string,
+               req.params.packageName,
+            ),
+         );
+      } catch (error) {
+         logger.error(error);
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
 app.post(
    `${API_PREFIX}/projects/:projectName/connections/:connectionName/sqlTemporaryTable`,
    async (req, res) => {
@@ -649,6 +837,26 @@ app.post(
    },
 );
 
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/sqlTemporaryTable`,
+   async (req, res) => {
+      try {
+         res.status(200).json(
+            await connectionController.getConnectionTemporaryTable(
+               req.params.projectName,
+               req.params.connectionName,
+               req.body.sqlStatement as string,
+               req.params.packageName,
+            ),
+         );
+      } catch (error) {
+         logger.error(error);
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
 app.get(`${API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
    if (req.query.versionId) {
       setVersionIdError(res);

package/src/service/connection.spec.ts

@@ -4,6 +4,7 @@ import path from "path";
 import sinon from "sinon";
 import { DuckDBConnection } from "@malloydata/db-duckdb";
 import { createProjectConnections, testConnectionConfig } from "./connection";
+import { assembleProjectConnections } from "./connection_config";
 import { components } from "../api";
 
 type ApiConnection = components["schemas"]["Connection"];
@@ -1121,19 +1122,264 @@ describe("connection integration tests", () => {
             ).rejects.toThrow(/cannot be 'duckdb'/);
          });
 
-         it("should throw error if no attached databases configured", async () => {
+         it("should allow DuckDB connections with no attachments", async () => {
+            const { malloyConnections } = await createProjectConnections(
+               [
+                  {
+                     name: "empty_duckdb",
+                     type: "duckdb",
+                     duckdbConnection: { attachedDatabases: [] },
+                  },
+               ],
+               testProjectPath,
+            );
+
+            const connection = malloyConnections.get("empty_duckdb");
+            expect(connection).toBeDefined();
+         });
+
+         it("should reject unsupported DuckDB connector fields", async () => {
             await expect(
                createProjectConnections(
                   [
                      {
-                        name: "empty_duckdb",
+                        name: "duckdb_with_setup_sql",
                         type: "duckdb",
-                        duckdbConnection: { attachedDatabases: [] },
+                        duckdbConnection: {
+                           attachedDatabases: [],
+                           setupSQL: "INSTALL httpfs",
+                        },
+                     } as unknown as ApiConnection,
+                  ],
+                  testProjectPath,
+               ),
+            ).rejects.toThrow(/Unsupported DuckDB connection field/);
+         });
+
+         it("should reject project-authored DuckDB policy fields", async () => {
+            await expect(
+               createProjectConnections(
+                  [
+                     {
+                        name: "duckdb_with_policy",
+                        type: "duckdb",
+                        duckdbConnection: {
+                           attachedDatabases: [],
+                           securityPolicy: "sandboxed",
+                        },
+                     } as unknown as ApiConnection,
+                  ],
+                  testProjectPath,
+               ),
+            ).rejects.toThrow(/Unsupported DuckDB connection field/);
+         });
+
+         it("should preserve Snowflake private-key auth options", async () => {
+            const { malloyConnections, releaseConnections } =
+               await createProjectConnections(
+                  [
+                     {
+                        name: "sf_private_key",
+                        type: "snowflake",
+                        snowflakeConnection: {
+                           account: "test-account",
+                           username: "test-user",
+                           privateKey:
+                              "-----BEGIN PRIVATE KEY-----MIIB-----END PRIVATE KEY-----",
+                           warehouse: "test-warehouse",
+                        },
+                     },
+                  ],
+                  testProjectPath,
+               );
+
+            try {
+               const connection = malloyConnections.get(
+                  "sf_private_key",
+               ) as unknown as { connOptions: Record<string, unknown> };
+               expect(connection.connOptions.authenticator).toBe(
+                  "SNOWFLAKE_JWT",
+               );
+               expect(connection.connOptions.privateKey).toContain(
+                  "BEGIN PRIVATE KEY",
+               );
+            } finally {
+               await releaseConnections();
+            }
+         });
+
+         it("should translate Trino Peaka credentials to core extraCredential", () => {
+            const assembled = assembleProjectConnections(
+               [
+                  {
+                     name: "trino_peaka",
+                     type: "trino",
+                     trinoConnection: {
+                        server: "https://example.com",
+                        port: 443,
+                        catalog: "catalog",
+                        schema: "schema",
+                        user: "user",
+                        peakaKey: "peaka-secret",
+                     },
+                  },
+               ],
+               testProjectPath,
+            );
+
+            expect(
+               assembled.pojo.connections.trino_peaka.extraCredential,
+            ).toEqual({ peakaKey: "peaka-secret" });
+            expect(
+               assembled.pojo.connections.trino_peaka.extraConfig,
+            ).toBeUndefined();
+         });
+
+         it("should validate project-level BigQuery service account keys", () => {
+            expect(() =>
+               assembleProjectConnections(
+                  [
+                     {
+                        name: "bq_invalid",
+                        type: "bigquery",
+                        bigqueryConnection: {
+                           defaultProjectId: "test-project",
+                           serviceAccountKeyJson: '{"invalid":"key"}',
+                        },
                      },
                   ],
                   testProjectPath,
                ),
-            ).rejects.toThrow(/at least one attached database/);
+            ).toThrow(/missing "type" field/);
+         });
+
+         it("should preserve PGSSLMODE for project-level Postgres", () => {
+            const previousPgSslMode = process.env.PGSSLMODE;
+            process.env.PGSSLMODE = "require";
+            try {
+               const assembled = assembleProjectConnections(
+                  [
+                     {
+                        name: "pg_ssl",
+                        type: "postgres",
+                        postgresConnection: {
+                           host: "localhost",
+                           port: 5432,
+                           userName: "user",
+                           password: "pass",
+                           databaseName: "db",
+                        },
+                     },
+                  ],
+                  testProjectPath,
+               );
+
+               expect(
+                  assembled.pojo.connections.pg_ssl.connectionString,
+               ).toContain("sslmode=require");
+            } finally {
+               if (previousPgSslMode === undefined) {
+                  delete process.env.PGSSLMODE;
+               } else {
+                  process.env.PGSSLMODE = previousPgSslMode;
+               }
+            }
+         });
+
+         it("should use project-root-relative file paths for project-level DuckDB", async () => {
+            const insideCsvPath = path.join(testProjectPath, "inside.csv");
+            await fs.writeFile(insideCsvPath, "id\n1\n");
+
+            const { malloyConnections } = await createProjectConnections(
+               [
+                  {
+                     name: "project_scoped_duckdb",
+                     type: "duckdb",
+                     duckdbConnection: { attachedDatabases: [] },
+                  },
+               ],
+               testProjectPath,
+            );
+
+            const connection = malloyConnections.get(
+               "project_scoped_duckdb",
+            ) as DuckDBConnection;
+            createdConnections.push(connection);
+
+            const assembled = assembleProjectConnections(
+               [
+                  {
+                     name: "project_scoped_duckdb",
+                     type: "duckdb",
+                     duckdbConnection: { attachedDatabases: [] },
+                  },
+               ],
+               testProjectPath,
+            );
+            expect(
+               assembled.pojo.connections.project_scoped_duckdb
+                  .workingDirectory,
+            ).toBeUndefined();
+            expect(
+               assembled.pojo.connections.project_scoped_duckdb.securityPolicy,
+            ).toBeUndefined();
+
+            await expect(
+               connection.runSQL("SELECT * FROM read_csv_auto('inside.csv')"),
+            ).resolves.toBeDefined();
+         });
+
+         it("should keep external access available for federated DuckDB entries", () => {
+            const assembled = assembleProjectConnections(
+               [
+                  {
+                     name: "federated_duckdb",
+                     type: "duckdb",
+                     duckdbConnection: {
+                        attachedDatabases: [
+                           {
+                              name: "pg",
+                              type: "postgres",
+                              postgresConnection: {
+                                 host: "localhost",
+                                 port: 5432,
+                                 userName: "user",
+                                 password: "pass",
+                                 databaseName: "db",
+                              },
+                           },
+                        ],
+                     },
+                  },
+               ],
+               testProjectPath,
+            );
+
+            const entry = assembled.pojo.connections.federated_duckdb;
+            expect(entry.securityPolicy).toBeUndefined();
+            expect(entry.enableExternalAccess).toBeUndefined();
+            expect(entry.allowedDirectories).toBeUndefined();
+         });
+
+         it("should keep external access available for MotherDuck entries", () => {
+            const assembled = assembleProjectConnections(
+               [
+                  {
+                     name: "md",
+                     type: "motherduck",
+                     motherduckConnection: {
+                        accessToken: "token",
+                        database: "db",
+                     },
+                  },
+               ],
+               testProjectPath,
+            );
+
+            const entry = assembled.pojo.connections.md;
+            expect(entry.securityPolicy).toBeUndefined();
+            expect(entry.enableExternalAccess).toBeUndefined();
+            expect(entry.allowedDirectories).toBeUndefined();
          });
 
          it("should handle already attached database gracefully", async () => {