@malloy-publisher/server 0.0.151 → 0.0.153

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@malloy-publisher/server",
   "description": "Malloy Publisher Server",
-  "version": "0.0.151",
+  "version": "0.0.153",
   "main": "dist/server.js",
   "bin": {
     "malloy-publisher": "dist/server.js"
@@ -24,6 +24,7 @@
     "generate-api-types": "bunx openapi-typescript ../../api-doc.yaml --output src/api.ts"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.958.0",
     "@google-cloud/storage": "^7.16.0",
     "@malloydata/db-bigquery": "^0.0.318",
     "@malloydata/db-duckdb": "^0.0.318",
@@ -48,7 +49,7 @@
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.1",
     "cors": "^2.8.5",
-    "duckdb": "^1.4.1",
+    "duckdb": "1.3.4",
     "express": "^4.21.0",
     "globals": "^15.9.0",
     "handlebars": "^4.7.8",

package/src/logger.ts

@@ -9,6 +9,9 @@ export const logger = winston.createLogger({
       ? winston.format.combine(
            winston.format.uncolorize(),
            winston.format.timestamp(),
+           winston.format.metadata({
+              fillExcept: ["message", "level", "timestamp"],
+           }),
            winston.format.json(),
         )
       : winston.format.combine(
@@ -18,6 +21,33 @@ export const logger = winston.createLogger({
    transports: [new winston.transports.Console()],
 });
 
+/**
+ * Extracts the trace ID from a W3C traceparent header.
+ * Format: version-trace-id-parent-id-trace-flags
+ * Example: 00-81f2264f363f1b5596c84ab29e6be171-83ef39df12ab6bab-01
+ *
+ * @param traceparent The traceparent header value
+ * @returns The trace ID (32 hex characters) or undefined if invalid
+ */
+function extractTraceIdFromTraceparent(
+   traceparent: string | undefined,
+): string | undefined {
+   if (!traceparent) {
+      return undefined;
+   }
+
+   // format of traceparent can either be: version-traceId-parentId-traceFlags or traceId
+   const parts = traceparent.split("-");
+   const traceId =
+      parts.length >= 2 ? parts[1] : parts.length == 1 ? parts[0] : undefined;
+   // Validate that the traceId is 32 hex characters
+   if (traceId && traceId.length === 32 && /^[0-9a-fA-F]{32}$/.test(traceId)) {
+      return traceId;
+   }
+
+   return undefined;
+}
+
 export const loggerMiddleware: RequestHandler = (req, res, next) => {
    const startTime = performance.now();
    const resJson = res.json;
@@ -27,14 +57,26 @@ export const loggerMiddleware: RequestHandler = (req, res, next) => {
    };
    res.on("finish", () => {
       const endTime = performance.now();
-      logger.info(`${req.method} ${req.url}`, {
+
+      // Extract trace ID from traceparent header if present
+      const traceparent = req.headers["traceparent"] as string | undefined;
+      const traceId = extractTraceIdFromTraceparent(traceparent);
+
+      const logMetadata: Record<string, unknown> = {
          statusCode: res.statusCode,
          duration: endTime - startTime,
          payload: req.body,
          response: res.locals.body,
          params: req.params,
          query: req.query,
-      });
+      };
+
+      // Add traceId to log metadata if present
+      if (traceId) {
+         logMetadata.traceId = traceId;
+      }
+
+      logger.info(`${req.method} ${req.url}`, logMetadata);
    });
    next();
 };

package/src/service/connection.ts

@@ -13,6 +13,7 @@ import { v4 as uuidv4 } from "uuid";
 import { components } from "../api";
 import { TEMP_DIR_PATH } from "../constants";
 import { logAxiosError, logger } from "../logger";
+import { CloudStorageCredentials } from "./gcs_s3_utils";
 
 type AttachedDatabase = components["schemas"]["AttachedDatabase"];
 type ApiConnection = components["schemas"]["Connection"];
@@ -377,6 +378,121 @@ async function attachPostgres(
    logger.info(`Successfully attached PostgreSQL database: ${attachedDb.name}`);
 }
 
+async function attachCloudStorage(
+   connection: DuckDBConnection,
+   attachedDb: AttachedDatabase,
+): Promise<void> {
+   const isGCS = attachedDb.type === "gcs";
+   const isS3 = attachedDb.type === "s3";
+
+   if (!isGCS && !isS3) {
+      throw new Error(`Invalid cloud storage type: ${attachedDb.type}`);
+   }
+
+   const storageType = attachedDb.type?.toUpperCase() || "";
+   let credentials: CloudStorageCredentials;
+
+   if (isGCS) {
+      if (!attachedDb.gcsConnection) {
+         throw new Error(
+            `GCS connection configuration missing for: ${attachedDb.name}`,
+         );
+      }
+      if (!attachedDb.gcsConnection.keyId || !attachedDb.gcsConnection.secret) {
+         throw new Error(
+            `GCS keyId and secret are required for: ${attachedDb.name}`,
+         );
+      }
+      credentials = {
+         type: "gcs",
+         accessKeyId: attachedDb.gcsConnection.keyId,
+         secretAccessKey: attachedDb.gcsConnection.secret,
+      };
+   } else {
+      if (!attachedDb.s3Connection) {
+         throw new Error(
+            `S3 connection configuration missing for: ${attachedDb.name}`,
+         );
+      }
+      if (
+         !attachedDb.s3Connection.accessKeyId ||
+         !attachedDb.s3Connection.secretAccessKey
+      ) {
+         throw new Error(
+            `S3 accessKeyId and secretAccessKey are required for: ${attachedDb.name}`,
+         );
+      }
+      credentials = {
+         type: "s3",
+         accessKeyId: attachedDb.s3Connection.accessKeyId,
+         secretAccessKey: attachedDb.s3Connection.secretAccessKey,
+         region: attachedDb.s3Connection.region,
+         endpoint: attachedDb.s3Connection.endpoint,
+         sessionToken: attachedDb.s3Connection.sessionToken,
+      };
+   }
+
+   await installAndLoadExtension(connection, "httpfs");
+
+   const secretName = sanitizeSecretName(
+      `${attachedDb.type}_${attachedDb.name}`,
+   );
+   const escapedKeyId = escapeSQL(credentials.accessKeyId);
+   const escapedSecret = escapeSQL(credentials.secretAccessKey);
+
+   let createSecretCommand: string;
+
+   if (isGCS) {
+      createSecretCommand = `
+         CREATE OR REPLACE SECRET ${secretName} (
+            TYPE gcs,
+            KEY_ID '${escapedKeyId}',
+            SECRET '${escapedSecret}'
+         );
+      `;
+   } else {
+      const region = credentials.region || "us-east-1";
+
+      if (credentials.endpoint) {
+         const escapedEndpoint = escapeSQL(credentials.endpoint);
+         createSecretCommand = `
+            CREATE OR REPLACE SECRET ${secretName} (
+               TYPE s3,
+               KEY_ID '${escapedKeyId}',
+               SECRET '${escapedSecret}',
+               REGION '${region}',
+               ENDPOINT '${escapedEndpoint}',
+               URL_STYLE 'path'
+            );
+         `;
+      } else if (credentials.sessionToken) {
+         const escapedToken = escapeSQL(credentials.sessionToken);
+         createSecretCommand = `
+            CREATE OR REPLACE SECRET ${secretName} (
+               TYPE s3,
+               KEY_ID '${escapedKeyId}',
+               SECRET '${escapedSecret}',
+               REGION '${region}',
+               SESSION_TOKEN '${escapedToken}'
+            );
+         `;
+      } else {
+         createSecretCommand = `
+            CREATE OR REPLACE SECRET ${secretName} (
+               TYPE s3,
+               KEY_ID '${escapedKeyId}',
+               SECRET '${escapedSecret}',
+               REGION '${region}'
+            );
+         `;
+      }
+   }
+
+   await connection.runSQL(createSecretCommand);
+   logger.info(`Created ${storageType} secret: ${secretName}`);
+   logger.info(`${storageType} connection configured for: ${attachedDb.name}`);
+}
+
 // Main attachment function
 async function attachDatabasesToDuckDB(
    duckdbConnection: DuckDBConnection,
@@ -386,6 +502,8 @@ async function attachDatabasesToDuckDB(
       bigquery: attachBigQuery,
       snowflake: attachSnowflake,
       postgres: attachPostgres,
+      gcs: attachCloudStorage,
+      s3: attachCloudStorage,
    };
 
    for (const attachedDb of attachedDatabases) {
@@ -633,9 +751,10 @@ export async function createProjectConnections(
 
             // Create DuckDB connection with project basePath as working directory
             // This ensures relative paths in the project are resolved correctly
+            // Use unique memory database path to prevent sharing across connections
             const duckdbConnection = new DuckDBConnection(
                connection.name,
-               ":memory:",
+               path.join(projectPath, `${connection.name}.duckdb`),
                projectPath,
             );
 
@@ -747,9 +866,10 @@ export async function createPackageDuckDBConnections(
 
       // Create DuckDB connection with project basePath as working directory
       // This ensures relative paths in the project are resolved correctly
+      // Use unique memory database path to prevent sharing across connections
       const duckdbConnection = new DuckDBConnection(
          connection.name,
-         ":memory:",
+         path.join(packagePath, `${connection.name}.duckdb`),
          packagePath,
       );
 

package/src/service/db_utils.ts

@@ -3,6 +3,15 @@ import { Connection, TableSourceDef } from "@malloydata/malloy";
 import { components } from "../api";
 import { ConnectionError } from "../errors";
 import { logger } from "../logger";
+import {
+   CloudStorageCredentials,
+   gcsConnectionToCredentials,
+   getCloudTablesWithColumns,
+   isDataFile,
+   listAllCloudFiles,
+   listCloudBuckets,
+   s3ConnectionToCredentials,
+} from "./gcs_s3_utils";
 import { ApiConnection } from "./model";
 
 type ApiSchema = components["schemas"]["Schema"];
@@ -68,6 +77,29 @@ function standardizeRunSQLResult(result: unknown): unknown[] {
       : (result as { rows?: unknown[] }).rows || [];
 }
 
+function getCloudCredentialsFromAttachedDatabases(
+   attachedDatabases: components["schemas"]["AttachedDatabase"][],
+   storageType: "gcs" | "s3",
+): CloudStorageCredentials | null {
+   for (const attachedDb of attachedDatabases) {
+      if (
+         attachedDb.type === "gcs" &&
+         storageType === "gcs" &&
+         attachedDb.gcsConnection
+      ) {
+         return gcsConnectionToCredentials(attachedDb.gcsConnection);
+      }
+      if (
+         attachedDb.type === "s3" &&
+         storageType === "s3" &&
+         attachedDb.s3Connection
+      ) {
+         return s3ConnectionToCredentials(attachedDb.s3Connection);
+      }
+   }
+   return null;
+}
+
 export async function getSchemasForConnection(
    connection: ApiConnection,
    malloyConnection: Connection,
@@ -265,7 +297,7 @@ export async function getSchemasForConnection(
 
          const rows = standardizeRunSQLResult(result);
 
-         return rows.map((row: unknown) => {
+         const schemas: ApiSchema[] = rows.map((row: unknown) => {
             const typedRow = row as Record<string, unknown>;
             const schemaName = typedRow.schema_name as string;
             const catalogName = typedRow.catalog_name as string;
@@ -288,6 +320,42 @@ export async function getSchemasForConnection(
                isDefault: catalogName === "main",
             };
          });
+
+         const attachedDatabases =
+            connection.duckdbConnection.attachedDatabases || [];
+
+         for (const attachedDb of attachedDatabases) {
+            if (
+               (attachedDb.type === "gcs" || attachedDb.type === "s3") &&
+               (attachedDb.gcsConnection || attachedDb.s3Connection)
+            ) {
+               const credentials =
+                  attachedDb.type === "gcs"
+                     ? gcsConnectionToCredentials(attachedDb.gcsConnection!)
+                     : s3ConnectionToCredentials(attachedDb.s3Connection!);
+
+               try {
+                  const buckets = await listCloudBuckets(credentials);
+                  for (const bucket of buckets) {
+                     schemas.push({
+                        name: `${attachedDb.type}.${bucket.name}`,
+                        isHidden: false,
+                        isDefault: false,
+                     });
+                  }
+                  logger.info(
+                     `Listed ${buckets.length} ${attachedDb.type.toUpperCase()} buckets for attached database ${attachedDb.name}`,
+                  );
+               } catch (cloudError) {
+                  logger.warn(
+                     `Failed to list ${attachedDb.type.toUpperCase()} buckets for ${attachedDb.name}`,
+                     { error: cloudError },
+                  );
+               }
+            }
+         }
+
+         return schemas;
       } catch (error) {
          console.error(
             `Error getting schemas for DuckDB connection ${connection.name}:`,
@@ -347,6 +415,41 @@ export async function getTablesForSchema(
       malloyConnection,
    );
 
+   const catalogName = schemaName.split(".")[0];
+
+   if (
+      (catalogName === "gcs" || catalogName === "s3") &&
+      connection.type === "duckdb"
+   ) {
+      console.log(
+         `Getting ${catalogName.toUpperCase()} tables for schema`,
+         schemaName,
+      );
+      console.log("tableNames", tableNames);
+      const bucketName = schemaName.split(".")[1];
+      console.log("bucketName", bucketName);
+
+      const attachedDatabases =
+         connection.duckdbConnection?.attachedDatabases || [];
+      const credentials = getCloudCredentialsFromAttachedDatabases(
+         attachedDatabases,
+         catalogName as "gcs" | "s3",
+      );
+
+      if (!credentials) {
+         throw new Error(
+            `${catalogName.toUpperCase()} credentials not found in attached databases`,
+         );
+      }
+
+      return await getCloudTablesWithColumns(
+         malloyConnection,
+         credentials,
+         bucketName,
+         tableNames,
+      );
+   }
+
    // Fetch all table sources in parallel
    const tableSourcePromises = tableNames.map(async (tableName) => {
       try {
@@ -598,11 +701,48 @@ export async function listTablesForSchema(
       if (!connection.duckdbConnection) {
          throw new Error("DuckDB connection is required");
       }
+
+      const catalogName = schemaName.split(".")[0];
+      const actualSchemaName = schemaName.split(".")[1];
+
+      if (catalogName === "gcs" || catalogName === "s3") {
+         const bucketName = actualSchemaName;
+         const attachedDatabases =
+            connection.duckdbConnection.attachedDatabases || [];
+
+         const credentials = getCloudCredentialsFromAttachedDatabases(
+            attachedDatabases,
+            catalogName as "gcs" | "s3",
+         );
+
+         if (!credentials) {
+            throw new Error(
+               `${catalogName.toUpperCase()} credentials not found in attached databases`,
+            );
+         }
+
+         try {
+            const objects = await listAllCloudFiles(credentials, bucketName);
+            return objects
+               .filter((obj) => isDataFile(obj.key))
+               .map((obj) => obj.key);
+         } catch (error) {
+            logger.error(
+               `Error listing ${catalogName.toUpperCase()} objects in bucket ${bucketName}`,
+               {
+                  error,
+               },
+            );
+            throw new Error(
+               `Failed to list files in ${catalogName.toUpperCase()} bucket ${bucketName}: ${(error as Error).message}`,
+            );
+         }
+      }
+
+      // Regular DuckDB table listing
       try {
-         const catalogName = schemaName.split(".")[0];
-         schemaName = schemaName.split(".")[1];
          const result = await malloyConnection.runSQL(
-            `SELECT table_name FROM information_schema.tables WHERE table_schema = '${schemaName}' and table_catalog = '${catalogName}' ORDER BY table_name`,
+            `SELECT table_name FROM information_schema.tables WHERE table_schema = '${actualSchemaName}' and table_catalog = '${catalogName}' ORDER BY table_name`,
             { rowLimit: 1000 },
          );