@malloy-publisher/server 0.0.151 → 0.0.152

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@malloy-publisher/server",
   "description": "Malloy Publisher Server",
-  "version": "0.0.151",
+  "version": "0.0.152",
   "main": "dist/server.js",
   "bin": {
     "malloy-publisher": "dist/server.js"
@@ -24,6 +24,7 @@
     "generate-api-types": "bunx openapi-typescript ../../api-doc.yaml --output src/api.ts"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.958.0",
     "@google-cloud/storage": "^7.16.0",
     "@malloydata/db-bigquery": "^0.0.318",
     "@malloydata/db-duckdb": "^0.0.318",
@@ -48,7 +49,7 @@
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.1",
     "cors": "^2.8.5",
-    "duckdb": "^1.4.1",
+    "duckdb": "1.3.4",
     "express": "^4.21.0",
     "globals": "^15.9.0",
     "handlebars": "^4.7.8",

package/src/service/connection.ts

@@ -13,6 +13,7 @@ import { v4 as uuidv4 } from "uuid";
 import { components } from "../api";
 import { TEMP_DIR_PATH } from "../constants";
 import { logAxiosError, logger } from "../logger";
+import { CloudStorageCredentials } from "./gcs_s3_utils";
 
 type AttachedDatabase = components["schemas"]["AttachedDatabase"];
 type ApiConnection = components["schemas"]["Connection"];
@@ -377,6 +378,121 @@ async function attachPostgres(
    logger.info(`Successfully attached PostgreSQL database: ${attachedDb.name}`);
 }
 
+async function attachCloudStorage(
+   connection: DuckDBConnection,
+   attachedDb: AttachedDatabase,
+): Promise<void> {
+   const isGCS = attachedDb.type === "gcs";
+   const isS3 = attachedDb.type === "s3";
+
+   if (!isGCS && !isS3) {
+      throw new Error(`Invalid cloud storage type: ${attachedDb.type}`);
+   }
+
+   const storageType = attachedDb.type?.toUpperCase() || "";
+   let credentials: CloudStorageCredentials;
+
+   if (isGCS) {
+      if (!attachedDb.gcsConnection) {
+         throw new Error(
+            `GCS connection configuration missing for: ${attachedDb.name}`,
+         );
+      }
+      if (!attachedDb.gcsConnection.keyId || !attachedDb.gcsConnection.secret) {
+         throw new Error(
+            `GCS keyId and secret are required for: ${attachedDb.name}`,
+         );
+      }
+      credentials = {
+         type: "gcs",
+         accessKeyId: attachedDb.gcsConnection.keyId,
+         secretAccessKey: attachedDb.gcsConnection.secret,
+      };
+   } else {
+      if (!attachedDb.s3Connection) {
+         throw new Error(
+            `S3 connection configuration missing for: ${attachedDb.name}`,
+         );
+      }
+      if (
+         !attachedDb.s3Connection.accessKeyId ||
+         !attachedDb.s3Connection.secretAccessKey
+      ) {
+         throw new Error(
+            `S3 accessKeyId and secretAccessKey are required for: ${attachedDb.name}`,
+         );
+      }
+      credentials = {
+         type: "s3",
+         accessKeyId: attachedDb.s3Connection.accessKeyId,
+         secretAccessKey: attachedDb.s3Connection.secretAccessKey,
+         region: attachedDb.s3Connection.region,
+         endpoint: attachedDb.s3Connection.endpoint,
+         sessionToken: attachedDb.s3Connection.sessionToken,
+      };
+   }
+
+   await installAndLoadExtension(connection, "httpfs");
+
+   const secretName = sanitizeSecretName(
+      `${attachedDb.type}_${attachedDb.name}`,
+   );
+   const escapedKeyId = escapeSQL(credentials.accessKeyId);
+   const escapedSecret = escapeSQL(credentials.secretAccessKey);
+
+   let createSecretCommand: string;
+
+   if (isGCS) {
+      createSecretCommand = `
+         CREATE OR REPLACE SECRET ${secretName} (
+            TYPE gcs,
+            KEY_ID '${escapedKeyId}',
+            SECRET '${escapedSecret}'
+         );
+      `;
+   } else {
+      const region = credentials.region || "us-east-1";
+
+      if (credentials.endpoint) {
+         const escapedEndpoint = escapeSQL(credentials.endpoint);
+         createSecretCommand = `
+            CREATE OR REPLACE SECRET ${secretName} (
+               TYPE s3,
+               KEY_ID '${escapedKeyId}',
+               SECRET '${escapedSecret}',
+               REGION '${region}',
+               ENDPOINT '${escapedEndpoint}',
+               URL_STYLE 'path'
+            );
+         `;
+      } else if (credentials.sessionToken) {
+         const escapedToken = escapeSQL(credentials.sessionToken);
+         createSecretCommand = `
+            CREATE OR REPLACE SECRET ${secretName} (
+               TYPE s3,
+               KEY_ID '${escapedKeyId}',
+               SECRET '${escapedSecret}',
+               REGION '${region}',
+               SESSION_TOKEN '${escapedToken}'
+            );
+         `;
+      } else {
+         createSecretCommand = `
+            CREATE OR REPLACE SECRET ${secretName} (
+               TYPE s3,
+               KEY_ID '${escapedKeyId}',
+               SECRET '${escapedSecret}',
+               REGION '${region}'
+            );
+         `;
+      }
+   }
+
+   await connection.runSQL(createSecretCommand);
+   logger.info(`Created ${storageType} secret: ${secretName}`);
+   logger.info(`${storageType} connection configured for: ${attachedDb.name}`);
+}
+
 // Main attachment function
 async function attachDatabasesToDuckDB(
    duckdbConnection: DuckDBConnection,
@@ -386,6 +502,8 @@ async function attachDatabasesToDuckDB(
       bigquery: attachBigQuery,
       snowflake: attachSnowflake,
       postgres: attachPostgres,
+      gcs: attachCloudStorage,
+      s3: attachCloudStorage,
    };
 
    for (const attachedDb of attachedDatabases) {
@@ -633,9 +751,10 @@ export async function createProjectConnections(
 
             // Create DuckDB connection with project basePath as working directory
             // This ensures relative paths in the project are resolved correctly
+            // Use unique memory database path to prevent sharing across connections
             const duckdbConnection = new DuckDBConnection(
                connection.name,
-               ":memory:",
+               path.join(projectPath, `${connection.name}.duckdb`),
                projectPath,
             );
 
@@ -747,9 +866,10 @@ export async function createPackageDuckDBConnections(
 
       // Create DuckDB connection with project basePath as working directory
       // This ensures relative paths in the project are resolved correctly
+      // Use unique memory database path to prevent sharing across connections
       const duckdbConnection = new DuckDBConnection(
          connection.name,
-         ":memory:",
+         path.join(packagePath, `${connection.name}.duckdb`),
          packagePath,
       );
 

package/src/service/db_utils.ts

@@ -3,6 +3,15 @@ import { Connection, TableSourceDef } from "@malloydata/malloy";
 import { components } from "../api";
 import { ConnectionError } from "../errors";
 import { logger } from "../logger";
+import {
+   CloudStorageCredentials,
+   gcsConnectionToCredentials,
+   getCloudTablesWithColumns,
+   isDataFile,
+   listAllCloudFiles,
+   listCloudBuckets,
+   s3ConnectionToCredentials,
+} from "./gcs_s3_utils";
 import { ApiConnection } from "./model";
 
 type ApiSchema = components["schemas"]["Schema"];
@@ -68,6 +77,29 @@ function standardizeRunSQLResult(result: unknown): unknown[] {
       : (result as { rows?: unknown[] }).rows || [];
 }
 
+function getCloudCredentialsFromAttachedDatabases(
+   attachedDatabases: components["schemas"]["AttachedDatabase"][],
+   storageType: "gcs" | "s3",
+): CloudStorageCredentials | null {
+   for (const attachedDb of attachedDatabases) {
+      if (
+         attachedDb.type === "gcs" &&
+         storageType === "gcs" &&
+         attachedDb.gcsConnection
+      ) {
+         return gcsConnectionToCredentials(attachedDb.gcsConnection);
+      }
+      if (
+         attachedDb.type === "s3" &&
+         storageType === "s3" &&
+         attachedDb.s3Connection
+      ) {
+         return s3ConnectionToCredentials(attachedDb.s3Connection);
+      }
+   }
+   return null;
+}
+
 export async function getSchemasForConnection(
    connection: ApiConnection,
    malloyConnection: Connection,
@@ -265,7 +297,7 @@ export async function getSchemasForConnection(
 
          const rows = standardizeRunSQLResult(result);
 
-         return rows.map((row: unknown) => {
+         const schemas: ApiSchema[] = rows.map((row: unknown) => {
             const typedRow = row as Record<string, unknown>;
             const schemaName = typedRow.schema_name as string;
             const catalogName = typedRow.catalog_name as string;
@@ -288,6 +320,42 @@ export async function getSchemasForConnection(
                isDefault: catalogName === "main",
             };
          });
+
+         const attachedDatabases =
+            connection.duckdbConnection.attachedDatabases || [];
+
+         for (const attachedDb of attachedDatabases) {
+            if (
+               (attachedDb.type === "gcs" || attachedDb.type === "s3") &&
+               (attachedDb.gcsConnection || attachedDb.s3Connection)
+            ) {
+               const credentials =
+                  attachedDb.type === "gcs"
+                     ? gcsConnectionToCredentials(attachedDb.gcsConnection!)
+                     : s3ConnectionToCredentials(attachedDb.s3Connection!);
+
+               try {
+                  const buckets = await listCloudBuckets(credentials);
+                  for (const bucket of buckets) {
+                     schemas.push({
+                        name: `${attachedDb.type}.${bucket.name}`,
+                        isHidden: false,
+                        isDefault: false,
+                     });
+                  }
+                  logger.info(
+                     `Listed ${buckets.length} ${attachedDb.type.toUpperCase()} buckets for attached database ${attachedDb.name}`,
+                  );
+               } catch (cloudError) {
+                  logger.warn(
+                     `Failed to list ${attachedDb.type.toUpperCase()} buckets for ${attachedDb.name}`,
+                     { error: cloudError },
+                  );
+               }
+            }
+         }
+
+         return schemas;
       } catch (error) {
          console.error(
             `Error getting schemas for DuckDB connection ${connection.name}:`,
@@ -347,6 +415,41 @@ export async function getTablesForSchema(
       malloyConnection,
    );
 
+   const catalogName = schemaName.split(".")[0];
+
+   if (
+      (catalogName === "gcs" || catalogName === "s3") &&
+      connection.type === "duckdb"
+   ) {
+      console.log(
+         `Getting ${catalogName.toUpperCase()} tables for schema`,
+         schemaName,
+      );
+      console.log("tableNames", tableNames);
+      const bucketName = schemaName.split(".")[1];
+      console.log("bucketName", bucketName);
+
+      const attachedDatabases =
+         connection.duckdbConnection?.attachedDatabases || [];
+      const credentials = getCloudCredentialsFromAttachedDatabases(
+         attachedDatabases,
+         catalogName as "gcs" | "s3",
+      );
+
+      if (!credentials) {
+         throw new Error(
+            `${catalogName.toUpperCase()} credentials not found in attached databases`,
+         );
+      }
+
+      return await getCloudTablesWithColumns(
+         malloyConnection,
+         credentials,
+         bucketName,
+         tableNames,
+      );
+   }
+
    // Fetch all table sources in parallel
    const tableSourcePromises = tableNames.map(async (tableName) => {
       try {
@@ -598,11 +701,48 @@ export async function listTablesForSchema(
       if (!connection.duckdbConnection) {
          throw new Error("DuckDB connection is required");
       }
+
+      const catalogName = schemaName.split(".")[0];
+      const actualSchemaName = schemaName.split(".")[1];
+
+      if (catalogName === "gcs" || catalogName === "s3") {
+         const bucketName = actualSchemaName;
+         const attachedDatabases =
+            connection.duckdbConnection.attachedDatabases || [];
+
+         const credentials = getCloudCredentialsFromAttachedDatabases(
+            attachedDatabases,
+            catalogName as "gcs" | "s3",
+         );
+
+         if (!credentials) {
+            throw new Error(
+               `${catalogName.toUpperCase()} credentials not found in attached databases`,
+            );
+         }
+
+         try {
+            const objects = await listAllCloudFiles(credentials, bucketName);
+            return objects
+               .filter((obj) => isDataFile(obj.key))
+               .map((obj) => obj.key);
+         } catch (error) {
+            logger.error(
+               `Error listing ${catalogName.toUpperCase()} objects in bucket ${bucketName}`,
+               {
+                  error,
+               },
+            );
+            throw new Error(
+               `Failed to list files in ${catalogName.toUpperCase()} bucket ${bucketName}: ${(error as Error).message}`,
+            );
+         }
+      }
+
+      // Regular DuckDB table listing
       try {
-         const catalogName = schemaName.split(".")[0];
-         schemaName = schemaName.split(".")[1];
          const result = await malloyConnection.runSQL(
-            `SELECT table_name FROM information_schema.tables WHERE table_schema = '${schemaName}' and table_catalog = '${catalogName}' ORDER BY table_name`,
+            `SELECT table_name FROM information_schema.tables WHERE table_schema = '${actualSchemaName}' and table_catalog = '${catalogName}' ORDER BY table_name`,
             { rowLimit: 1000 },
          );
 

package/src/service/gcs_s3_utils.ts

@@ -0,0 +1,304 @@
+import {
+   ListBucketsCommand,
+   ListObjectsV2Command,
+   S3Client,
+} from "@aws-sdk/client-s3";
+import { Connection } from "@malloydata/malloy";
+import { components } from "../api";
+import { logger } from "../logger";
+
+type ApiTable = components["schemas"]["Table"];
+
+export type CloudStorageType = "gcs" | "s3";
+
+export interface CloudStorageCredentials {
+   type: CloudStorageType;
+   accessKeyId: string;
+   secretAccessKey: string;
+   region?: string;
+   endpoint?: string;
+   sessionToken?: string;
+}
+
+export interface CloudStorageBucket {
+   name: string;
+   creationDate?: Date;
+}
+
+export interface CloudStorageObject {
+   key: string;
+   size?: number;
+   lastModified?: Date;
+   isFolder: boolean;
+}
+
+export function gcsConnectionToCredentials(gcsConnection: {
+   keyId?: string;
+   secret?: string;
+}): CloudStorageCredentials {
+   return {
+      type: "gcs",
+      accessKeyId: gcsConnection.keyId || "",
+      secretAccessKey: gcsConnection.secret || "",
+   };
+}
+
+export function s3ConnectionToCredentials(s3Connection: {
+   accessKeyId?: string;
+   secretAccessKey?: string;
+   region?: string;
+   endpoint?: string;
+   sessionToken?: string;
+}): CloudStorageCredentials {
+   return {
+      type: "s3",
+      accessKeyId: s3Connection.accessKeyId || "",
+      secretAccessKey: s3Connection.secretAccessKey || "",
+      region: s3Connection.region,
+      endpoint: s3Connection.endpoint,
+      sessionToken: s3Connection.sessionToken,
+   };
+}
+
+function createCloudStorageClient(
+   credentials: CloudStorageCredentials,
+): S3Client {
+   const isGCS = credentials.type === "gcs";
+
+   const client = new S3Client({
+      endpoint: isGCS ? "https://storage.googleapis.com" : credentials.endpoint,
+      region: isGCS ? "auto" : credentials.region || "us-east-1",
+      credentials: {
+         accessKeyId: credentials.accessKeyId,
+         secretAccessKey: credentials.secretAccessKey,
+         sessionToken: credentials.sessionToken,
+      },
+      forcePathStyle: isGCS || !!credentials.endpoint,
+   });
+
+   if (isGCS) {
+      client.middlewareStack.add(
+         (next) => async (args) => {
+            const request = args.request as { query?: Record<string, string> };
+            if (request.query) {
+               delete request.query["x-id"];
+            }
+            return next(args);
+         },
+         { step: "build", name: "removeXIdParam" },
+      );
+   }
+
+   return client;
+}
+
+export async function listCloudBuckets(
+   credentials: CloudStorageCredentials,
+): Promise<CloudStorageBucket[]> {
+   const client = createCloudStorageClient(credentials);
+   const storageType = credentials.type.toUpperCase();
+
+   try {
+      const response = await client.send(new ListBucketsCommand({}));
+      return (response.Buckets || []).map((bucket) => ({
+         name: bucket.Name || "",
+         creationDate: bucket.CreationDate,
+      }));
+   } catch (error) {
+      logger.error(`Failed to list ${storageType} buckets`, { error });
+      throw new Error(
+         `Failed to list ${storageType} buckets: ${error instanceof Error ? error.message : String(error)}`,
+      );
+   }
+}
+
+async function listCloudObjectsInFolder(
+   credentials: CloudStorageCredentials,
+   bucket: string,
+   prefix: string = "",
+): Promise<CloudStorageObject[]> {
+   const client = createCloudStorageClient(credentials);
+   const storageType = credentials.type.toUpperCase();
+   const uri = buildCloudUri(credentials.type, bucket, prefix);
+
+   try {
+      const response = await client.send(
+         new ListObjectsV2Command({
+            Bucket: bucket,
+            Prefix: prefix,
+            Delimiter: "/",
+         }),
+      );
+
+      const objects: CloudStorageObject[] = [];
+
+      for (const folderPrefix of response.CommonPrefixes || []) {
+         if (folderPrefix.Prefix) {
+            objects.push({
+               key: folderPrefix.Prefix,
+               isFolder: true,
+            });
+         }
+      }
+
+      for (const content of response.Contents || []) {
+         if (content.Key && content.Key !== prefix) {
+            objects.push({
+               key: content.Key,
+               size: content.Size,
+               lastModified: content.LastModified,
+               isFolder: false,
+            });
+         }
+      }
+
+      return objects;
+   } catch (error) {
+      logger.error(`Failed to list ${storageType} objects`, {
+         error,
+         bucket,
+         prefix,
+      });
+      throw new Error(
+         `Failed to list objects in ${uri}: ${error instanceof Error ? error.message : String(error)}`,
+      );
+   }
+}
+
+export async function listAllCloudFiles(
+   credentials: CloudStorageCredentials,
+   bucket: string,
+   prefix: string = "",
+): Promise<CloudStorageObject[]> {
+   const allFiles: CloudStorageObject[] = [];
+
+   async function traverse(currentPrefix: string): Promise<void> {
+      const objects = await listCloudObjectsInFolder(
+         credentials,
+         bucket,
+         currentPrefix,
+      );
+
+      for (const obj of objects) {
+         if (obj.isFolder) {
+            await traverse(obj.key);
+         } else {
+            allFiles.push(obj);
+         }
+      }
+   }
+
+   await traverse(prefix);
+   return allFiles;
+}
+
+export function isDataFile(key: string): boolean {
+   const lowerKey = key.toLowerCase();
+   return (
+      lowerKey.endsWith(".csv") ||
+      lowerKey.endsWith(".parquet") ||
+      lowerKey.endsWith(".json") ||
+      lowerKey.endsWith(".jsonl") ||
+      lowerKey.endsWith(".ndjson")
+   );
+}
+
+export function getFileType(key: string): string {
+   const lowerKey = key.toLowerCase();
+   if (lowerKey.endsWith(".csv")) return "csv";
+   if (lowerKey.endsWith(".parquet")) return "parquet";
+   if (lowerKey.endsWith(".json")) return "json";
+   if (lowerKey.endsWith(".jsonl") || lowerKey.endsWith(".ndjson"))
+      return "jsonl";
+   return "unknown";
+}
+
+export function buildCloudUri(
+   type: CloudStorageType,
+   bucket: string,
+   key: string,
+): string {
+   const scheme = type === "gcs" ? "gs" : "s3";
+   return `${scheme}://${bucket}/${key}`;
+}
+
+function standardizeRunSQLResult(result: unknown): unknown[] {
+   return Array.isArray(result)
+      ? result
+      : (result as { rows?: unknown[] }).rows || [];
+}
+
+export async function getCloudTablesWithColumns(
+   malloyConnection: Connection,
+   credentials: CloudStorageCredentials,
+   bucketName: string,
+   fileKeys: string[],
+): Promise<ApiTable[]> {
+   const tables: ApiTable[] = [];
+
+   for (const fileKey of fileKeys) {
+      const uri = buildCloudUri(credentials.type, bucketName, fileKey);
+      const fileType = getFileType(fileKey);
+
+      try {
+         let describeQuery: string;
+
+         switch (fileType) {
+            case "csv":
+               describeQuery = `DESCRIBE SELECT * FROM read_csv('${uri}', auto_detect=true) LIMIT 1`;
+               break;
+            case "parquet":
+               describeQuery = `DESCRIBE SELECT * FROM read_parquet('${uri}') LIMIT 1`;
+               break;
+            case "json":
+               describeQuery = `DESCRIBE SELECT * FROM read_json('${uri}', auto_detect=true) LIMIT 1`;
+               break;
+            case "jsonl":
+               describeQuery = `DESCRIBE SELECT * FROM read_json('${uri}', format='newline_delimited', auto_detect=true) LIMIT 1`;
+               break;
+            default:
+               logger.warn(`Unsupported file type for ${fileKey}`);
+               tables.push({
+                  resource: uri,
+                  columns: [],
+               });
+               continue;
+         }
+
+         const result = await malloyConnection.runSQL(describeQuery);
+         const rows = standardizeRunSQLResult(result);
+         const columns = rows.map((row: unknown) => {
+            const typedRow = row as Record<string, unknown>;
+            return {
+               name: (typedRow.column_name || typedRow.name) as string,
+               type: (typedRow.column_type || typedRow.type) as string,
+            };
+         });
+
+         tables.push({
+            resource: uri,
+            columns,
+         });
+
+         logger.info(
+            `Got schema for ${credentials.type.toUpperCase()} file: ${uri}`,
+            {
+               columnCount: columns.length,
+            },
+         );
+      } catch (error) {
+         logger.warn(
+            `Failed to get schema for ${credentials.type.toUpperCase()} file: ${uri}`,
+            {
+               error,
+            },
+         );
+         tables.push({
+            resource: uri,
+            columns: [],
+         });
+      }
+   }
+
+   return tables;
+}