@malloy-publisher/server 0.10.4 → 0.10.5

package/dist/app/api-doc.yaml

@@ -6685,8 +6685,11 @@ components:
             `sqlSource` answer 400 for anything else (another statement type, or
             more than one statement), and `sqlTemporaryTable` answers 400 always.
             This is a rule on statements, not on what a SELECT reads: that is
-            bounded by the session's storage secret and catalog role. Absent or
-            false is the pre-existing behavior.
+            bounded by the catalog role and by the session's storage secret,
+            which on a managed connection attached read-only is scoped to the
+            `main` schema's recorded path, so a SELECT reads only that schema's
+            files; another schema's files, and any other object in the bucket,
+            are unsigned (403). Absent or false is the pre-existing behavior.
         fingerprint:
           type: string
           description: >
@@ -7157,7 +7160,11 @@ components:
                 every attach uses the data path the catalog recorded at its
                 creation, and no attach ever creates a catalog: the shape for a
                 catalog created elsewhere, through the admin API
-                (admin-api-doc.yaml) or by another tool.
+                (admin-api-doc.yaml) or by another tool. Either way the storage
+                secret is scoped to the data path, the given one or the
+                recorded one: a read of any other object through this
+                connection is unsigned, and on S3 the host's AWS_* environment
+                never signs it either. A catalog with neither is not attached.
             s3Connection:
               $ref: "#/components/schemas/S3Connection"
               description: AWS S3 connection configuration for data storage

package/dist/instrumentation.mjs

@@ -164831,7 +164831,7 @@ var init_telemetry = __esm(() => {
 });
 
 // package.json
-var version = "0.10.4";
+var version = "0.10.5";
 var init_package = () => {};
 
 // src/version.ts

package/dist/server.mjs

@@ -164832,7 +164832,7 @@ var init_telemetry = __esm(() => {
 });
 
 // package.json
-var version = "0.10.4";
+var version = "0.10.5";
 var init_package = () => {};
 
 // src/version.ts
@@ -268349,7 +268349,7 @@ var require_bcrypt_pbkdf = __commonJS((exports, module) => {
 
 // ../../node_modules/cpu-features/build/Release/cpufeatures.node
 var require_cpufeatures = __commonJS((exports, module) => {
-  module.exports = __require("./cpufeatures-1yrn0vtw.node");
+  module.exports = __require("./cpufeatures-gknxfsk5.node");
 });
 
 // ../../node_modules/cpu-features/lib/index.js
@@ -268960,6 +268960,11 @@ var require_utils93 = __commonJS((exports, module) => {
   };
 });
 
+// ../../node_modules/ssh2/lib/protocol/crypto/build/Release/sshcrypto.node
+var require_sshcrypto = __commonJS((exports, module) => {
+  module.exports = __require("./sshcrypto-a4xyg5e8.node");
+});
+
 // ../../node_modules/ssh2/lib/protocol/crypto/poly1305.js
 var require_poly1305 = __commonJS((exports, module) => {
   var __dirname = "/home/runner/work/publisher/publisher/node_modules/ssh2/lib/protocol/crypto", __filename = "/home/runner/work/publisher/publisher/node_modules/ssh2/lib/protocol/crypto/poly1305.js";
@@ -269446,7 +269451,7 @@ var require_crypto = __commonJS((exports, module) => {
   var ChaChaPolyDecipher;
   var GenericDecipher;
   try {
-    binding = (()=>{throw new Error("Cannot require module "+"./crypto/build/Release/sshcrypto.node");})();
+    binding = require_sshcrypto();
     ({
       AESGCMCipher,
       ChaChaPolyCipher,
@@ -283780,7 +283785,7 @@ function validateMotherDuckShape(connection) {
     throw new Error("MotherDuck access token is required.");
   }
 }
-function validateDuckLakeShape(connection) {
+function validateDuckLakeShape(connection, origin2) {
   if (!connection.ducklakeConnection) {
     throw new Error("DuckLake connection configuration is missing.");
   }
@@ -283789,7 +283794,7 @@ function validateDuckLakeShape(connection) {
   }
   const storage = connection.ducklakeConnection.storage;
   if (storage?.s3Connection) {
-    validateS3ProviderShape(storage.s3Connection, connection.name);
+    validateS3ProviderShape(storage.s3Connection, connection.name, origin2);
   }
   if (connection.ducklakeConnection.catalog.metadataSchema !== undefined) {
     const schema = connection.ducklakeConnection.catalog.metadataSchema;
@@ -283866,7 +283871,7 @@ function validatePublisherShape(connection) {
     throw new Error(`Invalid publisher connection '${connection.name}': connectionUri must use http or https (got '${parsedUri.protocol}'). ` + `Fix: set connectionUri to an absolute https URL like "https://…/connections/${connection.name}".`);
   }
 }
-function validateTypeShape(connection) {
+function validateTypeShape(connection, origin2) {
   switch (connection.type) {
     case "sqlserver":
       return validateSqlServerShape(connection);
@@ -283877,7 +283882,7 @@ function validateTypeShape(connection) {
     case "motherduck":
       return validateMotherDuckShape(connection);
     case "ducklake":
-      return validateDuckLakeShape(connection);
+      return validateDuckLakeShape(connection, origin2);
     case "trino":
       return validateTrinoShape(connection);
     case "databricks":
@@ -283888,7 +283893,7 @@ function validateTypeShape(connection) {
       return validatePublisherShape(connection);
   }
 }
-function validateConnectionShape(connection) {
+function validateConnectionShape(connection, origin2 = "connection") {
   if (connection.proxy) {
     validateProxyShape(connection, connection.proxy);
   }
@@ -283901,7 +283906,7 @@ function validateConnectionShape(connection) {
   if (connection.postgresConnection) {
     postgresStatementTimeoutMs(connection.name ?? "", connection.postgresConnection);
   }
-  validateTypeShape(connection);
+  validateTypeShape(connection, origin2);
 }
 function storageDestinationRoot(environmentPath) {
   return path4.join(environmentPath, STORAGE_DESTINATIONS_DIR);
@@ -283950,7 +283955,7 @@ function validateStorageDestinations(destinations = []) {
       continue;
     }
     try {
-      validateConnectionShape(destination);
+      validateConnectionShape(destination, "storage-destination");
       const storage = destination.ducklakeConnection?.storage;
       if (storage?.s3Connection) {
         resolveCloudStorageCredentials({
@@ -284168,7 +284173,7 @@ function buildCoreEntry(connection, environmentPath) {
       throw new Error(`Unsupported connection type: ${connection.type}`);
   }
 }
-function assembleEnvironmentConnections(connections = [], environmentPath = "") {
+function assembleEnvironmentConnections(connections = [], environmentPath = "", origin2 = "connection") {
   const pojo = { connections: {} };
   const metadata = new Map;
   const apiConnections = [];
@@ -284185,7 +284190,7 @@ function assembleEnvironmentConnections(connections = [], environmentPath = "")
     }
     processedConnections.add(connection.name);
     validateDuckdbApiSurface(connection);
-    validateConnectionShape(connection);
+    validateConnectionShape(connection, origin2);
     warnOnConnectionQueryMetadata(connection);
     const apiConnection = cloneApiConnection(connection);
     apiConnection.attributes = getStaticConnectionAttributes(connection.type);
@@ -295166,17 +295171,19 @@ async function attachDuckLakeWithMode(connection, dbName, ducklakeConfig, option
   const hasS3 = !!ducklakeConfig.storage?.s3Connection;
   const hasGCS = !!ducklakeConfig.storage?.gcsConnection;
   if (hasS3) {
-    await attachCloudStorage(connection, {
-      name: `${dbName}_storage`,
-      type: "s3",
-      s3Connection: ducklakeConfig.storage.s3Connection
-    });
-  } else if (hasGCS) {
-    await attachCloudStorage(connection, {
-      name: `${dbName}_storage`,
-      type: "gcs",
-      gcsConnection: ducklakeConfig.storage.gcsConnection
-    });
+    await connection.runSQL("SET GLOBAL enable_global_s3_configuration=false");
+  }
+  const storage = hasS3 ? {
+    name: `${dbName}_storage`,
+    type: "s3",
+    s3Connection: ducklakeConfig.storage.s3Connection
+  } : hasGCS ? {
+    name: `${dbName}_storage`,
+    type: "gcs",
+    gcsConnection: ducklakeConfig.storage.gcsConnection
+  } : undefined;
+  if (storage) {
+    resolveCloudStorageCredentials(storage, options.origin ?? "connection");
   }
   const pg = ducklakeConfig.catalog.postgresConnection;
   const pgConnString = buildPgConnectionString(pg, {
@@ -295189,8 +295196,8 @@ async function attachDuckLakeWithMode(connection, dbName, ducklakeConfig, option
   logger.debug(`Final escaped connection string: ${redactPgSecrets(escapedPgConnString)}`);
   const dataPath = ducklakeConfig.storage?.bucketUrl;
   const metadataSchema = ducklakeConfig.catalog.metadataSchema;
-  const presence = await preflightDuckLakeCatalogFormat(connection, dbName, pgConnString, metadataSchema);
-  if (presence === "absent") {
+  const preflight = await preflightDuckLakeCatalogFormat(connection, dbName, pgConnString, metadataSchema);
+  if (preflight === "absent") {
     if (options.skipIfAbsent) {
       return false;
     }
@@ -295215,14 +295222,44 @@ async function attachDuckLakeWithMode(connection, dbName, ducklakeConfig, option
       throw error;
     }
   }
+  if (storage) {
+    try {
+      const scope = await attachedDuckLakeScope(connection, dbName, {
+        configuredDataPath: dataPath,
+        metadataSchema,
+        readOnly: options.readOnly,
+        managed: options.managed === true && options.origin !== "storage-destination"
+      });
+      if (!scope) {
+        throw new Error(`No storage secret could be scoped for DuckLake catalog '${dbName}': ` + `the config states no data path, and the catalog records no data ` + `path (or, for a managed view, no path for its main schema) to ` + `scope it to. The catalog was not attached.`);
+      }
+      await attachCloudStorage(connection, storage, options.origin, scope);
+    } catch (error) {
+      await connection.runSQL(`DETACH ${quoteIdentifier(dbName, "duckdb")};`).catch(() => {});
+      throw error;
+    }
+  }
   return true;
 }
-async function recordedCatalogSettings(connection, alias, metadataSchema) {
-  const metadataCatalog = [
+async function attachedDuckLakeScope(connection, dbName, input) {
+  const recordedDataPath = input.configuredDataPath ? undefined : (await recordedCatalogSettings(connection, dbName, input.metadataSchema)).data_path;
+  const mainSchemaPath = input.managed && input.readOnly ? await recordedSchemaPath(connection, dbName, input.metadataSchema, "main") : undefined;
+  return storageSecretScope({
+    configuredDataPath: input.configuredDataPath,
+    recordedDataPath,
+    mainSchemaPath,
+    readOnly: input.readOnly,
+    managed: input.managed
+  });
+}
+function metadataCatalogRef(alias, metadataSchema) {
+  return [
     `__ducklake_metadata_${alias}`,
     ...metadataSchema ? [metadataSchema] : []
   ].map((part) => quoteIdentifier(part, "duckdb")).join(".");
-  const rows = runSQLRows(await connection.runSQL(`SELECT key, value FROM ${metadataCatalog}.ducklake_metadata WHERE scope IS NULL`));
+}
+async function recordedCatalogSettings(connection, alias, metadataSchema) {
+  const rows = runSQLRows(await connection.runSQL(`SELECT key, value FROM ${metadataCatalogRef(alias, metadataSchema)}.ducklake_metadata WHERE scope IS NULL`));
   const settings = {};
   for (const row of rows) {
     if (typeof row.key === "string" && row.value != null) {
@@ -295231,6 +295268,43 @@ async function recordedCatalogSettings(connection, alias, metadataSchema) {
   }
   return settings;
 }
+async function recordedSchemaPath(connection, alias, metadataSchema, schemaName) {
+  const row = runSQLRows(await connection.runSQL(`SELECT path, path_is_relative FROM ${metadataCatalogRef(alias, metadataSchema)}.ducklake_schema ` + `WHERE schema_name = '${escapeSQL(schemaName)}' AND end_snapshot IS NULL LIMIT 1`))[0];
+  if (!row || typeof row.path !== "string")
+    return;
+  return { path: row.path, relative: row.path_is_relative !== false };
+}
+function stripTrailingSlashes(url3) {
+  let end = url3.length;
+  while (end > 0 && url3[end - 1] === "/")
+    end--;
+  return url3.slice(0, end);
+}
+function oneTrailingSlash(url3) {
+  return `${stripTrailingSlashes(url3)}/`;
+}
+function isBucketlessUrl(url3) {
+  const at = url3.indexOf("://");
+  return at >= 0 && stripTrailingSlashes(url3.slice(at + 3)) === "";
+}
+function storageSecretScope(input) {
+  const root = input.configuredDataPath || input.recordedDataPath;
+  if (!root || isBucketlessUrl(root))
+    return;
+  if (input.managed && input.readOnly) {
+    const main = input.mainSchemaPath;
+    if (!main)
+      return;
+    if (!main.relative) {
+      return isBucketlessUrl(main.path) ? undefined : oneTrailingSlash(main.path);
+    }
+    let start = 0;
+    while (start < main.path.length && main.path[start] === "/")
+      start++;
+    return oneTrailingSlash(`${oneTrailingSlash(root)}${main.path.slice(start)}`);
+  }
+  return oneTrailingSlash(root);
+}
 function dataPathClauses(dataPath, readOnly) {
   if (dataPath) {
     return `DATA_PATH '${escapeSQL(dataPath)}', OVERRIDE_DATA_PATH true`;
@@ -295250,13 +295324,16 @@ function firstAttachClauses(options) {
 async function attachDuckLakeForCreate(connection, dbName, ducklakeConfig, settings) {
   await attachDuckLakeWithMode(connection, dbName, ducklakeConfig, {
     readOnly: false,
+    origin: "storage-destination",
     encrypted: settings.encrypted,
     dataInliningRowLimit: settings.dataInliningRowLimit
   });
 }
-async function attachDuckLake(connection, dbName, ducklakeConfig) {
+async function attachDuckLake(connection, dbName, ducklakeConfig, options = {}) {
   await attachDuckLakeWithMode(connection, dbName, ducklakeConfig, {
-    readOnly: ducklakeConfig.readOnly !== false
+    readOnly: ducklakeConfig.readOnly !== false,
+    origin: options.origin,
+    managed: options.managed
   });
 }
 function managedSearchPathSql(dbName) {
@@ -295267,17 +295344,21 @@ async function setManagedSearchPath(connection, dbName) {
   await connection.runSQL(managedSearchPathSql(dbName));
 }
 async function attachDuckLakeReadOnly(connection, dbName, ducklakeConfig) {
-  await attachDuckLake(connection, dbName, ducklakeConfig);
+  await attachDuckLake(connection, dbName, ducklakeConfig, {
+    origin: "storage-destination"
+  });
 }
 async function attachDuckLakeReadOnlyIfPresent(connection, dbName, ducklakeConfig) {
   return attachDuckLakeWithMode(connection, dbName, ducklakeConfig, {
     readOnly: true,
-    skipIfAbsent: true
+    skipIfAbsent: true,
+    origin: "storage-destination"
   });
 }
 async function attachDuckLakeReadWrite(connection, dbName, ducklakeConfig) {
   await attachDuckLakeWithMode(connection, dbName, ducklakeConfig, {
-    readOnly: false
+    readOnly: false,
+    origin: "storage-destination"
   });
 }
 async function federateSourceForPassthrough(connection, sourceType, config, deps = {}) {
@@ -295481,13 +295562,15 @@ function buildCloudStorageSecretSQL(secretName, credentials) {
   const isGCS = credentials.type === "gcs";
   const escapedKeyId = escapeSQL(credentials.accessKeyId);
   const escapedSecret = escapeSQL(credentials.secretAccessKey);
+  const scopeClause = credentials.scope ? `SCOPE '${escapeSQL(credentials.scope)}'` : "";
   let createSecretCommand;
   if (isGCS) {
     createSecretCommand = `
          CREATE OR REPLACE SECRET ${secretName} (
             TYPE gcs,
             KEY_ID '${escapedKeyId}',
-            SECRET '${escapedSecret}'
+            SECRET '${escapedSecret}'${scopeClause ? `,
+            ${scopeClause}` : ""}
          );
       `;
   } else {
@@ -295523,6 +295606,9 @@ function buildCloudStorageSecretSQL(secretName, credentials) {
     if (credentials.provider !== "credential_chain" && credentials.sessionToken) {
       clauses.push(`SESSION_TOKEN '${escapeSQL(credentials.sessionToken)}'`);
     }
+    if (scopeClause) {
+      clauses.push(scopeClause);
+    }
     createSecretCommand = `
             CREATE OR REPLACE SECRET ${secretName} (
                ${clauses.join(`,
@@ -295532,9 +295618,12 @@ function buildCloudStorageSecretSQL(secretName, credentials) {
   }
   return createSecretCommand;
 }
-async function attachCloudStorage(connection, attachedDb) {
+async function attachCloudStorage(connection, attachedDb, origin2 = "connection", scope) {
   const storageType = attachedDb.type?.toUpperCase() || "";
-  const credentials = resolveCloudStorageCredentials(attachedDb);
+  const credentials = {
+    ...resolveCloudStorageCredentials(attachedDb, origin2),
+    scope
+  };
   await installAndLoadExtension(connection, "httpfs");
   if (credentials.provider === "credential_chain") {
     await installAndLoadExtension(connection, "aws");
@@ -295826,8 +295915,8 @@ function applyConnectionFingerprint(connection, metadata) {
 function isDuckDBConnection(connection) {
   return connection instanceof DuckDBConnection;
 }
-function buildEnvironmentMalloyConfig(connections = [], environmentPath = "", isUpdateConnectionRequest = false) {
-  const assembled = assembleEnvironmentConnections(connections, environmentPath);
+function buildEnvironmentMalloyConfig(connections = [], environmentPath = "", isUpdateConnectionRequest = false, { origin: origin2 = "connection" } = {}) {
+  const assembled = assembleEnvironmentConnections(connections, environmentPath, origin2);
   const duckLakeCache = new Map;
   const snowflakeJwtCache = new Map;
   const azureDuckDBCache = new Map;
@@ -295866,7 +295955,10 @@ function buildEnvironmentMalloyConfig(connections = [], environmentPath = "", is
         }
         const connection2 = await connectionPromise;
         if (isUpdateConnectionRequest || !await isDatabaseAttached(connection2, name)) {
-          await attachDuckLake(connection2, name, metadata.apiConnection.ducklakeConnection);
+          await attachDuckLake(connection2, name, metadata.apiConnection.ducklakeConnection, {
+            origin: origin2,
+            managed: metadata.apiConnection.managed === true
+          });
           if (metadata.apiConnection.managed === true) {
             await setManagedSearchPath(connection2, name);
           }
@@ -370400,10 +370492,10 @@ class Environment {
       if (this.destinations.length > 0) {
         fs17.mkdirSync(destinationRoot, { recursive: true });
       }
-      this.destinationMalloyConfig = buildEnvironmentMalloyConfig(this.destinations, destinationRoot);
+      this.destinationMalloyConfig = buildEnvironmentMalloyConfig(this.destinations, destinationRoot, false, { origin: "storage-destination" });
     } catch (error) {
       logger.error(`Failed to assemble storage destinations for environment ${this.environmentName}; serving without them`, { error });
-      this.destinationMalloyConfig = buildEnvironmentMalloyConfig([], storageDestinationRoot(this.environmentPath));
+      this.destinationMalloyConfig = buildEnvironmentMalloyConfig([], storageDestinationRoot(this.environmentPath), false, { origin: "storage-destination" });
     }
     if (previous && previous !== this.destinationMalloyConfig) {
       this.retireConnectionGeneration(`environment ${this.environmentName} destinations`, () => previous.releaseConnections());
@@ -377082,7 +377174,7 @@ function readerSql(uri, format, options = {}) {
   return `read_parquet(${path25})`;
 }
 function assertUnderDataPath(uri, dataPath) {
-  const root = dataPath.replace(/\/+$/, "");
+  const root = stripTrailingSlashes(dataPath);
   const outside = () => new BadRequestError(`The object must sit under the catalog's data path (${root}/).`);
   if (!root || !uri.startsWith(`${root}/`))
     throw outside();

package/package.json

@@ -19,7 +19,7 @@
       "snowflake"
    ],
    "homepage": "https://docs.malloydata.dev/documentation/user_guides/publishing/mcp_agents",
-   "version": "0.10.4",
+   "version": "0.10.5",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"