duckdb 1.3.1-dev6.0 → 1.3.1

package/src/duckdb/src/include/duckdb/function/function_serialization.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp"
 #include "duckdb/common/serializer/serializer.hpp"
 #include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/function/function_binder.hpp"
 
 namespace duckdb {
 
@@ -39,8 +40,8 @@ public:
 
 	template <class FUNC, class CATALOG_ENTRY>
 	static FUNC DeserializeFunction(ClientContext &context, CatalogType catalog_type, const string &catalog_name,
-	                                const string &schema_name, const string &name, vector<LogicalType> arguments,
-	                                vector<LogicalType> original_arguments) {
+	                                const string &schema_name, const string &name, const vector<LogicalType> &arguments,
+	                                const vector<LogicalType> &original_arguments) {
 		EntryLookupInfo lookup_info(catalog_type, name);
 		auto &func_catalog =
 		    Catalog::GetEntry(context, catalog_type, catalog_name.empty() ? SYSTEM_CATALOG : catalog_name,
@@ -52,8 +53,6 @@ public:
 		auto &functions = func_catalog.Cast<CATALOG_ENTRY>();
 		auto function = functions.functions.GetFunctionByArguments(
 		    context, original_arguments.empty() ? arguments : original_arguments);
-		function.arguments = std::move(arguments);
-		function.original_arguments = std::move(original_arguments);
 		return function;
 	}
 
@@ -72,8 +71,12 @@ public:
 			schema_name = DEFAULT_SCHEMA;
 		}
 		auto function = DeserializeFunction<FUNC, CATALOG_ENTRY>(context, catalog_type, catalog_name, schema_name, name,
-		                                                         std::move(arguments), std::move(original_arguments));
+		                                                         arguments, original_arguments);
 		auto has_serialize = deserializer.ReadProperty<bool>(503, "has_serialize");
+		if (has_serialize) {
+			function.arguments = std::move(arguments);
+			function.original_arguments = std::move(original_arguments);
+		}
 		return make_pair(std::move(function), has_serialize);
 	}
 
@@ -139,15 +142,20 @@ public:
 			deserializer.Set<const LogicalType &>(return_type);
 			bind_data = FunctionDeserialize<FUNC>(deserializer, function);
 			deserializer.Unset<LogicalType>();
-		} else if (function.bind) {
-			try {
-				bind_data = function.bind(context, function, children);
-			} catch (std::exception &ex) {
-				ErrorData error(ex);
-				throw SerializationException("Error during bind of function in deserialization: %s",
-				                             error.RawMessage());
+		} else {
+			if (function.bind) {
+				try {
+					bind_data = function.bind(context, function, children);
+				} catch (std::exception &ex) {
+					ErrorData error(ex);
+					throw SerializationException("Error during bind of function in deserialization: %s",
+					                             error.RawMessage());
+				}
 			}
+			FunctionBinder binder(context);
+			binder.CastToFunctionArguments(function, children);
 		}
+
 		if (TypeRequiresAssignment(function.return_type)) {
 			function.return_type = std::move(return_type);
 		}

package/src/duckdb/src/include/duckdb/function/lambda_functions.hpp

@@ -53,10 +53,11 @@ public:
 
 class LambdaFunctions {
 public:
+	//! Returns the list child type
+	static LogicalType DetermineListChildType(const LogicalType &child_type);
+
 	//! Returns the parameter type for binary lambdas
-	static LogicalType BindBinaryLambda(const idx_t parameter_idx, const LogicalType &list_child_type);
-	//! Returns the parameter type for ternary lambdas
-	static LogicalType BindTernaryLambda(const idx_t parameter_idx, const LogicalType &list_child_type);
+	static LogicalType BindBinaryChildren(const vector<LogicalType> &function_child_types, const idx_t parameter_idx);
 
 	//! Checks for NULL list parameter and prepared statements and adds bound cast expression
 	static unique_ptr<FunctionData> ListLambdaPrepareBind(vector<unique_ptr<Expression>> &arguments,

package/src/duckdb/src/include/duckdb/function/scalar_function.hpp

@@ -109,7 +109,9 @@ typedef unique_ptr<FunctionLocalState> (*init_local_state_t)(ExpressionState &st
 //! The type to propagate statistics for this scalar function
 typedef unique_ptr<BaseStatistics> (*function_statistics_t)(ClientContext &context, FunctionStatisticsInput &input);
 //! The type to bind lambda-specific parameter types
-typedef LogicalType (*bind_lambda_function_t)(const idx_t parameter_idx, const LogicalType &list_child_type);
+typedef LogicalType (*bind_lambda_function_t)(ClientContext &context, const vector<LogicalType> &function_child_types,
+                                              idx_t parameter_idx);
+
 //! The type to bind lambda-specific parameter types
 typedef void (*get_modified_databases_t)(ClientContext &context, FunctionModifiedDatabasesInput &input);
 

package/src/duckdb/src/include/duckdb/logging/log_type.hpp

@@ -9,11 +9,14 @@
 #pragma once
 
 #include "duckdb/logging/logging.hpp"
+#include "duckdb/common/string_util.hpp"
 
 namespace duckdb {
+
 struct FileHandle;
 struct BaseRequest;
 struct HTTPResponse;
+class PhysicalOperator;
 
 //! Log types provide some structure to the formats that the different log messages can have
 //! For now, this holds a type that the VARCHAR value will be auto-cast into.
@@ -88,4 +91,18 @@ public:
 	}
 };
 
+class PhysicalOperatorLogType : public LogType {
+public:
+	static constexpr const char *NAME = "PhysicalOperator";
+	static constexpr LogLevel LEVEL = LogLevel::LOG_DEBUG;
+
+	//! Construct the log type
+	PhysicalOperatorLogType();
+
+	static LogicalType GetLogType();
+
+	static string ConstructLogMessage(const PhysicalOperator &op, const string &class_p, const string &event,
+	                                  const vector<pair<string, string>> &info);
+};
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/attached_database.hpp

@@ -64,6 +64,7 @@ public:
 
 	//! Initializes the catalog and storage of the attached database.
 	void Initialize(optional_ptr<ClientContext> context = nullptr, StorageOptions options = StorageOptions());
+	void FinalizeLoad(optional_ptr<ClientContext> context);
 	void Close();
 
 	Catalog &ParentCatalog() override;

package/src/duckdb/src/include/duckdb/main/client_properties.hpp

@@ -8,8 +8,6 @@
 
 #pragma once
 
-#include <utility>
-
 #include "duckdb/common/string.hpp"
 #include "duckdb/common/types.hpp"
 
@@ -17,14 +15,31 @@ namespace duckdb {
 
 enum class ArrowOffsetSize : uint8_t { REGULAR, LARGE };
 
+enum ArrowFormatVersion : uint8_t {
+	//! Base Version
+	V1_0 = 10,
+	//! Added 256-bit Decimal type.
+	V1_1 = 11,
+	//! Added MonthDayNano interval type.
+	V1_2 = 12,
+	//! Added Run-End Encoded Layout.
+	V1_3 = 13,
+	//! Added Variable-size Binary View Layout and the associated BinaryView and Utf8View types.
+	//! Added ListView Layout and the associated ListView and LargeListView types. Added Variadic buffers.
+	V1_4 = 14,
+	//! Expanded Decimal type bit widths to allow 32-bit and 64-bit types.
+	V1_5 = 15
+};
+
 //! A set of properties from the client context that can be used to interpret the query result
 struct ClientProperties {
-	ClientProperties(string time_zone_p, ArrowOffsetSize arrow_offset_size_p, bool arrow_use_list_view_p,
-	                 bool produce_arrow_string_view_p, bool lossless_conversion,
-	                 optional_ptr<ClientContext> client_context)
+	ClientProperties(string time_zone_p, const ArrowOffsetSize arrow_offset_size_p, const bool arrow_use_list_view_p,
+	                 const bool produce_arrow_string_view_p, const bool lossless_conversion,
+	                 const ArrowFormatVersion arrow_output_version, const optional_ptr<ClientContext> client_context)
 	    : time_zone(std::move(time_zone_p)), arrow_offset_size(arrow_offset_size_p),
 	      arrow_use_list_view(arrow_use_list_view_p), produce_arrow_string_view(produce_arrow_string_view_p),
-	      arrow_lossless_conversion(lossless_conversion), client_context(client_context) {
+	      arrow_lossless_conversion(lossless_conversion), arrow_output_version(arrow_output_version),
+	      client_context(client_context) {
 	}
 
 	string time_zone = "UTC";
@@ -32,6 +47,7 @@ struct ClientProperties {
 	bool arrow_use_list_view = false;
 	bool produce_arrow_string_view = false;
 	bool arrow_lossless_conversion = false;
+	ArrowFormatVersion arrow_output_version = V1_0;
 	optional_ptr<ClientContext> client_context;
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -296,6 +296,8 @@ struct DBConfigOptions {
 	LogConfig log_config = LogConfig();
 	//! Whether to enable external file caching using CachingFileSystem
 	bool enable_external_file_cache = true;
+	//! Output version of arrow depending on the format version
+	ArrowFormatVersion arrow_output_version = V1_0;
 	//! Partially process tasks before rescheduling - allows for more scheduler fairness between separate queries
 #ifdef DUCKDB_ALTERNATIVE_VERIFY
 	bool scheduler_process_partial = true;

package/src/duckdb/src/include/duckdb/main/database_manager.hpp

@@ -43,6 +43,8 @@ public:
 
 	//! Initializes the system catalog of the attached SYSTEM_DATABASE.
 	void InitializeSystemCatalog();
+	//! Finalize starting up the system
+	void FinalizeStartup();
 	//! Get an attached database by its name
 	optional_ptr<AttachedDatabase> GetDatabase(ClientContext &context, const string &name);
 	//! Attach a new database
@@ -66,7 +68,8 @@ public:
 	void GetDatabaseType(ClientContext &context, AttachInfo &info, const DBConfig &config, AttachOptions &options);
 	//! Scans the catalog set and adds each committed database entry, and each database entry of the current
 	//! transaction, to a vector holding AttachedDatabase references
-	vector<reference<AttachedDatabase>> GetDatabases(ClientContext &context);
+	vector<reference<AttachedDatabase>> GetDatabases(ClientContext &context,
+	                                                 const optional_idx max_db_count = optional_idx());
 	//! Scans the catalog set and returns each committed database entry
 	vector<reference<AttachedDatabase>> GetDatabases();
 	//! Removes all databases from the catalog set. This is necessary for the database instance's destructor,

package/src/duckdb/src/include/duckdb/main/extension_entries.hpp

@@ -150,6 +150,20 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"delta_scan", "delta", CatalogType::TABLE_FUNCTION_ENTRY},
     {"drop_fts_index", "fts", CatalogType::PRAGMA_FUNCTION_ENTRY},
     {"dsdgen", "tpcds", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"duckdb_proj_compiled_version", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"duckdb_proj_version", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"ducklake_add_data_files", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"ducklake_cleanup_old_files", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"ducklake_expire_snapshots", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"ducklake_list_files", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"ducklake_merge_adjacent_files", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"ducklake_options", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"ducklake_set_option", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"ducklake_snapshots", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"ducklake_table_changes", "ducklake", CatalogType::TABLE_MACRO_ENTRY},
+    {"ducklake_table_deletions", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"ducklake_table_info", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"ducklake_table_insertions", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY},
     {"editdist3", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"element_at", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"encode", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -538,6 +552,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"st_assvg", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_astext", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_aswkb", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"st_azimuth", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_boundary", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_buffer", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_buildarea", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -561,11 +576,13 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"st_dimension", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_disjoint", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_distance", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"st_distance_geos", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_distance_sphere", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_distance_spheroid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_drivers", "spatial", CatalogType::TABLE_FUNCTION_ENTRY},
     {"st_dump", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_dwithin", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"st_dwithin_geos", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_dwithin_spheroid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_endpoint", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_envelope", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -612,6 +629,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"st_makepolygon", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_makevalid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_maximuminscribedcircle", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"st_memunion_agg", "spatial", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"st_minimumrotatedrectangle", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_mmax", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_mmin", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -654,6 +672,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"st_simplify", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_simplifypreservetopology", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_startpoint", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"st_tileenvelope", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_touches", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_transform", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_translate", "spatial", CatalogType::MACRO_ENTRY},
@@ -1025,21 +1044,15 @@ static constexpr ExtensionEntry EXTENSION_SETTINGS[] = {
 }; // END_OF_EXTENSION_SETTINGS
 
 static constexpr ExtensionEntry EXTENSION_SECRET_TYPES[] = {
-    {"aws", "httpfs"},
-    {"azure", "azure"},
-    {"gcs", "httpfs"},
-    {"huggingface", "httpfs"},
-    {"iceberg", "iceberg"},
-    {"mysql", "mysql_scanner"},
-    {"postgres", "postgres_scanner"},
-    {"r2", "httpfs"},
-    {"s3", "httpfs"},
+    {"aws", "httpfs"},         {"azure", "azure"},     {"ducklake", "ducklake"},   {"gcs", "httpfs"},
+    {"huggingface", "httpfs"}, {"iceberg", "iceberg"}, {"mysql", "mysql_scanner"}, {"postgres", "postgres_scanner"},
+    {"r2", "httpfs"},          {"s3", "httpfs"},
 }; // END_OF_EXTENSION_SECRET_TYPES
 
 // Note: these are currently hardcoded in scripts/generate_extensions_function.py
 // TODO: automate by passing though to script via duckdb
-static constexpr ExtensionEntry EXTENSION_COPY_FUNCTIONS[] = {{"parquet", "parquet"},
-                                                              {"json", "json"}}; // END_OF_EXTENSION_COPY_FUNCTIONS
+static constexpr ExtensionEntry EXTENSION_COPY_FUNCTIONS[] = {
+    {"parquet", "parquet"}, {"json", "json"}, {"avro", "avro"}}; // END_OF_EXTENSION_COPY_FUNCTIONS
 
 // Note: these are currently hardcoded in scripts/generate_extensions_function.py
 // TODO: automate by passing though to script via duckdb
@@ -1082,8 +1095,8 @@ static constexpr ExtensionEntry EXTENSION_FILE_PREFIXES[] = {
 // Note: these are currently hardcoded in scripts/generate_extensions_function.py
 // TODO: automate by passing though to script via duckdb
 static constexpr ExtensionEntry EXTENSION_FILE_POSTFIXES[] = {
-    {".parquet", "parquet"}, {".json", "json"},    {".jsonl", "json"},  {".ndjson", "json"},
-    {".shp", "spatial"},     {".gpkg", "spatial"}, {".fgb", "spatial"}, {".xlsx", "excel"},
+    {".parquet", "parquet"}, {".json", "json"},   {".jsonl", "json"}, {".ndjson", "json"}, {".shp", "spatial"},
+    {".gpkg", "spatial"},    {".fgb", "spatial"}, {".xlsx", "excel"}, {".avro", "avro"},
 }; // END_OF_EXTENSION_FILE_POSTFIXES
 
 // Note: these are currently hardcoded in scripts/generate_extensions_function.py
@@ -1119,6 +1132,7 @@ static constexpr const char *AUTOLOADABLE_EXTENSIONS[] = {"avro",
                                                           "autocomplete",
                                                           "core_functions",
                                                           "delta",
+                                                          "ducklake",
                                                           "encodings",
                                                           "excel",
                                                           "fts",

package/src/duckdb/src/include/duckdb/main/secret/secret_manager.hpp

@@ -222,6 +222,7 @@ protected:
 	unique_ptr<CatalogEntry> CreateDefaultEntryInternal(const string &entry_name);
 
 	SecretManager &secret_manager;
+	mutex lock;
 	case_insensitive_set_t persistent_secrets;
 };
 

package/src/duckdb/src/include/duckdb/main/settings.hpp

@@ -218,6 +218,17 @@ struct ArrowOutputListViewSetting {
 	static Value GetSetting(const ClientContext &context);
 };
 
+struct ArrowOutputVersionSetting {
+	using RETURN_TYPE = string;
+	static constexpr const char *Name = "arrow_output_version";
+	static constexpr const char *Description =
+	    "Whether strings should be produced by DuckDB in Utf8View format instead of Utf8";
+	static constexpr const char *InputType = "VARCHAR";
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
 struct AsofLoopJoinThresholdSetting {
 	using RETURN_TYPE = idx_t;
 	static constexpr const char *Name = "asof_loop_join_threshold";

package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp

@@ -8,6 +8,7 @@
 
 #pragma once
 
+#include "duckdb/main/client_context.hpp"
 #include "duckdb/common/constants.hpp"
 
 namespace duckdb {
@@ -17,13 +18,18 @@ class Optimizer;
 
 class TopN {
 public:
+	explicit TopN(ClientContext &context);
+
 	//! Optimize ORDER BY + LIMIT to TopN
 	unique_ptr<LogicalOperator> Optimize(unique_ptr<LogicalOperator> op);
 	//! Whether we can perform the optimization on this operator
-	static bool CanOptimize(LogicalOperator &op);
+	static bool CanOptimize(LogicalOperator &op, optional_ptr<ClientContext> context = nullptr);
 
 private:
 	void PushdownDynamicFilters(LogicalTopN &op);
+
+private:
+	ClientContext &context;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/original/std/locale.hpp

@@ -0,0 +1,10 @@
+#pragma once
+
+#include <cctype>
+#include <locale>
+
+#define DUCKDB_BASE_STD
+
+namespace duckdb_base_std {
+using ::std::isspace;
+} // namespace duckdb_base_std

package/src/duckdb/src/include/duckdb/original/std/memory.hpp

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <memory>
+
+#define DUCKDB_BASE_STD
+
+namespace duckdb_base_std {
+using ::std::make_shared;
+using ::std::shared_ptr;
+using ::std::unique_ptr;
+// using ::std::make_unique;
+} // namespace duckdb_base_std

package/src/duckdb/src/include/duckdb/original/std/sstream.hpp

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <sstream>
+
+#define DUCKDB_BASE_STD
+
+namespace duckdb_base_std {
+using ::std::basic_stringstream;
+using ::std::stringstream;
+using ::std::wstringstream;
+} // namespace duckdb_base_std

package/src/duckdb/src/include/duckdb/planner/expression_binder.hpp

@@ -175,7 +175,8 @@ protected:
 	BindResult BindExpression(ConjunctionExpression &expr, idx_t depth);
 	BindResult BindExpression(ConstantExpression &expr, idx_t depth);
 	BindResult BindExpression(FunctionExpression &expr, idx_t depth, unique_ptr<ParsedExpression> &expr_ptr);
-	BindResult BindExpression(LambdaExpression &expr, idx_t depth, const LogicalType &list_child_type,
+
+	BindResult BindExpression(LambdaExpression &expr, idx_t depth, const vector<LogicalType> &function_child_types,
 	                          optional_ptr<bind_lambda_function_t> bind_lambda_function);
 	BindResult BindExpression(OperatorExpression &expr, idx_t depth);
 	BindResult BindExpression(ParameterExpression &expr, idx_t depth);
@@ -185,10 +186,11 @@ protected:
 	void TransformCapturedLambdaColumn(unique_ptr<Expression> &original, unique_ptr<Expression> &replacement,
 	                                   BoundLambdaExpression &bound_lambda_expr,
 	                                   const optional_ptr<bind_lambda_function_t> bind_lambda_function,
-	                                   const LogicalType &list_child_type);
+	                                   const vector<LogicalType> &function_child_types);
+
 	void CaptureLambdaColumns(BoundLambdaExpression &bound_lambda_expr, unique_ptr<Expression> &expr,
 	                          const optional_ptr<bind_lambda_function_t> bind_lambda_function,
-	                          const LogicalType &list_child_type);
+	                          const vector<LogicalType> &function_child_types);
 
 	virtual unique_ptr<ParsedExpression> GetSQLValueFunction(const string &column_name);
 

package/src/duckdb/src/include/duckdb/storage/buffer/buffer_pool.hpp

@@ -91,12 +91,14 @@ protected:
 	//! Increments the dead nodes for the queue with specified type
 	void IncrementDeadNodes(const BlockHandle &handle);
 
+	//! How many eviction queue types we have (BLOCK and EXTERNAL_FILE go into same queue)
+	static constexpr idx_t EVICTION_QUEUE_TYPES = FILE_BUFFER_TYPE_COUNT - 1;
 	//! How many eviction queues we have for the different FileBufferTypes
-	static constexpr idx_t BLOCK_QUEUE_SIZE = 1;
+	static constexpr idx_t BLOCK_AND_EXTERNAL_FILE_QUEUE_SIZE = 1;
 	static constexpr idx_t MANAGED_BUFFER_QUEUE_SIZE = 6;
 	static constexpr idx_t TINY_BUFFER_QUEUE_SIZE = 1;
 	//! Mapping and priority order for the eviction queues
-	const array<idx_t, FILE_BUFFER_TYPE_COUNT> eviction_queue_sizes;
+	const array<idx_t, EVICTION_QUEUE_TYPES> eviction_queue_sizes;
 
 protected:
 	enum class MemoryUsageCaches {

package/src/duckdb/src/logging/log_manager.cpp

@@ -221,6 +221,7 @@ void LogManager::RegisterDefaultLogTypes() {
 	RegisterLogType(make_uniq<FileSystemLogType>());
 	RegisterLogType(make_uniq<HTTPLogType>());
 	RegisterLogType(make_uniq<QueryLogType>());
+	RegisterLogType(make_uniq<PhysicalOperatorLogType>());
 }
 
 } // namespace duckdb

package/src/duckdb/src/logging/log_types.cpp

@@ -4,6 +4,7 @@
 #include "duckdb/common/http_util.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/database.hpp"
+#include "duckdb/execution/physical_operator.hpp"
 
 namespace duckdb {
 
@@ -11,6 +12,7 @@ constexpr LogLevel DefaultLogType::LEVEL;
 constexpr LogLevel FileSystemLogType::LEVEL;
 constexpr LogLevel QueryLogType::LEVEL;
 constexpr LogLevel HTTPLogType::LEVEL;
+constexpr LogLevel PhysicalOperatorLogType::LEVEL;
 
 FileSystemLogType::FileSystemLogType() : LogType(NAME, LEVEL, GetLogType()) {
 }
@@ -90,4 +92,42 @@ string HTTPLogType::ConstructLogMessage(BaseRequest &request, optional_ptr<HTTPR
 	return Value::STRUCT(child_list).ToString();
 }
 
+PhysicalOperatorLogType::PhysicalOperatorLogType() : LogType(NAME, LEVEL, GetLogType()) {
+}
+
+LogicalType PhysicalOperatorLogType::GetLogType() {
+	child_list_t<LogicalType> child_list = {
+	    {"operator_type", LogicalType::VARCHAR},
+	    {"parameters", LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)},
+	    {"class", LogicalType::VARCHAR},
+	    {"event", LogicalType::VARCHAR},
+	    {"info", LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)},
+	};
+	return LogicalType::STRUCT(child_list);
+}
+
+template <class ITERABLE>
+static Value StringPairIterableToMap(const ITERABLE &iterable) {
+	vector<Value> keys;
+	vector<Value> values;
+	for (const auto &kv : iterable) {
+		keys.emplace_back(kv.first);
+		values.emplace_back(kv.second);
+	}
+	return Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, std::move(keys), std::move(values));
+}
+
+string PhysicalOperatorLogType::ConstructLogMessage(const PhysicalOperator &physical_operator, const string &class_p,
+                                                    const string &event, const vector<pair<string, string>> &info) {
+	child_list_t<Value> child_list = {
+	    {"operator_type", EnumUtil::ToString(physical_operator.type)},
+	    {"parameters", StringPairIterableToMap(physical_operator.ParamsToString())},
+	    {"class", class_p},
+	    {"event", event},
+	    {"info", StringPairIterableToMap(info)},
+	};
+
+	return Value::STRUCT(std::move(child_list)).ToString();
+}
+
 } // namespace duckdb

package/src/duckdb/src/main/attached_database.cpp

@@ -185,6 +185,10 @@ void AttachedDatabase::Initialize(optional_ptr<ClientContext> context, StorageOp
 	}
 }
 
+void AttachedDatabase::FinalizeLoad(optional_ptr<ClientContext> context) {
+	catalog->FinalizeLoad(context);
+}
+
 StorageManager &AttachedDatabase::GetStorageManager() {
 	if (!storage) {
 		throw InternalException("Internal system catalog does not have storage");

package/src/duckdb/src/main/client_context.cpp

@@ -1382,6 +1382,7 @@ ClientProperties ClientContext::GetClientProperties() {
 	        db->config.options.arrow_use_list_view,
 	        db->config.options.produce_arrow_string_views,
 	        db->config.options.arrow_lossless_conversion,
+	        db->config.options.arrow_output_version,
 	        this};
 }
 

package/src/duckdb/src/main/config.cpp

@@ -72,6 +72,7 @@ static const ConfigurationOption internal_options[] = {
     DUCKDB_GLOBAL(ArrowLargeBufferSizeSetting),
     DUCKDB_GLOBAL(ArrowLosslessConversionSetting),
     DUCKDB_GLOBAL(ArrowOutputListViewSetting),
+    DUCKDB_GLOBAL(ArrowOutputVersionSetting),
     DUCKDB_LOCAL(AsofLoopJoinThresholdSetting),
     DUCKDB_GLOBAL(AutoinstallExtensionRepositorySetting),
     DUCKDB_GLOBAL(AutoinstallKnownExtensionsSetting),

package/src/duckdb/src/main/database.cpp

@@ -345,6 +345,7 @@ DuckDB::DuckDB(const char *path, DBConfig *new_config) : instance(make_shared_pt
 	if (instance->config.options.load_extensions) {
 		ExtensionHelper::LoadAllExtensions(*this);
 	}
+	instance->db_manager->FinalizeStartup();
 }
 
 DuckDB::DuckDB(const string &path, DBConfig *config) : DuckDB(path.c_str(), config) {

package/src/duckdb/src/main/database_manager.cpp

@@ -29,6 +29,13 @@ void DatabaseManager::InitializeSystemCatalog() {
 	system->Initialize();
 }
 
+void DatabaseManager::FinalizeStartup() {
+	auto databases = GetDatabases();
+	for (auto &db : databases) {
+		db.get().FinalizeLoad(nullptr);
+	}
+}
+
 optional_ptr<AttachedDatabase> DatabaseManager::GetDatabase(ClientContext &context, const string &name) {
 	if (StringUtil::Lower(name) == TEMP_CATALOG) {
 		return context.client_data->temporary_objects.get();
@@ -236,9 +243,19 @@ void DatabaseManager::SetDefaultDatabase(ClientContext &context, const string &n
 }
 // LCOV_EXCL_STOP
 
-vector<reference<AttachedDatabase>> DatabaseManager::GetDatabases(ClientContext &context) {
+vector<reference<AttachedDatabase>> DatabaseManager::GetDatabases(ClientContext &context,
+                                                                  const optional_idx max_db_count) {
 	vector<reference<AttachedDatabase>> result;
-	databases->Scan(context, [&](CatalogEntry &entry) { result.push_back(entry.Cast<AttachedDatabase>()); });
+	idx_t count = 2;
+	databases->ScanWithReturn(context, [&](CatalogEntry &entry) {
+		if (max_db_count.IsValid() && count >= max_db_count.GetIndex()) {
+			return false;
+		}
+		result.push_back(entry.Cast<AttachedDatabase>());
+		count++;
+		return true;
+	});
+
 	result.push_back(*system);
 	result.push_back(*context.client_data->temporary_objects);
 	return result;

package/src/duckdb/src/main/extension/extension_helper.cpp

@@ -123,6 +123,7 @@ static const DefaultExtension internal_extensions[] = {
     {"delta", "Adds support for Delta Lake", false},
     {"fts", "Adds support for Full-Text Search Indexes", false},
     {"ui", "Adds local UI for DuckDB", false},
+    {"ducklake", "Adds support for DuckLake, SQL as a Lakehouse Format", false},
     {nullptr, nullptr, false}};
 
 idx_t ExtensionHelper::DefaultExtensionCount() {
@@ -140,9 +141,9 @@ DefaultExtension ExtensionHelper::GetDefaultExtension(idx_t index) {
 //===--------------------------------------------------------------------===//
 // Allow Auto-Install Extensions
 //===--------------------------------------------------------------------===//
-static const char *const auto_install[] = {"motherduck", "postgres_scanner", "mysql_scanner", "sqlite_scanner",
-                                           "delta",      "iceberg",          "uc_catalog",    "ui",
-                                           nullptr};
+static const char *const auto_install[] = {
+    "motherduck", "postgres_scanner", "mysql_scanner", "sqlite_scanner", "delta", "iceberg", "uc_catalog",
+    "ui",         "ducklake",         nullptr};
 
 // TODO: unify with new autoload mechanism
 bool ExtensionHelper::AllowAutoInstall(const string &extension) {