duckdb 1.1.0 → 1.1.1-dev3.0

package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp

@@ -483,6 +483,66 @@ DUCKDB_API bool TryCast::Operation(double input, double &result, bool strict);
 //===--------------------------------------------------------------------===//
 // String -> Numeric Casts
 //===--------------------------------------------------------------------===//
+static inline bool TryCastStringBool(const char *input_data, idx_t input_size, bool &result, bool strict) {
+	switch (input_size) {
+	case 1: {
+		unsigned char c = static_cast<uint8_t>(std::tolower(*input_data));
+		if (c == 't' || (!strict && c == 'y') || (!strict && c == '1')) {
+			result = true;
+			return true;
+		} else if (c == 'f' || (!strict && c == 'n') || (!strict && c == '0')) {
+			result = false;
+			return true;
+		}
+		return false;
+	}
+	case 2: {
+		unsigned char n = static_cast<uint8_t>(std::tolower(input_data[0]));
+		unsigned char o = static_cast<uint8_t>(std::tolower(input_data[1]));
+		if (n == 'n' && o == 'o') {
+			result = false;
+			return true;
+		}
+		return false;
+	}
+	case 3: {
+		unsigned char y = static_cast<uint8_t>(std::tolower(input_data[0]));
+		unsigned char e = static_cast<uint8_t>(std::tolower(input_data[1]));
+		unsigned char s = static_cast<uint8_t>(std::tolower(input_data[2]));
+		if (y == 'y' && e == 'e' && s == 's') {
+			result = true;
+			return true;
+		}
+		return false;
+	}
+	case 4: {
+		unsigned char t = static_cast<uint8_t>(std::tolower(input_data[0]));
+		unsigned char r = static_cast<uint8_t>(std::tolower(input_data[1]));
+		unsigned char u = static_cast<uint8_t>(std::tolower(input_data[2]));
+		unsigned char e = static_cast<uint8_t>(std::tolower(input_data[3]));
+		if (t == 't' && r == 'r' && u == 'u' && e == 'e') {
+			result = true;
+			return true;
+		}
+		return false;
+	}
+	case 5: {
+		unsigned char f = static_cast<uint8_t>(std::tolower(input_data[0]));
+		unsigned char a = static_cast<uint8_t>(std::tolower(input_data[1]));
+		unsigned char l = static_cast<uint8_t>(std::tolower(input_data[2]));
+		unsigned char s = static_cast<uint8_t>(std::tolower(input_data[3]));
+		unsigned char e = static_cast<uint8_t>(std::tolower(input_data[4]));
+		if (f == 'f' && a == 'a' && l == 'l' && s == 's' && e == 'e') {
+			result = false;
+			return true;
+		}
+		return false;
+	}
+	default:
+		return false;
+	}
+}
+
 template <>
 DUCKDB_API bool TryCast::Operation(string_t input, bool &result, bool strict);
 template <>

package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp

@@ -32,6 +32,7 @@ public:
 	explicit ColumnDataAllocator(BufferManager &buffer_manager);
 	ColumnDataAllocator(ClientContext &context, ColumnDataAllocatorType allocator_type);
 	ColumnDataAllocator(ColumnDataAllocator &allocator);
+	~ColumnDataAllocator();
 
 	//! Returns an allocator object to allocate with. This returns the allocator in IN_MEMORY_ALLOCATOR, and a buffer
 	//! allocator in case of BUFFER_MANAGER_ALLOCATOR.

package/src/duckdb/src/include/duckdb/common/types/hugeint.hpp

@@ -128,7 +128,6 @@ public:
 
 	static int Sign(hugeint_t n);
 	static hugeint_t Abs(hugeint_t n);
-
 	// comparison operators
 	// note that everywhere here we intentionally use bitwise ops
 	// this is because they seem to be consistently much faster (benchmarked on a Macbook Pro)

package/src/duckdb/src/include/duckdb/common/types/row/row_data_collection.hpp

@@ -20,7 +20,8 @@ public:
 	RowDataBlock(MemoryTag tag, BufferManager &buffer_manager, idx_t capacity, idx_t entry_size)
 	    : capacity(capacity), entry_size(entry_size), count(0), byte_offset(0) {
 		auto size = MaxValue<idx_t>(buffer_manager.GetBlockSize(), capacity * entry_size);
-		buffer_manager.Allocate(tag, size, false, &block);
+		auto buffer_handle = buffer_manager.Allocate(tag, size, false);
+		block = buffer_handle.GetBlockHandle();
 		D_ASSERT(BufferManager::GetAllocSize(size) == block->GetMemoryUsage());
 	}
 

package/src/duckdb/src/include/duckdb/core_functions/aggregate/minmax_n_helpers.hpp

@@ -1,7 +1,14 @@
 #pragma once
 
-#include "duckdb/common/vector.hpp"
+#include "duckdb/common/common.hpp"
+#include "duckdb/storage/arena_allocator.hpp"
 #include "duckdb/common/algorithm.hpp"
+#include "duckdb/common/pair.hpp"
+#include "duckdb/common/types/string_type.hpp"
+#include "duckdb/common/types/vector.hpp"
+#include "duckdb/common/enums/order_type.hpp"
+#include "duckdb/function/aggregate_function.hpp"
+#include "duckdb/core_functions/create_sort_key.hpp"
 
 namespace duckdb {
 
@@ -294,11 +301,8 @@ struct MinMaxFallbackValue {
 
 	static void PrepareData(Vector &input, const idx_t count, EXTRA_STATE &extra_state, UnifiedVectorFormat &format) {
 		const OrderModifiers modifiers(OrderType::ASCENDING, OrderByNullType::NULLS_LAST);
-		CreateSortKeyHelpers::CreateSortKey(input, count, modifiers, extra_state);
+		CreateSortKeyHelpers::CreateSortKeyWithValidity(input, extra_state, modifiers, count);
 		input.Flatten(count);
-		extra_state.Flatten(count);
-		// Ensure the validity vectors match, because we want to ignore nulls
-		FlatVector::Validity(extra_state).Initialize(FlatVector::Validity(input));
 		extra_state.ToUnifiedFormat(count, format);
 	}
 };

package/src/duckdb/src/include/duckdb/execution/executor.hpp

@@ -13,6 +13,7 @@
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/common/pair.hpp"
 #include "duckdb/common/reference_map.hpp"
+#include "duckdb/main/query_result.hpp"
 #include "duckdb/execution/task_error_manager.hpp"
 #include "duckdb/parallel/pipeline.hpp"
 

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/base_scanner.hpp

@@ -73,7 +73,7 @@ public:
 	virtual ~BaseScanner() = default;
 
 	//! Returns true if the scanner is finished
-	bool FinishedFile();
+	bool FinishedFile() const;
 
 	//! Parses data into a output_chunk
 	virtual ScannerResult &ParseChunk();
@@ -97,7 +97,7 @@ public:
 		return iterator.pos;
 	}
 
-	CSVStateMachine &GetStateMachine();
+	CSVStateMachine &GetStateMachine() const;
 
 	shared_ptr<CSVFileScan> csv_file_scan;
 
@@ -313,6 +313,9 @@ protected:
 	//! Internal function for parse chunk
 	template <class T>
 	void ParseChunkInternal(T &result) {
+		if (iterator.done) {
+			return;
+		}
 		if (!initialized) {
 			Initialize();
 			initialized = true;

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/column_count_scanner.hpp

@@ -41,7 +41,8 @@ public:
 	bool error = false;
 	idx_t result_position = 0;
 	bool cur_line_starts_as_comment = false;
-
+	//! How many rows fit a given column count
+	map<idx_t, idx_t> rows_per_column_count;
 	//! Adds a Value to the result
 	static inline void AddValue(ColumnCountResult &result, idx_t buffer_pos);
 	//! Adds a Row to the result
@@ -57,6 +58,9 @@ public:
 
 	static inline void SetComment(ColumnCountResult &result, idx_t buffer_pos);
 
+	//! Returns the column count
+	idx_t GetMostFrequentColumnCount() const;
+
 	inline void InternalAddRow();
 };
 

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp

@@ -56,7 +56,7 @@ struct DialectCandidates {
 	//! Candidates for the comment
 	vector<char> comment_candidates;
 	//! Quote-Rule Candidates
-	vector<QuoteRule> quoterule_candidates;
+	vector<QuoteRule> quote_rule_candidates;
 	//! Candidates for the quote option
 	unordered_map<uint8_t, vector<char>> quote_candidates_map;
 	//! Candidates for the escape option
@@ -181,7 +181,7 @@ private:
 	void RefineCandidates();
 
 	//! Checks if candidate still produces good values for the next chunk
-	bool RefineCandidateNextChunk(ColumnCountScanner &candidate);
+	bool RefineCandidateNextChunk(ColumnCountScanner &candidate) const;
 
 	//! ------------------------------------------------------//
 	//! ------------------- Type Detection ------------------ //
@@ -198,7 +198,7 @@ private:
 	void InitializeDateAndTimeStampDetection(CSVStateMachine &candidate, const string &separator,
 	                                         const LogicalType &sql_type);
 	//! Sets user defined date and time formats (if any)
-	void SetUserDefinedDateTimeFormat(CSVStateMachine &candidate);
+	void SetUserDefinedDateTimeFormat(CSVStateMachine &candidate) const;
 	//! Functions that performs detection for date and timestamp formats
 	void DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const LogicalType &sql_type, const string &separator,
 	                                   const string_t &dummy_val);
@@ -235,10 +235,10 @@ private:
 	//! ------------------------------------------------------//
 	void DetectHeader();
 	static bool DetectHeaderWithSetColumn(ClientContext &context, vector<HeaderValue> &best_header_row,
-	                                      SetColumns &set_columns, CSVReaderOptions &options);
+	                                      const SetColumns &set_columns, CSVReaderOptions &options);
 	static vector<string>
 	DetectHeaderInternal(ClientContext &context, vector<HeaderValue> &best_header_row, CSVStateMachine &state_machine,
-	                     SetColumns &set_columns,
+	                     const SetColumns &set_columns,
 	                     unordered_map<idx_t, vector<LogicalType>> &best_sql_types_candidates_per_column_idx,
 	                     CSVReaderOptions &options, CSVErrorHandler &error_handler);
 	vector<string> names;

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_result_collector.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/execution/physical_operator.hpp"
+#include "duckdb/main/query_result.hpp"
 #include "duckdb/common/enums/statement_type.hpp"
 
 namespace duckdb {

package/src/duckdb/src/include/duckdb/function/table/arrow/arrow_duck_schema.hpp

@@ -8,6 +8,8 @@
 
 #pragma once
 
+#include <utility>
+
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/common/vector.hpp"
@@ -22,6 +24,10 @@ public:
 	explicit ArrowType(LogicalType type_p, unique_ptr<ArrowTypeInfo> type_info = nullptr)
 	    : type(std::move(type_p)), type_info(std::move(type_info)) {
 	}
+	explicit ArrowType(string error_message_p, bool not_implemented_p = false)
+	    : type(LogicalTypeId::INVALID), type_info(nullptr), error_message(std::move(error_message_p)),
+	      not_implemented(not_implemented_p) {
+	}
 
 public:
 	LogicalType GetDuckType(bool use_dictionary = false) const;
@@ -37,6 +43,7 @@ public:
 	const T &GetTypeInfo() const {
 		return type_info->Cast<T>();
 	}
+	void ThrowIfInvalid() const;
 
 private:
 	LogicalType type;
@@ -45,6 +52,10 @@ private:
 	//! Is run-end-encoded
 	bool run_end_encoded = false;
 	unique_ptr<ArrowTypeInfo> type_info;
+	//! Error message in case of an invalid type (i.e., from an unsupported extension)
+	string error_message;
+	//! In case of an error do we throw not implemented?
+	bool not_implemented = false;
 };
 
 using arrow_column_map_t = unordered_map<idx_t, unique_ptr<ArrowType>>;

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -245,7 +245,7 @@ struct DBConfigOptions {
 	//! The peak allocation threshold at which to flush the allocator after completing a task (1 << 27, ~128MB)
 	idx_t allocator_flush_threshold = 134217728ULL;
 	//! If bulk deallocation larger than this occurs, flush outstanding allocations (1 << 30, ~1GB)
-	idx_t allocator_bulk_deallocation_flush_threshold = 1073741824ULL;
+	idx_t allocator_bulk_deallocation_flush_threshold = 536870912ULL;
 	//! Whether the allocator background thread is enabled
 	bool allocator_background_threads = false;
 	//! DuckDB API surface
@@ -365,7 +365,7 @@ public:
 	DUCKDB_API IndexTypeSet &GetIndexTypes();
 	static idx_t GetSystemMaxThreads(FileSystem &fs);
 	static idx_t GetSystemAvailableMemory(FileSystem &fs);
-	static idx_t ParseMemoryLimitSlurm(const string &arg);
+	static optional_idx ParseMemoryLimitSlurm(const string &arg);
 	void SetDefaultMaxMemory();
 	void SetDefaultTempDirectory();
 

package/src/duckdb/src/include/duckdb/main/extension.hpp

@@ -50,6 +50,7 @@ struct ParsedExtensionMetaData {
 	string duckdb_capi_version;
 	string extension_version;
 	string signature;
+	string extension_abi_metadata;
 
 	bool AppearsValid() {
 		return magic_value == EXPECTED_MAGIC_VALUE;

package/src/duckdb/src/include/duckdb/main/extension_entries.hpp

@@ -293,6 +293,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"st_envelope_agg", "spatial", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"st_equals", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_extent", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"st_extent_agg", "spatial", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"st_extent_approx", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_exteriorring", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_flipcoordinates", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -331,6 +332,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"st_makevalid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_mmax", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_mmin", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"st_multi", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_ngeometries", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_ninteriorrings", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_normalize", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -511,11 +513,18 @@ static constexpr ExtensionEntry EXTENSION_SECRET_TYPES[] = {
 // Note: these are currently hardcoded in scripts/generate_extensions_function.py
 // TODO: automate by passing though to script via duckdb
 static constexpr ExtensionEntry EXTENSION_SECRET_PROVIDERS[] = {
-    {"s3/config", "httpfs"},          {"gcs/config", "httpfs"},
-    {"r2/config", "httpfs"},          {"s3/credential_chain", "aws"},
-    {"gcs/credential_chain", "aws"},  {"r2/credential_chain", "aws"},
-    {"azure/config", "azure"},        {"azure/credential_chain", "azure"},
-    {"huggingface/config", "httfps"}, {"huggingface/credential_chain", "httpfs"},
+    {"s3/config", "httpfs"},
+    {"gcs/config", "httpfs"},
+    {"r2/config", "httpfs"},
+    {"s3/credential_chain", "aws"},
+    {"gcs/credential_chain", "aws"},
+    {"r2/credential_chain", "aws"},
+    {"azure/access_token", "azure"},
+    {"azure/config", "azure"},
+    {"azure/credential_chain", "azure"},
+    {"azure/service_principal", "azure"},
+    {"huggingface/config", "httfps"},
+    {"huggingface/credential_chain", "httpfs"},
     {"bearer/config", "httpfs"}}; // EXTENSION_SECRET_PROVIDERS
 
 static constexpr const char *AUTOLOADABLE_EXTENSIONS[] = {

package/src/duckdb/src/include/duckdb/main/extension_helper.hpp

@@ -124,7 +124,7 @@ public:
 
 	static bool CheckExtensionSignature(FileHandle &handle, ParsedExtensionMetaData &parsed_metadata,
 	                                    const bool allow_community_extensions);
-	static ParsedExtensionMetaData ParseExtensionMetaData(const char *metadata);
+	static ParsedExtensionMetaData ParseExtensionMetaData(const char *metadata) noexcept;
 	static ParsedExtensionMetaData ParseExtensionMetaData(FileHandle &handle);
 
 	//! Get the extension url template, containing placeholders for version, platform and extension name

package/src/duckdb/src/include/duckdb/main/settings.hpp

@@ -506,7 +506,7 @@ struct HTTPProxyUsername {
 };
 
 struct HTTPProxyPassword {
-	static constexpr const char *Name = "http_proxy";
+	static constexpr const char *Name = "http_proxy_password";
 	static constexpr const char *Description = "Password for HTTP proxy";
 	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
 	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
@@ -596,7 +596,9 @@ struct StreamingBufferSize {
 struct MaximumTempDirectorySize {
 	static constexpr const char *Name = "max_temp_directory_size";
 	static constexpr const char *Description =
-	    "The maximum amount of data stored inside the 'temp_directory' (when set) (e.g. 1GB)";
+	    "The maximum amount of data stored inside the 'temp_directory' (when set). If the `temp_directory` is set to "
+	    "an existing directory, this option defaults to the available disk space on "
+	    "that drive. Otherwise, it defaults to 0 (implying that the temporary directory is not used).";
 	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
 	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
 	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);

package/src/duckdb/src/include/duckdb/parser/keyword_helper.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/common/common.hpp"
+#include "duckdb/parser/simplified_token.hpp"
 
 namespace duckdb {
 
@@ -17,6 +18,8 @@ public:
 	//! Returns true if the given text matches a keyword of the parser
 	static bool IsKeyword(const string &text);
 
+	static KeywordCategory KeywordCategoryType(const string &text);
+
 	static string EscapeQuotes(const string &text, char quote = '"');
 
 	//! Returns true if the given string needs to be quoted when written as an identifier

package/src/duckdb/src/include/duckdb/parser/parser.hpp

@@ -45,7 +45,7 @@ public:
 	static vector<SimplifiedToken> Tokenize(const string &query);
 
 	//! Returns true if the given text matches a keyword of the parser
-	static bool IsKeyword(const string &text);
+	static KeywordCategory IsKeyword(const string &text);
 	//! Returns a list of all keywords in the parser
 	static vector<ParserKeyword> KeywordList();
 

package/src/duckdb/src/include/duckdb/parser/simplified_token.hpp

@@ -28,7 +28,13 @@ struct SimplifiedToken {
 	idx_t start;
 };
 
-enum class KeywordCategory : uint8_t { KEYWORD_RESERVED, KEYWORD_UNRESERVED, KEYWORD_TYPE_FUNC, KEYWORD_COL_NAME };
+enum class KeywordCategory : uint8_t {
+	KEYWORD_RESERVED,
+	KEYWORD_UNRESERVED,
+	KEYWORD_TYPE_FUNC,
+	KEYWORD_COL_NAME,
+	KEYWORD_NONE
+};
 
 struct ParserKeyword {
 	string name;

package/src/duckdb/src/include/duckdb/planner/binder.hpp

@@ -416,6 +416,8 @@ private:
 	unique_ptr<BoundTableRef> BindShowTable(ShowRef &ref);
 	unique_ptr<BoundTableRef> BindSummarize(ShowRef &ref);
 
+	unique_ptr<LogicalOperator> UnionOperators(vector<unique_ptr<LogicalOperator>> nodes);
+
 private:
 	Binder(ClientContext &context, shared_ptr<Binder> parent, BinderType binder_type);
 };

package/src/duckdb/src/include/duckdb/planner/expression_binder/select_binder.hpp

@@ -18,10 +18,12 @@ public:
 	SelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info);
 
 protected:
+	void ThrowIfUnnestInLambda(const ColumnBinding &column_binding) override;
 	BindResult BindUnnest(FunctionExpression &function, idx_t depth, bool root_expression) override;
 	BindResult BindColumnRef(unique_ptr<ParsedExpression> &expr_ptr, idx_t depth, bool root_expression) override;
 
 	bool QualifyColumnAlias(const ColumnRefExpression &colref) override;
+	unique_ptr<ParsedExpression> GetSQLValueFunction(const string &column_name) override;
 
 protected:
 	idx_t unnest_level = 0;

package/src/duckdb/src/include/duckdb/planner/expression_binder.hpp

@@ -21,6 +21,7 @@
 #include "duckdb/catalog/catalog_entry_retriever.hpp"
 #include "duckdb/planner/expression/bound_lambda_expression.hpp"
 #include "duckdb/function/scalar_function.hpp"
+#include "duckdb/planner/column_binding.hpp"
 
 namespace duckdb {
 
@@ -186,7 +187,7 @@ protected:
 	                          const optional_ptr<bind_lambda_function_t> bind_lambda_function,
 	                          const LogicalType &list_child_type);
 
-	static unique_ptr<ParsedExpression> GetSQLValueFunction(const string &column_name);
+	virtual unique_ptr<ParsedExpression> GetSQLValueFunction(const string &column_name);
 
 	LogicalType ResolveOperatorType(OperatorExpression &op, vector<unique_ptr<Expression>> &children);
 	LogicalType ResolveCoalesceType(OperatorExpression &op, vector<unique_ptr<Expression>> &children);
@@ -219,6 +220,7 @@ protected:
 	//! Returns true if the function name is an alias for the UNNEST function
 	static bool IsUnnestFunction(const string &function_name);
 	BindResult TryBindLambdaOrJson(FunctionExpression &function, idx_t depth, CatalogEntry &func);
+	virtual void ThrowIfUnnestInLambda(const ColumnBinding &column_binding);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/block_manager.hpp

@@ -88,7 +88,9 @@ public:
 	//! Convert an existing in-memory buffer into a persistent disk-backed block
 	shared_ptr<BlockHandle> ConvertToPersistent(block_id_t block_id, shared_ptr<BlockHandle> old_block);
 
-	void UnregisterBlock(block_id_t block_id);
+	void UnregisterBlock(BlockHandle &block);
+	//! UnregisterBlock, only accepts non-temporary block ids
+	void UnregisterBlock(block_id_t id);
 
 	//! Returns a reference to the metadata manager of this block manager.
 	MetadataManager &GetMetadataManager();

package/src/duckdb/src/include/duckdb/storage/buffer/block_handle.hpp

@@ -54,7 +54,7 @@ struct TempBufferPoolReservation : BufferPoolReservation {
 	}
 };
 
-class BlockHandle {
+class BlockHandle : public enable_shared_from_this<BlockHandle> {
 	friend class BlockManager;
 	friend struct BufferEvictionNode;
 	friend class BufferHandle;
@@ -96,6 +96,10 @@ public:
 		unswizzled = unswizzler;
 	}
 
+	MemoryTag GetMemoryTag() const {
+		return tag;
+	}
+
 	inline void SetDestroyBufferUpon(DestroyBufferUpon destroy_buffer_upon_p) {
 		lock_guard<mutex> guard(lock);
 		destroy_buffer_upon = destroy_buffer_upon_p;
@@ -117,9 +121,8 @@ public:
 	}
 
 private:
-	static BufferHandle Load(shared_ptr<BlockHandle> &handle, unique_ptr<FileBuffer> buffer = nullptr);
-	static BufferHandle LoadFromBuffer(shared_ptr<BlockHandle> &handle, data_ptr_t data,
-	                                   unique_ptr<FileBuffer> reusable_buffer);
+	BufferHandle Load(unique_ptr<FileBuffer> buffer = nullptr);
+	BufferHandle LoadFromBuffer(data_ptr_t data, unique_ptr<FileBuffer> reusable_buffer);
 	unique_ptr<FileBuffer> UnloadAndTakeBlock();
 	void Unload();
 	bool CanUnload();

package/src/duckdb/src/include/duckdb/storage/buffer/buffer_handle.hpp

@@ -18,7 +18,7 @@ class FileBuffer;
 class BufferHandle {
 public:
 	DUCKDB_API BufferHandle();
-	DUCKDB_API BufferHandle(shared_ptr<BlockHandle> handle, FileBuffer *node);
+	DUCKDB_API explicit BufferHandle(shared_ptr<BlockHandle> handle);
 	DUCKDB_API ~BufferHandle();
 	// disable copy constructors
 	BufferHandle(const BufferHandle &other) = delete;
@@ -53,7 +53,7 @@ private:
 	//! The block handle
 	shared_ptr<BlockHandle> handle;
 	//! The managed buffer node
-	FileBuffer *node;
+	optional_ptr<FileBuffer> node;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/buffer/buffer_pool.hpp

@@ -41,7 +41,7 @@ class BufferPool {
 	friend class StandardBufferManager;
 
 public:
-	explicit BufferPool(idx_t maximum_memory, bool track_eviction_timestamps);
+	BufferPool(idx_t maximum_memory, bool track_eviction_timestamps, idx_t allocator_bulk_deallocation_flush_threshold);
 	virtual ~BufferPool();
 
 	//! Set a new memory limit to the buffer pool, throws an exception if the new limit is too low and not enough
@@ -50,6 +50,7 @@ public:
 
 	//! If bulk deallocation larger than this occurs, flush outstanding allocations
 	void SetAllocatorBulkDeallocationFlushThreshold(idx_t threshold);
+	idx_t GetAllocatorBulkDeallocationFlushThreshold();
 
 	void UpdateUsedMemory(MemoryTag tag, int64_t size);
 

package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp

@@ -34,8 +34,7 @@ public:
 	}
 
 public:
-	virtual BufferHandle Allocate(MemoryTag tag, idx_t block_size, bool can_destroy = true,
-	                              shared_ptr<BlockHandle> *block = nullptr) = 0;
+	virtual BufferHandle Allocate(MemoryTag tag, idx_t block_size, bool can_destroy = true) = 0;
 	//! Reallocate an in-memory buffer that is pinned.
 	virtual void ReAllocate(shared_ptr<BlockHandle> &handle, idx_t block_size) = 0;
 	virtual BufferHandle Pin(shared_ptr<BlockHandle> &handle) = 0;
@@ -102,8 +101,9 @@ protected:
 	virtual void PurgeQueue(FileBufferType type) = 0;
 	virtual void AddToEvictionQueue(shared_ptr<BlockHandle> &handle);
 	virtual void WriteTemporaryBuffer(MemoryTag tag, block_id_t block_id, FileBuffer &buffer);
-	virtual unique_ptr<FileBuffer> ReadTemporaryBuffer(MemoryTag tag, block_id_t id, unique_ptr<FileBuffer> buffer);
-	virtual void DeleteTemporaryFile(block_id_t id);
+	virtual unique_ptr<FileBuffer> ReadTemporaryBuffer(MemoryTag tag, BlockHandle &block,
+	                                                   unique_ptr<FileBuffer> buffer);
+	virtual void DeleteTemporaryFile(BlockHandle &block);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/standard_buffer_manager.hpp

@@ -62,8 +62,7 @@ public:
 
 	//! Allocate an in-memory buffer with a single pin.
 	//! The allocated memory is released when the buffer handle is destroyed.
-	DUCKDB_API BufferHandle Allocate(MemoryTag tag, idx_t block_size, bool can_destroy = true,
-	                                 shared_ptr<BlockHandle> *block = nullptr) final;
+	DUCKDB_API BufferHandle Allocate(MemoryTag tag, idx_t block_size, bool can_destroy = true) final;
 
 	//! Reallocate an in-memory buffer that is pinned.
 	void ReAllocate(shared_ptr<BlockHandle> &handle, idx_t block_size) final;
@@ -129,12 +128,12 @@ protected:
 	//! Write a temporary buffer to disk
 	void WriteTemporaryBuffer(MemoryTag tag, block_id_t block_id, FileBuffer &buffer) final;
 	//! Read a temporary buffer from disk
-	unique_ptr<FileBuffer> ReadTemporaryBuffer(MemoryTag tag, block_id_t id,
+	unique_ptr<FileBuffer> ReadTemporaryBuffer(MemoryTag tag, BlockHandle &block,
 	                                           unique_ptr<FileBuffer> buffer = nullptr) final;
 	//! Get the path of the temporary buffer
 	string GetTemporaryPath(block_id_t id);
 
-	void DeleteTemporaryFile(block_id_t id) final;
+	void DeleteTemporaryFile(BlockHandle &block) final;
 
 	void RequireTemporaryDirectory();
 

package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -88,7 +88,7 @@ public:
 	//! Whether or not the column has any updates
 	bool HasUpdates() const;
 	//! Whether or not we can scan an entire vector
-	virtual ScanVectorType GetVectorScanType(ColumnScanState &state, idx_t scan_count);
+	virtual ScanVectorType GetVectorScanType(ColumnScanState &state, idx_t scan_count, Vector &result);
 
 	//! Initialize prefetch state with required I/O data for the next N rows
 	virtual void InitializePrefetch(PrefetchState &prefetch_state, ColumnScanState &scan_state, idx_t rows);

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -33,6 +33,7 @@ class MetadataManager;
 struct VacuumState;
 struct CollectionCheckpointState;
 struct PersistentCollectionData;
+class CheckpointTask;
 
 class RowGroupCollection {
 public:
@@ -98,8 +99,9 @@ public:
 
 	void InitializeVacuumState(CollectionCheckpointState &checkpoint_state, VacuumState &state,
 	                           vector<SegmentNode<RowGroup>> &segments);
-	bool ScheduleVacuumTasks(CollectionCheckpointState &checkpoint_state, VacuumState &state, idx_t segment_idx);
-	void ScheduleCheckpointTask(CollectionCheckpointState &checkpoint_state, idx_t segment_idx);
+	bool ScheduleVacuumTasks(CollectionCheckpointState &checkpoint_state, VacuumState &state, idx_t segment_idx,
+	                         bool schedule_vacuum);
+	unique_ptr<CheckpointTask> GetCheckpointTask(CollectionCheckpointState &checkpoint_state, idx_t segment_idx);
 
 	void CommitDropColumn(idx_t index);
 	void CommitDropTable();

package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp

@@ -25,7 +25,7 @@ public:
 public:
 	void SetStart(idx_t new_start) override;
 
-	ScanVectorType GetVectorScanType(ColumnScanState &state, idx_t scan_count) override;
+	ScanVectorType GetVectorScanType(ColumnScanState &state, idx_t scan_count, Vector &result) override;
 	void InitializePrefetch(PrefetchState &prefetch_state, ColumnScanState &scan_state, idx_t rows) override;
 	void InitializeScan(ColumnScanState &state) override;
 	void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) override;

package/src/duckdb/src/include/duckdb/transaction/duck_transaction.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/transaction/transaction.hpp"
 #include "duckdb/common/reference_map.hpp"
+#include "duckdb/common/error_data.hpp"
 
 namespace duckdb {
 class CheckpointLock;