duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/src/include/duckdb/planner/operator/logical_copy_database.hpp

@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/planner/operator/logical_copy_database.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/parser/parsed_data/copy_info.hpp"
+#include "duckdb/parser/parsed_data/exported_table_data.hpp"
+#include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/function/copy_function.hpp"
+
+namespace duckdb {
+
+struct CopyDatabaseInfo {
+	CopyDatabaseInfo(Catalog &from_database, Catalog &to_database);
+	~CopyDatabaseInfo();
+
+	Catalog &from_database;
+	Catalog &to_database;
+	vector<unique_ptr<CreateInfo>> entries;
+};
+
+class LogicalCopyDatabase : public LogicalOperator {
+public:
+	static constexpr const LogicalOperatorType TYPE = LogicalOperatorType::LOGICAL_COPY_DATABASE;
+
+public:
+	explicit LogicalCopyDatabase(unique_ptr<CopyDatabaseInfo> info_p);
+	~LogicalCopyDatabase() override;
+
+	unique_ptr<CopyDatabaseInfo> info;
+
+public:
+	void Serialize(Serializer &serializer) const override;
+	static unique_ptr<LogicalOperator> Deserialize(Deserializer &deserializer);
+
+protected:
+	void ResolveTypes() override;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/operator/logical_copy_to_file.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/filename_pattern.hpp"
 #include "duckdb/common/local_file_system.hpp"
+#include "duckdb/common/optional_idx.hpp"
 #include "duckdb/function/copy_function.hpp"
 #include "duckdb/planner/logical_operator.hpp"
 
@@ -31,8 +32,10 @@ public:
 	std::string file_path;
 	bool use_tmp_file;
 	FilenamePattern filename_pattern;
+	string file_extension;
 	bool overwrite_or_ignore;
 	bool per_thread_output;
+	optional_idx file_size_bytes;
 
 	bool partition_output;
 	vector<idx_t> partition_columns;

package/src/duckdb/src/include/duckdb/planner/operator/logical_create_secret.hpp

@@ -0,0 +1,43 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// planner/operator/logical_create_secret.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/parser/parsed_data/create_secret_info.hpp"
+#include "duckdb/planner/logical_operator.hpp"
+
+namespace duckdb {
+
+//! LogicalCreateSecret represents a simple logical operator that only passes on the parse info
+class LogicalCreateSecret : public LogicalOperator {
+public:
+	static constexpr const LogicalOperatorType TYPE = LogicalOperatorType::LOGICAL_CREATE_SECRET;
+
+public:
+	LogicalCreateSecret(CreateSecretFunction function_p, CreateSecretInfo info_p)
+	    : LogicalOperator(LogicalOperatorType::LOGICAL_CREATE_SECRET), info(std::move(info_p)) {
+	}
+
+	CreateSecretInfo info;
+
+public:
+	idx_t EstimateCardinality(ClientContext &context) override {
+		return 1;
+	};
+
+	//! Skips the serialization check in VerifyPlan
+	bool SupportSerialization() const override {
+		return false;
+	}
+
+protected:
+	void ResolveTypes() override {
+		types.emplace_back(LogicalType::BOOLEAN);
+	}
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp

@@ -8,7 +8,6 @@
 
 #pragma once
 
-#include "duckdb/common/types/chunk_collection.hpp"
 #include "duckdb/planner/logical_operator.hpp"
 #include "duckdb/common/enums/cte_materialize.hpp"
 
@@ -23,7 +22,7 @@ public:
 	LogicalCTERef(idx_t table_index, idx_t cte_index, vector<LogicalType> types, vector<string> colnames,
 	              CTEMaterialize materialized_cte)
 	    : LogicalOperator(LogicalOperatorType::LOGICAL_CTE_REF), table_index(table_index), cte_index(cte_index),
-	      materialized_cte(materialized_cte) {
+	      correlated_columns(0), materialized_cte(materialized_cte) {
 		D_ASSERT(types.size() > 0);
 		chunk_types = types;
 		bound_columns = colnames;
@@ -36,6 +35,8 @@ public:
 	idx_t cte_index;
 	//! The types of the chunk
 	vector<LogicalType> chunk_types;
+	//! Number of correlated columns
+	idx_t correlated_columns;
 	//! Does this operator read a materialized CTE?
 	CTEMaterialize materialized_cte;
 

package/src/duckdb/src/include/duckdb/planner/operator/logical_pragma.hpp

@@ -8,27 +8,22 @@
 
 #pragma once
 
-#include "duckdb/function/pragma_function.hpp"
-#include "duckdb/parser/parsed_data/pragma_info.hpp"
 #include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/parser/parsed_data/bound_pragma_info.hpp"
 
 namespace duckdb {
 
-//! LogicalSimple represents a simple logical operator that only passes on the parse info
+//! LogicalPragma represents a simple logical operator that only passes on the parse info
 class LogicalPragma : public LogicalOperator {
 public:
 	static constexpr const LogicalOperatorType TYPE = LogicalOperatorType::LOGICAL_PRAGMA;
 
 public:
-	LogicalPragma(PragmaFunction function_p, PragmaInfo info_p)
-	    : LogicalOperator(LogicalOperatorType::LOGICAL_PRAGMA), function(std::move(function_p)),
-	      info(std::move(info_p)) {
+	LogicalPragma(unique_ptr<BoundPragmaInfo> info_p)
+	    : LogicalOperator(LogicalOperatorType::LOGICAL_PRAGMA), info(std::move(info_p)) {
 	}
 
-	//! The pragma function to call
-	PragmaFunction function;
-	//! The context of the call
-	PragmaInfo info;
+	unique_ptr<BoundPragmaInfo> info;
 
 public:
 	idx_t EstimateCardinality(ClientContext &context) override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/planner/binder.hpp"
 
 namespace duckdb {
 
@@ -32,6 +33,7 @@ public:
 	string ctename;
 	idx_t table_index;
 	idx_t column_count;
+	vector<CorrelatedColumnInfo> correlated_columns;
 
 public:
 	vector<ColumnBinding> GetColumnBindings() override {

package/src/duckdb/src/include/duckdb/planner/operator/logical_set_operation.hpp

@@ -13,8 +13,10 @@
 namespace duckdb {
 
 class LogicalSetOperation : public LogicalOperator {
-	LogicalSetOperation(idx_t table_index, idx_t column_count, LogicalOperatorType type)
-	    : LogicalOperator(type), table_index(table_index), column_count(column_count) {
+	LogicalSetOperation(idx_t table_index, idx_t column_count, LogicalOperatorType type, bool setop_all,
+	                    bool allow_out_of_order)
+	    : LogicalOperator(type), table_index(table_index), column_count(column_count), setop_all(setop_all),
+	      allow_out_of_order(allow_out_of_order) {
 	}
 
 public:
@@ -22,8 +24,10 @@ public:
 
 public:
 	LogicalSetOperation(idx_t table_index, idx_t column_count, unique_ptr<LogicalOperator> top,
-	                    unique_ptr<LogicalOperator> bottom, LogicalOperatorType type)
-	    : LogicalOperator(type), table_index(table_index), column_count(column_count) {
+	                    unique_ptr<LogicalOperator> bottom, LogicalOperatorType type, bool setop_all,
+	                    bool allow_out_of_order = true)
+	    : LogicalOperator(type), table_index(table_index), column_count(column_count), setop_all(setop_all),
+	      allow_out_of_order(allow_out_of_order) {
 		D_ASSERT(type == LogicalOperatorType::LOGICAL_UNION || type == LogicalOperatorType::LOGICAL_EXCEPT ||
 		         type == LogicalOperatorType::LOGICAL_INTERSECT);
 		children.push_back(std::move(top));
@@ -32,6 +36,9 @@ public:
 
 	idx_t table_index;
 	idx_t column_count;
+	bool setop_all;
+	//! Whether or not UNION statements can be executed out of order
+	bool allow_out_of_order;
 
 public:
 	vector<ColumnBinding> GetColumnBindings() override {

package/src/duckdb/src/include/duckdb/planner/parsed_data/bound_create_table_info.hpp

@@ -16,8 +16,8 @@
 #include "duckdb/common/case_insensitive_map.hpp"
 #include "duckdb/catalog/catalog_entry/table_column_type.hpp"
 #include "duckdb/catalog/catalog_entry/column_dependency_manager.hpp"
-#include "duckdb/storage/table/table_index_list.hpp"
 #include "duckdb/catalog/dependency_list.hpp"
+#include "duckdb/storage/table_storage_info.hpp"
 
 namespace duckdb {
 class CatalogEntry;
@@ -46,8 +46,8 @@ struct BoundCreateTableInfo {
 	unique_ptr<PersistentTableData> data;
 	//! CREATE TABLE from QUERY
 	unique_ptr<LogicalOperator> query;
-	//! Indexes created by this table <Block_ID, Offset>
-	vector<BlockPointer> indexes;
+	//! Indexes created by this table
+	vector<IndexStorageInfo> indexes;
 
 	CreateTableInfo &Base() {
 		D_ASSERT(base);

package/src/duckdb/src/include/duckdb/planner/pragma_handler.hpp

@@ -31,7 +31,7 @@ private:
 private:
 	//! Handles a pragma statement, returns whether the statement was expanded, if it was expanded the 'resulting_query'
 	//! contains the statement(s) to replace the current one
-	bool HandlePragma(SQLStatement *statement, string &resulting_query);
+	bool HandlePragma(SQLStatement &statement, string &resulting_query);
 
 	void HandlePragmaStatementsInternal(vector<unique_ptr<SQLStatement>> &statements);
 };

package/src/duckdb/src/include/duckdb/planner/query_node/bound_set_operation_node.hpp

@@ -25,6 +25,8 @@ public:
 
 	//! The type of set operation
 	SetOperationType setop_type = SetOperationType::NONE;
+	//! whether the ALL modifier was used or not
+	bool setop_all = false;
 	//! The left side of the set operation
 	unique_ptr<BoundQueryNode> left;
 	//! The right side of the set operation

package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp

@@ -25,6 +25,9 @@ struct FlattenDependentJoins {
 	//! has_correlated_expressions map.
 	bool DetectCorrelatedExpressions(LogicalOperator *op, bool lateral = false, idx_t lateral_depth = 0);
 
+	//! Mark entire subtree of Logical Operators as correlated by adding them to the has_correlated_expressions map.
+	bool MarkSubtreeCorrelated(LogicalOperator &op);
+
 	//! Push the dependent join down a LogicalOperator
 	unique_ptr<LogicalOperator> PushDownDependentJoin(unique_ptr<LogicalOperator> plan);
 

package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_cte_scan.hpp

@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/planner/subquery/rewrite_cte_scan.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/planner/binder.hpp"
+#include "duckdb/planner/column_binding_map.hpp"
+#include "duckdb/planner/logical_operator.hpp"
+
+namespace duckdb {
+
+//! Helper class to rewrite correlated cte scans within a single LogicalOperator
+class RewriteCTEScan : public LogicalOperatorVisitor {
+public:
+	RewriteCTEScan(idx_t table_index, const vector<CorrelatedColumnInfo> &correlated_columns);
+
+	void VisitOperator(LogicalOperator &op) override;
+
+private:
+	idx_t table_index;
+	const vector<CorrelatedColumnInfo> &correlated_columns;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/table_binding.hpp

@@ -52,7 +52,7 @@ public:
 	bool TryGetBindingIndex(const string &column_name, column_t &column_index);
 	column_t GetBindingIndex(const string &column_name);
 	bool HasMatchingBinding(const string &column_name);
-	virtual string ColumnNotFoundError(const string &column_name) const;
+	virtual ErrorData ColumnNotFoundError(const string &column_name) const;
 	virtual BindResult Bind(ColumnRefExpression &colref, idx_t depth);
 	virtual optional_ptr<StandardEntry> GetStandardEntry();
 
@@ -107,7 +107,7 @@ public:
 	unique_ptr<ParsedExpression> ExpandGeneratedColumn(const string &column_name);
 	BindResult Bind(ColumnRefExpression &colref, idx_t depth) override;
 	optional_ptr<StandardEntry> GetStandardEntry() override;
-	string ColumnNotFoundError(const string &column_name) const override;
+	ErrorData ColumnNotFoundError(const string &column_name) const override;
 	// These are columns that are present in the name_map, appearing in the order that they're bound
 	const vector<column_t> &GetBoundColumnIds() const;
 
@@ -115,8 +115,8 @@ protected:
 	ColumnBinding GetColumnBinding(column_t column_index);
 };
 
-//! DummyBinding is like the Binding, except the alias and index are set by default. Used for binding lambdas and macro
-//! parameters.
+//! DummyBinding is like the Binding, except the alias and index are set by default.
+//! Used for binding lambdas and macro parameters.
 struct DummyBinding : public Binding {
 public:
 	static constexpr const BindingType TYPE = BindingType::DUMMY;
@@ -124,16 +124,18 @@ public:
 	static constexpr const char *DUMMY_NAME = "0_macro_parameters";
 
 public:
-	DummyBinding(vector<LogicalType> types_p, vector<string> names_p, string dummy_name_p);
+	DummyBinding(vector<LogicalType> types, vector<string> names, string dummy_name);
 
-	//! Arguments
+	//! Arguments (for macros)
 	vector<unique_ptr<ParsedExpression>> *arguments;
 	//! The name of the dummy binding
 	string dummy_name;
 
 public:
+	//! Binding macros
 	BindResult Bind(ColumnRefExpression &colref, idx_t depth) override;
-	BindResult Bind(ColumnRefExpression &colref, idx_t lambda_index, idx_t depth);
+	//! Binding lambdas
+	BindResult Bind(LambdaRefExpression &lambdaref, idx_t depth);
 
 	//! Given the parameter colref, returns a copy of the argument that was supplied for this parameter
 	unique_ptr<ParsedExpression> ParamToArg(ColumnRefExpression &colref);

package/src/duckdb/src/include/duckdb/planner/table_filter.hpp

@@ -21,7 +21,8 @@ enum class TableFilterType : uint8_t {
 	IS_NULL = 1,
 	IS_NOT_NULL = 2,
 	CONJUNCTION_OR = 3,
-	CONJUNCTION_AND = 4
+	CONJUNCTION_AND = 4,
+	STRUCT_EXTRACT = 5
 };
 
 //! TableFilter represents a filter pushed down into the table scan.

package/src/duckdb/src/include/duckdb/planner/tableref/bound_dummytableref.hpp

@@ -15,10 +15,11 @@ namespace duckdb {
 //! Represents a cross product
 class BoundEmptyTableRef : public BoundTableRef {
 public:
-	static constexpr const TableReferenceType TYPE = TableReferenceType::EMPTY;
+	static constexpr const TableReferenceType TYPE = TableReferenceType::EMPTY_FROM;
 
 public:
-	explicit BoundEmptyTableRef(idx_t bind_index) : BoundTableRef(TableReferenceType::EMPTY), bind_index(bind_index) {
+	explicit BoundEmptyTableRef(idx_t bind_index)
+	    : BoundTableRef(TableReferenceType::EMPTY_FROM), bind_index(bind_index) {
 	}
 	idx_t bind_index;
 };

package/src/duckdb/src/include/duckdb/storage/buffer/block_handle.hpp

@@ -13,6 +13,7 @@
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/storage/storage_info.hpp"
 #include "duckdb/common/file_buffer.hpp"
+#include "duckdb/common/enums/memory_tag.hpp"
 
 namespace duckdb {
 class BlockManager;
@@ -23,10 +24,11 @@ class DatabaseInstance;
 enum class BlockState : uint8_t { BLOCK_UNLOADED = 0, BLOCK_LOADED = 1 };
 
 struct BufferPoolReservation {
+	MemoryTag tag;
 	idx_t size {0};
 	BufferPool &pool;
 
-	BufferPoolReservation(BufferPool &pool);
+	BufferPoolReservation(MemoryTag tag, BufferPool &pool);
 	BufferPoolReservation(const BufferPoolReservation &) = delete;
 	BufferPoolReservation &operator=(const BufferPoolReservation &) = delete;
 
@@ -40,7 +42,7 @@ struct BufferPoolReservation {
 };
 
 struct TempBufferPoolReservation : BufferPoolReservation {
-	TempBufferPoolReservation(BufferPool &pool, idx_t size) : BufferPoolReservation(pool) {
+	TempBufferPoolReservation(MemoryTag tag, BufferPool &pool, idx_t size) : BufferPoolReservation(tag, pool) {
 		Resize(size);
 	}
 	TempBufferPoolReservation(TempBufferPoolReservation &&) = default;
@@ -58,9 +60,9 @@ class BlockHandle {
 	friend class BufferPool;
 
 public:
-	BlockHandle(BlockManager &block_manager, block_id_t block_id);
-	BlockHandle(BlockManager &block_manager, block_id_t block_id, unique_ptr<FileBuffer> buffer, bool can_destroy,
-	            idx_t block_size, BufferPoolReservation &&reservation);
+	BlockHandle(BlockManager &block_manager, block_id_t block_id, MemoryTag tag);
+	BlockHandle(BlockManager &block_manager, block_id_t block_id, MemoryTag tag, unique_ptr<FileBuffer> buffer,
+	            bool can_destroy, idx_t block_size, BufferPoolReservation &&reservation);
 	~BlockHandle();
 
 	BlockManager &block_manager;
@@ -115,6 +117,8 @@ private:
 	atomic<int32_t> readers;
 	//! The block id of the block
 	const block_id_t block_id;
+	//! Memory tag
+	MemoryTag tag;
 	//! Pointer to loaded data (if any)
 	unique_ptr<FileBuffer> buffer;
 	//! Internal eviction timestamp

package/src/duckdb/src/include/duckdb/storage/buffer/buffer_pool.hpp

@@ -1,11 +1,20 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/buffer/buffer_pool.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
 #pragma once
 
-#include "duckdb/common/mutex.hpp"
 #include "duckdb/common/file_buffer.hpp"
+#include "duckdb/common/mutex.hpp"
 #include "duckdb/storage/buffer/block_handle.hpp"
 
 namespace duckdb {
 
+class TemporaryMemoryManager;
 struct EvictionQueue;
 
 struct BufferEvictionNode {
@@ -40,11 +49,15 @@ public:
 	//! blocks can be evicted
 	void SetLimit(idx_t limit, const char *exception_postscript);
 
-	void IncreaseUsedMemory(idx_t size);
+	void IncreaseUsedMemory(MemoryTag tag, idx_t size);
 
-	idx_t GetUsedMemory();
+	idx_t GetUsedMemory() const;
 
-	idx_t GetMaxMemory();
+	idx_t GetMaxMemory() const;
+
+	virtual idx_t GetQueryMaxMemory() const;
+
+	TemporaryMemoryManager &GetTemporaryMemoryManager();
 
 protected:
 	//! Evict blocks until the currently used memory + extra_memory fit, returns false if this was not possible
@@ -57,14 +70,14 @@ protected:
 		bool success;
 		TempBufferPoolReservation reservation;
 	};
-	virtual EvictionResult EvictBlocks(idx_t extra_memory, idx_t memory_limit,
+	virtual EvictionResult EvictBlocks(MemoryTag tag, idx_t extra_memory, idx_t memory_limit,
 	                                   unique_ptr<FileBuffer> *buffer = nullptr);
 
 	//! Garbage collect eviction queue
 	void PurgeQueue();
 	void AddToEvictionQueue(shared_ptr<BlockHandle> &handle);
 
-private:
+protected:
 	//! The lock for changing the memory limit
 	mutex limit_lock;
 	//! The current amount of memory that is occupied by the buffer manager (in bytes)
@@ -75,6 +88,10 @@ private:
 	unique_ptr<EvictionQueue> queue;
 	//! Total number of insertions into the eviction queue. This guides the schedule for calling PurgeQueue.
 	atomic<uint32_t> queue_insertions;
+	//! Memory manager for concurrently used temporary memory, e.g., for physical operators
+	unique_ptr<TemporaryMemoryManager> temporary_memory_manager;
+	//! Memory usage per tag
+	atomic<idx_t> memory_usage_per_tag[MEMORY_TAG_COUNT];
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/buffer/temporary_file_information.hpp

@@ -1,9 +1,16 @@
 #pragma once
 
 #include "duckdb/common/common.hpp"
+#include "duckdb/common/enums/memory_tag.hpp"
 
 namespace duckdb {
 
+struct MemoryInformation {
+	MemoryTag tag;
+	idx_t size;
+	idx_t evicted_data;
+};
+
 struct TemporaryFileInformation {
 	string path;
 	idx_t size;

package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/storage/buffer/buffer_handle.hpp"
 #include "duckdb/storage/block_manager.hpp"
 #include "duckdb/common/file_system.hpp"
+#include "duckdb/common/enums/memory_tag.hpp"
 #include "duckdb/storage/buffer/temporary_file_information.hpp"
 #include "duckdb/main/config.hpp"
 
@@ -19,6 +20,7 @@ namespace duckdb {
 
 class Allocator;
 class BufferPool;
+class TemporaryMemoryManager;
 
 class BufferManager {
 	friend class BufferHandle;
@@ -33,7 +35,7 @@ public:
 
 public:
 	static unique_ptr<BufferManager> CreateStandardBufferManager(DatabaseInstance &db, DBConfig &config);
-	virtual BufferHandle Allocate(idx_t block_size, bool can_destroy = true,
+	virtual BufferHandle Allocate(MemoryTag tag, idx_t block_size, bool can_destroy = true,
 	                              shared_ptr<BlockHandle> *block = nullptr) = 0;
 	//! Reallocate an in-memory buffer that is pinned.
 	virtual void ReAllocate(shared_ptr<BlockHandle> &handle, idx_t block_size) = 0;
@@ -47,6 +49,7 @@ public:
 	virtual DUCKDB_API Allocator &GetBufferAllocator();
 	virtual DUCKDB_API void ReserveMemory(idx_t size);
 	virtual DUCKDB_API void FreeReservedMemory(idx_t size);
+	virtual vector<MemoryInformation> GetMemoryUsageInfo() const = 0;
 	//! Set a new memory limit to the buffer manager, throws an exception if the new limit is too low and not enough
 	//! blocks can be evicted
 	virtual void SetLimit(idx_t limit = (idx_t)-1);
@@ -59,7 +62,9 @@ public:
 	virtual unique_ptr<FileBuffer> ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source,
 	                                                      FileBufferType type = FileBufferType::MANAGED_BUFFER);
 	//! Get the underlying buffer pool responsible for managing the buffers
-	virtual BufferPool &GetBufferPool();
+	virtual BufferPool &GetBufferPool() const;
+	//! Get the manager that assigns reservations for temporary memory, e.g., for query intermediates
+	virtual TemporaryMemoryManager &GetTemporaryMemoryManager();
 
 	// Static methods
 	DUCKDB_API static BufferManager &GetBufferManager(DatabaseInstance &db);
@@ -69,12 +74,14 @@ public:
 	static idx_t GetAllocSize(idx_t block_size) {
 		return AlignValue<idx_t, Storage::SECTOR_SIZE>(block_size + Storage::BLOCK_HEADER_SIZE);
 	}
+	//! Returns the maximum available memory for a given query
+	idx_t GetQueryMaxMemory() const;
 
 protected:
 	virtual void PurgeQueue() = 0;
 	virtual void AddToEvictionQueue(shared_ptr<BlockHandle> &handle);
-	virtual void WriteTemporaryBuffer(block_id_t block_id, FileBuffer &buffer);
-	virtual unique_ptr<FileBuffer> ReadTemporaryBuffer(block_id_t id, unique_ptr<FileBuffer> buffer);
+	virtual void WriteTemporaryBuffer(MemoryTag tag, block_id_t block_id, FileBuffer &buffer);
+	virtual unique_ptr<FileBuffer> ReadTemporaryBuffer(MemoryTag tag, block_id_t id, unique_ptr<FileBuffer> buffer);
 	virtual void DeleteTemporaryFile(block_id_t id);
 };
 

package/src/duckdb/src/include/duckdb/storage/checkpoint/row_group_writer.hpp

@@ -34,9 +34,6 @@ public:
 
 	virtual MetadataWriter &GetPayloadWriter() = 0;
 
-	void RegisterPartialBlock(PartialBlockAllocation &&allocation);
-	PartialBlockAllocation GetBlockAllocation(uint32_t segment_size);
-
 	PartialBlockManager &GetPartialBlockManager() {
 		return partial_block_manager;
 	}

package/src/duckdb/src/include/duckdb/storage/checkpoint/string_checkpoint_state.hpp

@@ -39,7 +39,7 @@ struct string_location_t {
 	string_location_t() {
 	}
 	bool IsValid() {
-		return offset < Storage::BLOCK_SIZE && (block_id == INVALID_BLOCK || block_id >= MAXIMUM_BLOCK);
+		return offset < int32_t(Storage::BLOCK_SIZE) && (block_id == INVALID_BLOCK || block_id >= MAXIMUM_BLOCK);
 	}
 	block_id_t block_id;
 	int32_t offset;

package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp

@@ -31,15 +31,16 @@ public:
 
 	CompressionType GetColumnCompressionType(idx_t i);
 
-	virtual void FinalizeTable(TableStatistics &&global_stats, DataTableInfo *info,
-	                           Serializer &metadata_serializer) = 0;
+	virtual void FinalizeTable(TableStatistics &&global_stats, DataTableInfo *info, Serializer &serializer) = 0;
 	virtual unique_ptr<RowGroupWriter> GetRowGroupWriter(RowGroup &row_group) = 0;
 
 	virtual void AddRowGroup(RowGroupPointer &&row_group_pointer, unique_ptr<RowGroupWriter> &&writer);
 
+	TaskScheduler &GetScheduler();
+
 protected:
 	DuckTableEntry &table;
-	// Pointers to the start of each row group.
+	//! Pointers to the start of each row group.
 	vector<RowGroupPointer> row_group_pointers;
 };
 
@@ -49,13 +50,12 @@ public:
 	                          MetadataWriter &table_data_writer);
 
 public:
-	virtual void FinalizeTable(TableStatistics &&global_stats, DataTableInfo *info,
-	                           Serializer &metadata_serializer) override;
-	virtual unique_ptr<RowGroupWriter> GetRowGroupWriter(RowGroup &row_group) override;
+	void FinalizeTable(TableStatistics &&global_stats, DataTableInfo *info, Serializer &serializer) override;
+	unique_ptr<RowGroupWriter> GetRowGroupWriter(RowGroup &row_group) override;
 
 private:
 	SingleFileCheckpointWriter &checkpoint_manager;
-	// Writes the actual table data
+	//! Writes the actual table data
 	MetadataWriter &table_data_writer;
 };
 

package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp

@@ -45,7 +45,7 @@ protected:
 	virtual void WriteSequence(SequenceCatalogEntry &table, Serializer &serializer);
 	virtual void WriteMacro(ScalarMacroCatalogEntry &table, Serializer &serializer);
 	virtual void WriteTableMacro(TableMacroCatalogEntry &table, Serializer &serializer);
-	virtual void WriteIndex(IndexCatalogEntry &index_catalog, Serializer &serializer);
+	virtual void WriteIndex(IndexCatalogEntry &index_catalog_entry, Serializer &serializer);
 	virtual void WriteType(TypeCatalogEntry &type, Serializer &serializer);
 };