duckdb 1.3.2 → 1.3.3-dev4.0

package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp

@@ -58,6 +58,8 @@ public:
 	ExtraOperatorInfo extra_info;
 	//! Contains a reference to dynamically generated table filters (through e.g. a join up in the tree)
 	shared_ptr<DynamicTableFilterSet> dynamic_filters;
+	//! Information for WITH ORDINALITY
+	optional_idx ordinality_idx;
 
 	string GetName() const override;
 	InsertionOrderPreservingMap<string> ParamsToString() const override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_insert.hpp

@@ -17,28 +17,8 @@ class TableCatalogEntry;
 
 class Index;
 
-//! LogicalInsert represents an insertion of data into a base table
-class LogicalInsert : public LogicalOperator {
-public:
-	static constexpr const LogicalOperatorType TYPE = LogicalOperatorType::LOGICAL_INSERT;
-
-public:
-	LogicalInsert(TableCatalogEntry &table, idx_t table_index);
-
-	vector<vector<unique_ptr<Expression>>> insert_values;
-	//! The insertion map ([table_index -> index in result, or DConstants::INVALID_INDEX if not specified])
-	physical_index_vector_t<idx_t> column_index_map;
-	//! The expected types for the INSERT statement (obtained from the column types)
-	vector<LogicalType> expected_types;
-	//! The base table to insert into
-	TableCatalogEntry &table;
-	idx_t table_index;
-	//! if returning option is used, return actual chunk to projection
-	bool return_chunk;
-	//! The default statements used by the table
-	vector<unique_ptr<Expression>> bound_defaults;
-	//! The constraints used by the table
-	vector<unique_ptr<BoundConstraint>> bound_constraints;
+struct BoundOnConflictInfo {
+	BoundOnConflictInfo();
 
 	//! Which action to take on conflict
 	OnConflictAction action_type;
@@ -62,6 +42,32 @@ public:
 	vector<column_t> source_columns;
 	//! True, if the INSERT OR REPLACE requires delete + insert.
 	bool update_is_del_and_insert;
+};
+
+//! LogicalInsert represents an insertion of data into a base table
+class LogicalInsert : public LogicalOperator {
+public:
+	static constexpr const LogicalOperatorType TYPE = LogicalOperatorType::LOGICAL_INSERT;
+
+public:
+	LogicalInsert(TableCatalogEntry &table, idx_t table_index);
+
+	vector<vector<unique_ptr<Expression>>> insert_values;
+	//! The insertion map ([table_index -> index in result, or DConstants::INVALID_INDEX if not specified])
+	physical_index_vector_t<idx_t> column_index_map;
+	//! The expected types for the INSERT statement (obtained from the column types)
+	vector<LogicalType> expected_types;
+	//! The base table to insert into
+	TableCatalogEntry &table;
+	idx_t table_index;
+	//! if returning option is used, return actual chunk to projection
+	bool return_chunk;
+	//! The default statements used by the table
+	vector<unique_ptr<Expression>> bound_defaults;
+	//! The constraints used by the table
+	vector<unique_ptr<BoundConstraint>> bound_constraints;
+
+	BoundOnConflictInfo on_conflict_info;
 
 public:
 	void Serialize(Serializer &serializer) const override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_join.hpp

@@ -25,7 +25,7 @@ public:
 
 	// Gets the set of table references that are reachable from this node
 	static void GetTableReferences(LogicalOperator &op, unordered_set<idx_t> &bindings);
-	static void GetExpressionBindings(Expression &expr, unordered_set<idx_t> &bindings);
+	static void GetExpressionBindings(const Expression &expr, unordered_set<idx_t> &bindings);
 
 	bool HasProjectionMap() const override {
 		return !left_projection_map.empty() || !right_projection_map.empty();

package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp

@@ -22,11 +22,14 @@ public:
 
 public:
 	LogicalMaterializedCTE(string ctename_p, idx_t table_index, idx_t column_count, unique_ptr<LogicalOperator> cte,
-	                       unique_ptr<LogicalOperator> child)
+	                       unique_ptr<LogicalOperator> child, CTEMaterialize materialize)
 	    : LogicalCTE(std::move(ctename_p), table_index, column_count, std::move(cte), std::move(child),
-	                 LogicalOperatorType::LOGICAL_MATERIALIZED_CTE) {
+	                 LogicalOperatorType::LOGICAL_MATERIALIZED_CTE),
+	      materialize(materialize) {
 	}
 
+	CTEMaterialize materialize = CTEMaterialize::CTE_MATERIALIZE_ALWAYS;
+
 public:
 	InsertionOrderPreservingMap<string> ParamsToString() const override;
 	vector<ColumnBinding> GetColumnBindings() override {

package/src/duckdb/src/include/duckdb/planner/operator/logical_merge_into.hpp

@@ -0,0 +1,75 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/planner/operator/logical_merge_into.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/common/enums/merge_action_type.hpp"
+#include "duckdb/common/index_vector.hpp"
+
+namespace duckdb {
+class TableCatalogEntry;
+class LogicalGet;
+class LogicalProjection;
+
+class BoundMergeIntoAction {
+public:
+	//! The merge action type
+	MergeActionType action_type;
+	//! Condition - or NULL if this should always be performed for the given action
+	unique_ptr<Expression> condition;
+	//! The set of referenced physical columns (for UPDATE)
+	vector<PhysicalIndex> columns;
+	//! Set of expressions for INSERT or UPDATE
+	vector<unique_ptr<Expression>> expressions;
+	//! Column index map (for INSERT)
+	physical_index_vector_t<idx_t> column_index_map;
+	//! Whether or not an UPDATE is a DELETE + INSERT
+	bool update_is_del_and_insert = false;
+
+	void Serialize(Serializer &serializer) const;
+	static unique_ptr<BoundMergeIntoAction> Deserialize(Deserializer &deserializer);
+};
+
+class LogicalMergeInto : public LogicalOperator {
+public:
+	static constexpr const LogicalOperatorType TYPE = LogicalOperatorType::LOGICAL_MERGE_INTO;
+
+public:
+	explicit LogicalMergeInto(TableCatalogEntry &table);
+
+	//! The base table to merge into
+	TableCatalogEntry &table;
+	//! projection index
+	idx_t table_index;
+	vector<unique_ptr<Expression>> bound_defaults;
+	idx_t row_id_start;
+	optional_idx source_marker;
+	//! Bound constraints
+	vector<unique_ptr<BoundConstraint>> bound_constraints;
+	//! Whether or not to return the input data
+	bool return_chunk = false;
+
+	map<MergeActionCondition, vector<unique_ptr<BoundMergeIntoAction>>> actions;
+
+public:
+	void Serialize(Serializer &serializer) const override;
+	static unique_ptr<LogicalOperator> Deserialize(Deserializer &deserializer);
+
+	idx_t EstimateCardinality(ClientContext &context) override;
+	vector<idx_t> GetTableIndex() const override;
+
+protected:
+	vector<ColumnBinding> GetColumnBindings() override;
+	void ResolveTypes() override;
+
+private:
+	LogicalMergeInto(ClientContext &context, const unique_ptr<CreateInfo> &table_info);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp

@@ -36,6 +36,8 @@ public:
 	vector<unique_ptr<Expression>> key_targets;
 
 public:
+	InsertionOrderPreservingMap<string> ParamsToString() const override;
+
 	vector<ColumnBinding> GetColumnBindings() override {
 		return GenerateColumnBindings(table_index, column_count);
 	}

package/src/duckdb/src/include/duckdb/planner/operator/logical_update.hpp

@@ -26,7 +26,7 @@ public:
 
 	//! The base table to update
 	TableCatalogEntry &table;
-	//! table catalog index
+	//! projection index
 	idx_t table_index;
 	//! if returning option is used, return the update chunk
 	bool return_chunk;

package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp

@@ -35,6 +35,8 @@ public:
 	//! The binder used by the child side of the CTE
 	shared_ptr<Binder> child_binder;
 
+	CTEMaterialize materialized = CTEMaterialize::CTE_MATERIALIZE_DEFAULT;
+
 public:
 	idx_t GetRootIndex() override {
 		return child->GetRootIndex();

package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/allocator.hpp"
 #include "duckdb/common/common.hpp"
+#include "duckdb/common/types/string.hpp"
 
 namespace duckdb {
 
@@ -77,6 +78,31 @@ public:
 		return arena_allocator;
 	}
 
+	template <class T, class... ARGS>
+	T *Make(ARGS &&... args) {
+		auto mem = AllocateAligned(sizeof(T));
+		return new (mem) T(std::forward<ARGS>(args)...);
+	}
+
+	String MakeString(const char *data, const size_t len) {
+		data_ptr_t mem = nullptr;
+
+		D_ASSERT(len < NumericLimits<uint32_t>::Maximum());
+		const auto size = static_cast<uint32_t>(len);
+		if (!String::CanBeInlined(size)) {
+			// If the string can't be inlined, we allocate it on the arena allocator
+			mem = AllocateAligned(sizeof(char) * size + 1); // +1 for null terminator
+			memcpy(mem, data, size);
+			mem[size] = '\0';
+		}
+
+		return String::Reference(mem ? reinterpret_cast<char *>(mem) : data, size);
+	}
+
+	String MakeString(const std::string &data) {
+		return MakeString(data.c_str(), data.size());
+	}
+
 private:
 	void AllocateNewBlock(idx_t min_size);
 

package/src/duckdb/src/include/duckdb/storage/block.hpp

@@ -22,7 +22,7 @@ public:
 	Block(Allocator &allocator, const block_id_t id, const idx_t block_size, const idx_t block_header_size);
 	Block(Allocator &allocator, block_id_t id, uint32_t internal_size, idx_t block_header_size);
 	Block(Allocator &allocator, const block_id_t id, BlockManager &block_manager);
-	Block(FileBuffer &source, block_id_t id);
+	Block(FileBuffer &source, block_id_t id, idx_t block_header_size);
 
 	block_id_t id;
 };

package/src/duckdb/src/include/duckdb/storage/block_manager.hpp

@@ -11,11 +11,12 @@
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/common/optional_idx.hpp"
+#include "duckdb/common/unordered_map.hpp"
 #include "duckdb/storage/block.hpp"
 #include "duckdb/storage/storage_info.hpp"
-#include "duckdb/common/unordered_map.hpp"
 
 namespace duckdb {
+
 class BlockHandle;
 class BufferHandle;
 class BufferManager;
@@ -57,17 +58,19 @@ public:
 	//! Get the first meta block id
 	virtual idx_t GetMetaBlock() = 0;
 	//! Read the content of the block from disk
-	virtual void Read(Block &block) = 0;
+	virtual void Read(QueryContext context, Block &block) = 0;
+
 	//! Read the content of the block from disk
 	virtual void ReadBlocks(FileBuffer &buffer, block_id_t start_block, idx_t block_count) = 0;
-	//! Writes the block to disk
+	//! Writes the block to disk.
 	virtual void Write(FileBuffer &block, block_id_t block_id) = 0;
-	//! Writes the block to disk
+	virtual void Write(QueryContext context, FileBuffer &block, block_id_t block_id);
+	//! Writes the block to disk.
 	void Write(Block &block) {
 		Write(block, block.id);
 	}
 	//! Write the header; should be the final step of a checkpoint
-	virtual void WriteHeader(DatabaseHeader header) = 0;
+	virtual void WriteHeader(QueryContext context, DatabaseHeader header) = 0;
 
 	//! Returns the number of total blocks
 	virtual idx_t TotalBlocks() = 0;
@@ -88,9 +91,10 @@ public:
 	//! Register a block with the given block id in the base file
 	shared_ptr<BlockHandle> RegisterBlock(block_id_t block_id);
 	//! Convert an existing in-memory buffer into a persistent disk-backed block
-	shared_ptr<BlockHandle> ConvertToPersistent(block_id_t block_id, shared_ptr<BlockHandle> old_block,
-	                                            BufferHandle old_handle);
-	shared_ptr<BlockHandle> ConvertToPersistent(block_id_t block_id, shared_ptr<BlockHandle> old_block);
+	shared_ptr<BlockHandle> ConvertToPersistent(QueryContext context, block_id_t block_id,
+	                                            shared_ptr<BlockHandle> old_block, BufferHandle old_handle);
+	shared_ptr<BlockHandle> ConvertToPersistent(QueryContext context, block_id_t block_id,
+	                                            shared_ptr<BlockHandle> old_block);
 
 	void UnregisterBlock(BlockHandle &block);
 	//! UnregisterBlock, only accepts non-temporary block ids
@@ -143,6 +147,18 @@ public:
 	virtual void VerifyBlocks(const unordered_map<block_id_t, idx_t> &block_usage_count) {
 	}
 
+public:
+	template <class TARGET>
+	TARGET &Cast() {
+		DynamicCastCheck<TARGET>(this);
+		return reinterpret_cast<TARGET &>(*this);
+	}
+	template <class TARGET>
+	const TARGET &Cast() const {
+		DynamicCastCheck<TARGET>(this);
+		return reinterpret_cast<const TARGET &>(*this);
+	}
+
 private:
 	//! The lock for the set of blocks
 	mutex blocks_lock;

package/src/duckdb/src/include/duckdb/storage/buffer/block_handle.hpp

@@ -162,7 +162,9 @@ public:
 
 	//! Resize the actual buffer
 	void ResizeBuffer(BlockLock &, idx_t block_size, int64_t memory_delta);
-	BufferHandle Load(unique_ptr<FileBuffer> buffer = nullptr);
+
+	BufferHandle Load(QueryContext context, unique_ptr<FileBuffer> buffer = nullptr);
+
 	BufferHandle LoadFromBuffer(BlockLock &l, data_ptr_t data, unique_ptr<FileBuffer> reusable_buffer,
 	                            BufferPoolReservation reservation);
 	unique_ptr<FileBuffer> UnloadAndTakeBlock(BlockLock &);

package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp

@@ -8,13 +8,13 @@
 
 #pragma once
 
-#include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/buffer/buffer_handle.hpp"
-#include "duckdb/storage/block_manager.hpp"
-#include "duckdb/common/file_system.hpp"
 #include "duckdb/common/enums/memory_tag.hpp"
-#include "duckdb/storage/buffer/temporary_file_information.hpp"
+#include "duckdb/common/file_system.hpp"
 #include "duckdb/main/config.hpp"
+#include "duckdb/storage/block_manager.hpp"
+#include "duckdb/storage/buffer/buffer_handle.hpp"
+#include "duckdb/storage/buffer/temporary_file_information.hpp"
+#include "duckdb/storage/buffer_manager.hpp"
 
 namespace duckdb {
 
@@ -34,85 +34,104 @@ public:
 	}
 
 public:
+	DUCKDB_API static BufferManager &GetBufferManager(DatabaseInstance &db);
+	DUCKDB_API static const BufferManager &GetBufferManager(const DatabaseInstance &db);
+	DUCKDB_API static BufferManager &GetBufferManager(ClientContext &context);
+	DUCKDB_API static const BufferManager &GetBufferManager(const ClientContext &context);
+	DUCKDB_API static BufferManager &GetBufferManager(AttachedDatabase &db);
+	DUCKDB_API static idx_t GetAllocSize(const idx_t alloc_size);
+
+public:
+	//! Allocate temporary memory of size block_size.
 	virtual shared_ptr<BlockHandle> AllocateTemporaryMemory(MemoryTag tag, idx_t block_size,
 	                                                        bool can_destroy = true) = 0;
+	//! Allocate block-based memory.
+	//! The block manager provides the block size, and the block header size.
+	//! Returns the BlockHandle managing the registered memory.
 	virtual shared_ptr<BlockHandle> AllocateMemory(MemoryTag tag, BlockManager *block_manager,
 	                                               bool can_destroy = true) = 0;
+	//! Allocate (temporary) memory of size block_size, and pin it.
 	virtual BufferHandle Allocate(MemoryTag tag, idx_t block_size, bool can_destroy = true) = 0;
+	//! Allocate block-based memory and pin it.
 	virtual BufferHandle Allocate(MemoryTag tag, BlockManager *block_manager, bool can_destroy = true) = 0;
-	//! Reallocate an in-memory buffer that is pinned.
+	//! Reallocate a pinned in-memory buffer.
 	virtual void ReAllocate(shared_ptr<BlockHandle> &handle, idx_t block_size) = 0;
+	//! Pin a block handle.
 	virtual BufferHandle Pin(shared_ptr<BlockHandle> &handle) = 0;
-	//! Prefetch a series of blocks. Note that this is a performance suggestion.
+	//! Pre-fetch a series of blocks.
+	//! Using this function is a performance suggestion.
 	virtual void Prefetch(vector<shared_ptr<BlockHandle>> &handles) = 0;
+	//! Unpin a block handle.
 	virtual void Unpin(shared_ptr<BlockHandle> &handle) = 0;
 
-	//! Returns the currently allocated memory
+	//! Returns the currently allocated memory.
 	virtual idx_t GetUsedMemory() const = 0;
-	//! Returns the maximum available memory
+	//! Returns the maximum available memory.
 	virtual idx_t GetMaxMemory() const = 0;
-	//! Returns the currently used swap space
+	//! Returns the currently used swap space.
 	virtual idx_t GetUsedSwap() const = 0;
-	//! Returns the maximum swap space that can be used
+	//! Returns the maximum swap space that can be used.
 	virtual optional_idx GetMaxSwap() const = 0;
 	//! Returns the block allocation size for buffer-managed blocks.
 	virtual idx_t GetBlockAllocSize() const = 0;
 	//! Returns the block size for buffer-managed blocks.
 	virtual idx_t GetBlockSize() const = 0;
-	//! Returns the block header size for buffer-managed blocks.
-	virtual idx_t GetTemporaryBlockHeaderSize() const = 0;
+	//! Returns the maximum available memory for a given query.
+	virtual idx_t GetQueryMaxMemory() const = 0;
 
-	//! Returns a new block of transient memory.
+	//! Returns a newly registered block of transient memory.
 	virtual shared_ptr<BlockHandle> RegisterTransientMemory(const idx_t size, BlockManager &block_manager);
-	//! Returns a new block of memory that is smaller than the block size setting.
+	//! Returns a newly registered block of memory that is smaller than the block size setting.
 	virtual shared_ptr<BlockHandle> RegisterSmallMemory(const idx_t size);
+	//! Returns a newly registered block of memory that is smaller than the block size setting and has a memory tag.
 	virtual shared_ptr<BlockHandle> RegisterSmallMemory(MemoryTag tag, const idx_t size);
 
+	//! Get the buffer allocator.
 	virtual DUCKDB_API Allocator &GetBufferAllocator();
+	//! Reserve memory.
 	virtual DUCKDB_API void ReserveMemory(idx_t size);
+	//! Free reserved memory.
 	virtual DUCKDB_API void FreeReservedMemory(idx_t size);
+	//! GetMemoryUsageInfo returns MemoryInformation for each memory tag.
 	virtual vector<MemoryInformation> GetMemoryUsageInfo() const = 0;
-	//! Set a new memory limit to the buffer manager, throws an exception if the new limit is too low and not enough
-	//! blocks can be evicted
+	//! Set a new memory limit.
+	//! Throws an exception, if the new limit is too low, meaning not enough blocks can be evicted.
 	virtual void SetMemoryLimit(idx_t limit = (idx_t)-1);
+	//! Set a new swap limit.
 	virtual void SetSwapLimit(optional_idx limit = optional_idx());
 
+	//! Get the temporary file information of each temporary file.
 	virtual vector<TemporaryFileInformation> GetTemporaryFiles();
+	//! Get the path to the temporary file directory.
 	virtual const string &GetTemporaryDirectory() const;
+	//! Set the path to the temporary file directory.
 	virtual void SetTemporaryDirectory(const string &new_dir);
+	//! Returns true, if the path to the temporary file directory is not empty.
 	virtual bool HasTemporaryDirectory() const;
+	//! Returns true if there are files found in the temporary directory
+	virtual bool HasFilesInTemporaryDirectory() const;
 
 	//! Construct a managed buffer.
 	virtual unique_ptr<FileBuffer> ConstructManagedBuffer(idx_t size, idx_t block_header_size,
 	                                                      unique_ptr<FileBuffer> &&source,
 	                                                      FileBufferType type = FileBufferType::MANAGED_BUFFER);
-	//! Get the underlying buffer pool responsible for managing the buffers
+	//! Get the buffer pool.
 	virtual BufferPool &GetBufferPool() const;
-
+	//! Get the database.
 	virtual DatabaseInstance &GetDatabase() = 0;
-	// Static methods
-	DUCKDB_API static BufferManager &GetBufferManager(DatabaseInstance &db);
-	DUCKDB_API static const BufferManager &GetBufferManager(const DatabaseInstance &db);
-	DUCKDB_API static BufferManager &GetBufferManager(ClientContext &context);
-	DUCKDB_API static const BufferManager &GetBufferManager(const ClientContext &context);
-	DUCKDB_API static BufferManager &GetBufferManager(AttachedDatabase &db);
-
-	static idx_t GetAllocSize(const idx_t alloc_size) {
-		return AlignValue<idx_t, Storage::SECTOR_SIZE>(alloc_size);
-	}
-
-	//! Returns the maximum available memory for a given query
-	idx_t GetQueryMaxMemory() const;
-
-	//! Get the manager that assigns reservations for temporary memory, e.g., for query intermediates
+	//! Get the manager assigning reservations for temporary memory, e.g., for query intermediates.
 	virtual TemporaryMemoryManager &GetTemporaryMemoryManager();
 
-protected:
+	//! Purge the eviction queue of the block handle.
 	virtual void PurgeQueue(const BlockHandle &handle) = 0;
+	//! Add the block handle to the eviction queue.
 	virtual void AddToEvictionQueue(shared_ptr<BlockHandle> &handle);
+	//! Write a temporary file buffer.
 	virtual void WriteTemporaryBuffer(MemoryTag tag, block_id_t block_id, FileBuffer &buffer);
-	virtual unique_ptr<FileBuffer> ReadTemporaryBuffer(MemoryTag tag, BlockHandle &block,
+	//! Read a temporary buffer.
+	virtual unique_ptr<FileBuffer> ReadTemporaryBuffer(QueryContext context, MemoryTag tag, BlockHandle &block,
 	                                                   unique_ptr<FileBuffer> buffer);
+	//! Delete the temporary file containing the block.
 	virtual void DeleteTemporaryFile(BlockHandle &block);
 };
 

package/src/duckdb/src/include/duckdb/storage/caching_file_system.hpp

@@ -12,12 +12,14 @@
 #include "duckdb/common/file_open_flags.hpp"
 #include "duckdb/common/open_file_info.hpp"
 #include "duckdb/common/shared_ptr.hpp"
+#include "duckdb/main/client_context.hpp"
 #include "duckdb/storage/storage_lock.hpp"
 #include "duckdb/storage/external_file_cache.hpp"
 
 namespace duckdb {
 
 class ClientContext;
+class QueryContext;
 class BufferHandle;
 class FileOpenFlags;
 class FileSystem;
@@ -31,8 +33,8 @@ public:
 	using CachedFile = ExternalFileCache::CachedFile;
 
 public:
-	DUCKDB_API CachingFileHandle(CachingFileSystem &caching_file_system, const OpenFileInfo &path, FileOpenFlags flags,
-	                             CachedFile &cached_file);
+	DUCKDB_API CachingFileHandle(QueryContext context, CachingFileSystem &caching_file_system, const OpenFileInfo &path,
+	                             FileOpenFlags flags, CachedFile &cached_file);
 	DUCKDB_API ~CachingFileHandle();
 
 public:
@@ -46,7 +48,7 @@ public:
 	//! Get some properties of the file
 	DUCKDB_API string GetPath() const;
 	DUCKDB_API idx_t GetFileSize();
-	DUCKDB_API time_t GetLastModifiedTime();
+	DUCKDB_API timestamp_t GetLastModifiedTime();
 	DUCKDB_API string GetVersionTag();
 	DUCKDB_API bool Validate() const;
 	DUCKDB_API bool CanSeek();
@@ -73,6 +75,8 @@ private:
 	                             idx_t location, bool actually_read);
 
 private:
+	QueryContext context;
+
 	//! The client caching file system that was used to create this CachingFileHandle
 	CachingFileSystem &caching_file_system;
 	//! The DB external file cache
@@ -89,7 +93,7 @@ private:
 	//! The underlying FileHandle (optional)
 	unique_ptr<FileHandle> file_handle;
 	//! Last modified time and version tag (if FileHandle is opened)
-	time_t last_modified;
+	timestamp_t last_modified;
 	string version_tag;
 
 	//! Current position (if non-seeking reads)
@@ -111,6 +115,8 @@ public:
 	DUCKDB_API static CachingFileSystem Get(ClientContext &context);
 
 	DUCKDB_API unique_ptr<CachingFileHandle> OpenFile(const OpenFileInfo &path, FileOpenFlags flags);
+	DUCKDB_API unique_ptr<CachingFileHandle> OpenFile(QueryContext context, const OpenFileInfo &path,
+	                                                  FileOpenFlags flags);
 
 private:
 	//! The Client FileSystem (needs to be client-specific so we can do, e.g., HTTPFS profiling)

package/src/duckdb/src/include/duckdb/storage/checkpoint/row_group_writer.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/storage/checkpoint_manager.hpp"
+#include "duckdb/common/serializer/memory_stream.hpp"
 
 namespace duckdb {
 struct ColumnCheckpointState;
@@ -22,17 +23,24 @@ class SegmentStatistics;
 // Writes data for an entire row group.
 class RowGroupWriter {
 public:
-	RowGroupWriter(TableCatalogEntry &table, PartialBlockManager &partial_block_manager)
-	    : table(table), partial_block_manager(partial_block_manager) {
-	}
+	RowGroupWriter(TableCatalogEntry &table, PartialBlockManager &partial_block_manager);
 	virtual ~RowGroupWriter() {
 	}
 
-	CompressionType GetColumnCompressionType(idx_t i);
+	const vector<CompressionType> &GetCompressionTypes() const {
+		return compression_types;
+	}
 
 	virtual CheckpointType GetCheckpointType() const = 0;
-	virtual MetadataWriter &GetPayloadWriter() = 0;
+	virtual WriteStream &GetPayloadWriter() = 0;
+	virtual MetaBlockPointer GetMetaBlockPointer() = 0;
+	virtual optional_ptr<MetadataManager> GetMetadataManager() = 0;
+	virtual void StartWritingColumns(vector<MetaBlockPointer> &column_metadata) {
+	}
+	virtual void FinishWritingColumns() {
+	}
 
+	DatabaseInstance &GetDatabase();
 	PartialBlockManager &GetPartialBlockManager() {
 		return partial_block_manager;
 	}
@@ -40,6 +48,7 @@ public:
 protected:
 	TableCatalogEntry &table;
 	PartialBlockManager &partial_block_manager;
+	vector<CompressionType> compression_types;
 };
 
 // Writes data for an entire row group.
@@ -50,7 +59,11 @@ public:
 
 public:
 	CheckpointType GetCheckpointType() const override;
-	MetadataWriter &GetPayloadWriter() override;
+	WriteStream &GetPayloadWriter() override;
+	MetaBlockPointer GetMetaBlockPointer() override;
+	optional_ptr<MetadataManager> GetMetadataManager() override;
+	void StartWritingColumns(vector<MetaBlockPointer> &column_metadata) override;
+	void FinishWritingColumns() override;
 
 private:
 	//! Underlying writer object

package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_reader.hpp

@@ -16,7 +16,7 @@ struct BoundCreateTableInfo;
 //! The table data reader is responsible for reading the data of a table from the block manager
 class TableDataReader {
 public:
-	TableDataReader(MetadataReader &reader, BoundCreateTableInfo &info);
+	TableDataReader(MetadataReader &reader, BoundCreateTableInfo &info, MetaBlockPointer table_pointer);
 
 	void ReadTableData();