duckdb 1.3.4 → 1.4.1-dev2.0

package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp

@@ -8,13 +8,13 @@
 
 #pragma once
 
-#include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/buffer/buffer_handle.hpp"
-#include "duckdb/storage/block_manager.hpp"
-#include "duckdb/common/file_system.hpp"
 #include "duckdb/common/enums/memory_tag.hpp"
-#include "duckdb/storage/buffer/temporary_file_information.hpp"
+#include "duckdb/common/file_system.hpp"
 #include "duckdb/main/config.hpp"
+#include "duckdb/storage/block_manager.hpp"
+#include "duckdb/storage/buffer/buffer_handle.hpp"
+#include "duckdb/storage/buffer/temporary_file_information.hpp"
+#include "duckdb/storage/buffer_manager.hpp"
 
 namespace duckdb {
 
@@ -34,85 +34,104 @@ public:
 	}
 
 public:
+	DUCKDB_API static BufferManager &GetBufferManager(DatabaseInstance &db);
+	DUCKDB_API static const BufferManager &GetBufferManager(const DatabaseInstance &db);
+	DUCKDB_API static BufferManager &GetBufferManager(ClientContext &context);
+	DUCKDB_API static const BufferManager &GetBufferManager(const ClientContext &context);
+	DUCKDB_API static BufferManager &GetBufferManager(AttachedDatabase &db);
+	DUCKDB_API static idx_t GetAllocSize(const idx_t alloc_size);
+
+public:
+	//! Allocate temporary memory of size block_size.
 	virtual shared_ptr<BlockHandle> AllocateTemporaryMemory(MemoryTag tag, idx_t block_size,
 	                                                        bool can_destroy = true) = 0;
+	//! Allocate block-based memory.
+	//! The block manager provides the block size, and the block header size.
+	//! Returns the BlockHandle managing the registered memory.
 	virtual shared_ptr<BlockHandle> AllocateMemory(MemoryTag tag, BlockManager *block_manager,
 	                                               bool can_destroy = true) = 0;
+	//! Allocate (temporary) memory of size block_size, and pin it.
 	virtual BufferHandle Allocate(MemoryTag tag, idx_t block_size, bool can_destroy = true) = 0;
+	//! Allocate block-based memory and pin it.
 	virtual BufferHandle Allocate(MemoryTag tag, BlockManager *block_manager, bool can_destroy = true) = 0;
-	//! Reallocate an in-memory buffer that is pinned.
+	//! Reallocate a pinned in-memory buffer.
 	virtual void ReAllocate(shared_ptr<BlockHandle> &handle, idx_t block_size) = 0;
+	//! Pin a block handle.
 	virtual BufferHandle Pin(shared_ptr<BlockHandle> &handle) = 0;
-	//! Prefetch a series of blocks. Note that this is a performance suggestion.
+	//! Pre-fetch a series of blocks.
+	//! Using this function is a performance suggestion.
 	virtual void Prefetch(vector<shared_ptr<BlockHandle>> &handles) = 0;
+	//! Unpin a block handle.
 	virtual void Unpin(shared_ptr<BlockHandle> &handle) = 0;
 
-	//! Returns the currently allocated memory
+	//! Returns the currently allocated memory.
 	virtual idx_t GetUsedMemory() const = 0;
-	//! Returns the maximum available memory
+	//! Returns the maximum available memory.
 	virtual idx_t GetMaxMemory() const = 0;
-	//! Returns the currently used swap space
+	//! Returns the currently used swap space.
 	virtual idx_t GetUsedSwap() const = 0;
-	//! Returns the maximum swap space that can be used
+	//! Returns the maximum swap space that can be used.
 	virtual optional_idx GetMaxSwap() const = 0;
 	//! Returns the block allocation size for buffer-managed blocks.
 	virtual idx_t GetBlockAllocSize() const = 0;
 	//! Returns the block size for buffer-managed blocks.
 	virtual idx_t GetBlockSize() const = 0;
-	//! Returns the block header size for buffer-managed blocks.
-	virtual idx_t GetTemporaryBlockHeaderSize() const = 0;
+	//! Returns the maximum available memory for a given query.
+	virtual idx_t GetQueryMaxMemory() const = 0;
 
-	//! Returns a new block of transient memory.
+	//! Returns a newly registered block of transient memory.
 	virtual shared_ptr<BlockHandle> RegisterTransientMemory(const idx_t size, BlockManager &block_manager);
-	//! Returns a new block of memory that is smaller than the block size setting.
+	//! Returns a newly registered block of memory that is smaller than the block size setting.
 	virtual shared_ptr<BlockHandle> RegisterSmallMemory(const idx_t size);
+	//! Returns a newly registered block of memory that is smaller than the block size setting and has a memory tag.
 	virtual shared_ptr<BlockHandle> RegisterSmallMemory(MemoryTag tag, const idx_t size);
 
+	//! Get the buffer allocator.
 	virtual DUCKDB_API Allocator &GetBufferAllocator();
+	//! Reserve memory.
 	virtual DUCKDB_API void ReserveMemory(idx_t size);
+	//! Free reserved memory.
 	virtual DUCKDB_API void FreeReservedMemory(idx_t size);
+	//! GetMemoryUsageInfo returns MemoryInformation for each memory tag.
 	virtual vector<MemoryInformation> GetMemoryUsageInfo() const = 0;
-	//! Set a new memory limit to the buffer manager, throws an exception if the new limit is too low and not enough
-	//! blocks can be evicted
+	//! Set a new memory limit.
+	//! Throws an exception, if the new limit is too low, meaning not enough blocks can be evicted.
 	virtual void SetMemoryLimit(idx_t limit = (idx_t)-1);
+	//! Set a new swap limit.
 	virtual void SetSwapLimit(optional_idx limit = optional_idx());
 
+	//! Get the temporary file information of each temporary file.
 	virtual vector<TemporaryFileInformation> GetTemporaryFiles();
+	//! Get the path to the temporary file directory.
 	virtual const string &GetTemporaryDirectory() const;
+	//! Set the path to the temporary file directory.
 	virtual void SetTemporaryDirectory(const string &new_dir);
+	//! Returns true, if the path to the temporary file directory is not empty.
 	virtual bool HasTemporaryDirectory() const;
+	//! Returns true if there are files found in the temporary directory
+	virtual bool HasFilesInTemporaryDirectory() const;
 
 	//! Construct a managed buffer.
 	virtual unique_ptr<FileBuffer> ConstructManagedBuffer(idx_t size, idx_t block_header_size,
 	                                                      unique_ptr<FileBuffer> &&source,
 	                                                      FileBufferType type = FileBufferType::MANAGED_BUFFER);
-	//! Get the underlying buffer pool responsible for managing the buffers
+	//! Get the buffer pool.
 	virtual BufferPool &GetBufferPool() const;
-
+	//! Get the database.
 	virtual DatabaseInstance &GetDatabase() = 0;
-	// Static methods
-	DUCKDB_API static BufferManager &GetBufferManager(DatabaseInstance &db);
-	DUCKDB_API static const BufferManager &GetBufferManager(const DatabaseInstance &db);
-	DUCKDB_API static BufferManager &GetBufferManager(ClientContext &context);
-	DUCKDB_API static const BufferManager &GetBufferManager(const ClientContext &context);
-	DUCKDB_API static BufferManager &GetBufferManager(AttachedDatabase &db);
-
-	static idx_t GetAllocSize(const idx_t alloc_size) {
-		return AlignValue<idx_t, Storage::SECTOR_SIZE>(alloc_size);
-	}
-
-	//! Returns the maximum available memory for a given query
-	idx_t GetQueryMaxMemory() const;
-
-	//! Get the manager that assigns reservations for temporary memory, e.g., for query intermediates
+	//! Get the manager assigning reservations for temporary memory, e.g., for query intermediates.
 	virtual TemporaryMemoryManager &GetTemporaryMemoryManager();
 
-protected:
+	//! Purge the eviction queue of the block handle.
 	virtual void PurgeQueue(const BlockHandle &handle) = 0;
+	//! Add the block handle to the eviction queue.
 	virtual void AddToEvictionQueue(shared_ptr<BlockHandle> &handle);
+	//! Write a temporary file buffer.
 	virtual void WriteTemporaryBuffer(MemoryTag tag, block_id_t block_id, FileBuffer &buffer);
-	virtual unique_ptr<FileBuffer> ReadTemporaryBuffer(MemoryTag tag, BlockHandle &block,
+	//! Read a temporary buffer.
+	virtual unique_ptr<FileBuffer> ReadTemporaryBuffer(QueryContext context, MemoryTag tag, BlockHandle &block,
 	                                                   unique_ptr<FileBuffer> buffer);
+	//! Delete the temporary file containing the block.
 	virtual void DeleteTemporaryFile(BlockHandle &block);
 };
 

package/src/duckdb/src/include/duckdb/storage/caching_file_system.hpp

@@ -12,12 +12,14 @@
 #include "duckdb/common/file_open_flags.hpp"
 #include "duckdb/common/open_file_info.hpp"
 #include "duckdb/common/shared_ptr.hpp"
+#include "duckdb/main/client_context.hpp"
 #include "duckdb/storage/storage_lock.hpp"
 #include "duckdb/storage/external_file_cache.hpp"
 
 namespace duckdb {
 
 class ClientContext;
+class QueryContext;
 class BufferHandle;
 class FileOpenFlags;
 class FileSystem;
@@ -31,8 +33,8 @@ public:
 	using CachedFile = ExternalFileCache::CachedFile;
 
 public:
-	DUCKDB_API CachingFileHandle(CachingFileSystem &caching_file_system, const OpenFileInfo &path, FileOpenFlags flags,
-	                             CachedFile &cached_file);
+	DUCKDB_API CachingFileHandle(QueryContext context, CachingFileSystem &caching_file_system, const OpenFileInfo &path,
+	                             FileOpenFlags flags, CachedFile &cached_file);
 	DUCKDB_API ~CachingFileHandle();
 
 public:
@@ -46,7 +48,7 @@ public:
 	//! Get some properties of the file
 	DUCKDB_API string GetPath() const;
 	DUCKDB_API idx_t GetFileSize();
-	DUCKDB_API time_t GetLastModifiedTime();
+	DUCKDB_API timestamp_t GetLastModifiedTime();
 	DUCKDB_API string GetVersionTag();
 	DUCKDB_API bool Validate() const;
 	DUCKDB_API bool CanSeek();
@@ -73,6 +75,8 @@ private:
 	                             idx_t location, bool actually_read);
 
 private:
+	QueryContext context;
+
 	//! The client caching file system that was used to create this CachingFileHandle
 	CachingFileSystem &caching_file_system;
 	//! The DB external file cache
@@ -89,7 +93,7 @@ private:
 	//! The underlying FileHandle (optional)
 	unique_ptr<FileHandle> file_handle;
 	//! Last modified time and version tag (if FileHandle is opened)
-	time_t last_modified;
+	timestamp_t last_modified;
 	string version_tag;
 
 	//! Current position (if non-seeking reads)
@@ -111,6 +115,8 @@ public:
 	DUCKDB_API static CachingFileSystem Get(ClientContext &context);
 
 	DUCKDB_API unique_ptr<CachingFileHandle> OpenFile(const OpenFileInfo &path, FileOpenFlags flags);
+	DUCKDB_API unique_ptr<CachingFileHandle> OpenFile(QueryContext context, const OpenFileInfo &path,
+	                                                  FileOpenFlags flags);
 
 private:
 	//! The Client FileSystem (needs to be client-specific so we can do, e.g., HTTPFS profiling)

package/src/duckdb/src/include/duckdb/storage/checkpoint/row_group_writer.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/storage/checkpoint_manager.hpp"
+#include "duckdb/common/serializer/memory_stream.hpp"
 
 namespace duckdb {
 struct ColumnCheckpointState;
@@ -22,17 +23,24 @@ class SegmentStatistics;
 // Writes data for an entire row group.
 class RowGroupWriter {
 public:
-	RowGroupWriter(TableCatalogEntry &table, PartialBlockManager &partial_block_manager)
-	    : table(table), partial_block_manager(partial_block_manager) {
-	}
+	RowGroupWriter(TableCatalogEntry &table, PartialBlockManager &partial_block_manager);
 	virtual ~RowGroupWriter() {
 	}
 
-	CompressionType GetColumnCompressionType(idx_t i);
+	const vector<CompressionType> &GetCompressionTypes() const {
+		return compression_types;
+	}
 
 	virtual CheckpointType GetCheckpointType() const = 0;
-	virtual MetadataWriter &GetPayloadWriter() = 0;
+	virtual WriteStream &GetPayloadWriter() = 0;
+	virtual MetaBlockPointer GetMetaBlockPointer() = 0;
+	virtual optional_ptr<MetadataManager> GetMetadataManager() = 0;
+	virtual void StartWritingColumns(vector<MetaBlockPointer> &column_metadata) {
+	}
+	virtual void FinishWritingColumns() {
+	}
 
+	DatabaseInstance &GetDatabase();
 	PartialBlockManager &GetPartialBlockManager() {
 		return partial_block_manager;
 	}
@@ -40,6 +48,7 @@ public:
 protected:
 	TableCatalogEntry &table;
 	PartialBlockManager &partial_block_manager;
+	vector<CompressionType> compression_types;
 };
 
 // Writes data for an entire row group.
@@ -50,7 +59,11 @@ public:
 
 public:
 	CheckpointType GetCheckpointType() const override;
-	MetadataWriter &GetPayloadWriter() override;
+	WriteStream &GetPayloadWriter() override;
+	MetaBlockPointer GetMetaBlockPointer() override;
+	optional_ptr<MetadataManager> GetMetadataManager() override;
+	void StartWritingColumns(vector<MetaBlockPointer> &column_metadata) override;
+	void FinishWritingColumns() override;
 
 private:
 	//! Underlying writer object

package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_reader.hpp

@@ -16,7 +16,7 @@ struct BoundCreateTableInfo;
 //! The table data reader is responsible for reading the data of a table from the block manager
 class TableDataReader {
 public:
-	TableDataReader(MetadataReader &reader, BoundCreateTableInfo &info);
+	TableDataReader(MetadataReader &reader, BoundCreateTableInfo &info, MetaBlockPointer table_pointer);
 
 	void ReadTableData();
 

package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp

@@ -15,8 +15,7 @@ namespace duckdb {
 class DuckTableEntry;
 class TableStatistics;
 
-//! The table data writer is responsible for writing the data of a table to
-//! storage.
+//! The table data writer is responsible for writing the data of a table to storage.
 //
 //! This is meant to encapsulate and abstract:
 //!  - Storage/encoding of table metadata (block pointers)
@@ -24,26 +23,27 @@ class TableStatistics;
 //! Abstraction will support, for example: tiering, versioning, or splitting into multiple block managers.
 class TableDataWriter {
 public:
-	explicit TableDataWriter(TableCatalogEntry &table, optional_ptr<ClientContext> client_context);
+	explicit TableDataWriter(TableCatalogEntry &table, QueryContext context);
 	virtual ~TableDataWriter();
 
 public:
 	void WriteTableData(Serializer &metadata_serializer);
 
-	CompressionType GetColumnCompressionType(idx_t i);
-
-	virtual void FinalizeTable(const TableStatistics &global_stats, DataTableInfo *info, Serializer &serializer) = 0;
+	virtual void WriteUnchangedTable(MetaBlockPointer pointer, idx_t total_rows) = 0;
+	virtual void FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info, RowGroupCollection &collection,
+	                           Serializer &serializer) = 0;
 	virtual unique_ptr<RowGroupWriter> GetRowGroupWriter(RowGroup &row_group) = 0;
 
 	virtual void AddRowGroup(RowGroupPointer &&row_group_pointer, unique_ptr<RowGroupWriter> writer);
 	virtual CheckpointType GetCheckpointType() const = 0;
+	virtual MetadataManager &GetMetadataManager() = 0;
 
 	DatabaseInstance &GetDatabase();
 	unique_ptr<TaskExecutor> CreateTaskExecutor();
 
 protected:
 	DuckTableEntry &table;
-	optional_ptr<ClientContext> client_context;
+	optional_ptr<ClientContext> context;
 	//! Pointers to the start of each row group.
 	vector<RowGroupPointer> row_group_pointers;
 };
@@ -54,14 +54,21 @@ public:
 	                          MetadataWriter &table_data_writer);
 
 public:
-	void FinalizeTable(const TableStatistics &global_stats, DataTableInfo *info, Serializer &serializer) override;
+	void WriteUnchangedTable(MetaBlockPointer pointer, idx_t total_rows) override;
+	void FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info, RowGroupCollection &collection,
+	                   Serializer &serializer) override;
 	unique_ptr<RowGroupWriter> GetRowGroupWriter(RowGroup &row_group) override;
 	CheckpointType GetCheckpointType() const override;
+	MetadataManager &GetMetadataManager() override;
 
 private:
 	SingleFileCheckpointWriter &checkpoint_manager;
 	//! Writes the actual table data
 	MetadataWriter &table_data_writer;
+	//! The root pointer, if we are re-using metadata of the table
+	MetaBlockPointer existing_pointer;
+	optional_idx existing_rows;
+	vector<MetaBlockPointer> existing_pointers;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp

@@ -33,6 +33,7 @@ public:
 	//! The database
 	AttachedDatabase &db;
 
+	virtual void CreateCheckpoint() = 0;
 	virtual MetadataManager &GetMetadataManager() = 0;
 	virtual MetadataWriter &GetMetadataWriter() = 0;
 	virtual unique_ptr<TableDataWriter> GetTableDataWriter(TableCatalogEntry &table) = 0;
@@ -97,12 +98,10 @@ class SingleFileCheckpointWriter final : public CheckpointWriter {
 	friend class SingleFileTableDataWriter;
 
 public:
-	SingleFileCheckpointWriter(optional_ptr<ClientContext> client_context, AttachedDatabase &db,
-	                           BlockManager &block_manager, CheckpointType checkpoint_type);
+	SingleFileCheckpointWriter(QueryContext context, AttachedDatabase &db, BlockManager &block_manager,
+	                           CheckpointType checkpoint_type);
 
-	//! Checkpoint the current state of the WAL and flush it to the main storage. This should be called BEFORE any
-	//! connection is available because right now the checkpointing cannot be done online. (TODO)
-	void CreateCheckpoint();
+	void CreateCheckpoint() override;
 
 	MetadataWriter &GetMetadataWriter() override;
 	MetadataManager &GetMetadataManager() override;
@@ -112,16 +111,15 @@ public:
 	CheckpointType GetCheckpointType() const {
 		return checkpoint_type;
 	}
-
 	optional_ptr<ClientContext> GetClientContext() const {
-		return client_context;
+		return context;
 	}
 
 public:
 	void WriteTable(TableCatalogEntry &table, Serializer &serializer) override;
 
 private:
-	optional_ptr<ClientContext> client_context;
+	optional_ptr<ClientContext> context;
 	//! The metadata writer is responsible for writing schema information
 	unique_ptr<MetadataWriter> metadata_writer;
 	//! The table data writer is responsible for writing the DataPointers used by the table chunks

package/src/duckdb/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp

@@ -128,7 +128,8 @@ struct AlpCompression {
 	 */
 	static T DecodeValue(int64_t encoded_value, AlpEncodingIndices encoding_indices) {
 		//! The cast to T is needed to prevent a signed integer overflow
-		T decoded_value = static_cast<T>(encoded_value) * AlpConstants::FACT_ARR[encoding_indices.factor] *
+		T decoded_value = static_cast<T>(encoded_value) *
+		                  static_cast<T>(AlpConstants::FACT_ARR[encoding_indices.factor]) *
 		                  AlpTypedConstants<T>::FRAC_ARR[encoding_indices.exponent];
 		return decoded_value;
 	}

package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_analyze.hpp

@@ -82,7 +82,12 @@ unique_ptr<AnalyzeState> AlpInitAnalyze(ColumnData &col_data, PhysicalType type)
  */
 template <class T>
 bool AlpAnalyze(AnalyzeState &state, Vector &input, idx_t count) {
-	auto &analyze_state = (AlpAnalyzeState<T> &)state;
+	if (state.info.GetBlockSize() + state.info.GetBlockHeaderSize() < DEFAULT_BLOCK_ALLOC_SIZE) {
+		return false;
+	}
+
+	auto &analyze_state = state.Cast<AlpAnalyzeState<T>>();
+
 	bool must_skip_current_vector = alp::AlpUtils::MustSkipSamplingFromCurrentVector(
 	    analyze_state.vectors_count, analyze_state.vectors_sampled_count, count);
 	analyze_state.vectors_count += 1;

package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_fetch.hpp

@@ -29,7 +29,7 @@ void AlpFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id,
 
 	AlpScanState<T> scan_state(segment);
 	scan_state.Skip(segment, UnsafeNumericCast<idx_t>(row_id));
-	auto result_data = FlatVector::GetData<EXACT_TYPE>(result);
+	auto result_data = FlatVector::GetDataUnsafe<EXACT_TYPE>(result);
 	result_data[result_idx] = (EXACT_TYPE)0;
 
 	if (scan_state.VectorFinished() && scan_state.total_value_count < scan_state.count) {

package/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp

@@ -47,8 +47,12 @@ unique_ptr<AnalyzeState> AlpRDInitAnalyze(ColumnData &col_data, PhysicalType typ
  */
 template <class T>
 bool AlpRDAnalyze(AnalyzeState &state, Vector &input, idx_t count) {
+	if (state.info.GetBlockSize() + state.info.GetBlockHeaderSize() < DEFAULT_BLOCK_ALLOC_SIZE) {
+		return false;
+	}
+
 	using EXACT_TYPE = typename FloatingToExact<T>::TYPE;
-	auto &analyze_state = (AlpRDAnalyzeState<T> &)state;
+	auto &analyze_state = state.Cast<AlpRDAnalyzeState<T>>();
 
 	bool must_skip_current_vector = alp::AlpUtils::MustSkipSamplingFromCurrentVector(
 	    analyze_state.vectors_count, analyze_state.vectors_sampled_count, count);
@@ -114,12 +118,14 @@ idx_t AlpRDFinalAnalyze(AnalyzeState &state) {
 	if (analyze_state.total_values_count == 0) {
 		return DConstants::INVALID_INDEX;
 	}
-	double factor_of_sampling = 1 / ((double)analyze_state.rowgroup_sample.size() / analyze_state.total_values_count);
+	double factor_of_sampling = 1 / (static_cast<double>(analyze_state.rowgroup_sample.size()) /
+	                                 static_cast<double>(analyze_state.total_values_count));
 
 	// Finding which is the best dictionary for the sample
 	double estimated_bits_per_value =
 	    alp::AlpRDCompression<T, true>::FindBestDictionary(analyze_state.rowgroup_sample, analyze_state.state);
-	double estimated_compressed_bits = estimated_bits_per_value * analyze_state.rowgroup_sample.size();
+	double estimated_compressed_bits =
+	    estimated_bits_per_value * static_cast<double>(analyze_state.rowgroup_sample.size());
 	double estimed_compressed_bytes = estimated_compressed_bits / 8;
 
 	//! Overhead per segment: [Pointer to metadata + right bitwidth + left bitwidth + n dict elems] + Dictionary Size

package/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_fetch.hpp

@@ -28,7 +28,7 @@ void AlpRDFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id
 	using EXACT_TYPE = typename FloatingToExact<T>::TYPE;
 	AlpRDScanState<T> scan_state(segment);
 	scan_state.Skip(segment, UnsafeNumericCast<idx_t>(row_id));
-	auto result_data = FlatVector::GetData<EXACT_TYPE>(result);
+	auto result_data = FlatVector::GetDataUnsafe<EXACT_TYPE>(result);
 	result_data[result_idx] = (EXACT_TYPE)0;
 
 	if (scan_state.VectorFinished() && scan_state.total_value_count < scan_state.count) {

package/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_scan.hpp

@@ -223,7 +223,7 @@ void AlpRDScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan
 	auto &scan_state = (AlpRDScanState<T> &)*state.scan_state;
 
 	// Get the pointer to the result values
-	auto current_result_ptr = FlatVector::GetData<EXACT_TYPE>(result);
+	auto current_result_ptr = FlatVector::GetDataUnsafe<EXACT_TYPE>(result);
 	result.SetVectorType(VectorType::FLAT_VECTOR);
 	current_result_ptr += result_offset;
 

package/src/duckdb/src/include/duckdb/storage/compression/chimp/algorithm/packed_data.hpp

@@ -1,14 +1,16 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/storage/compression/chimp/packed_data.hpp
+// duckdb/storage/compression/chimp/algorithm/packed_data.hpp
 //
 //
 //===----------------------------------------------------------------------===//
 
 #pragma once
 
+#include "duckdb/common/common.hpp"
 #include "duckdb/storage/compression/chimp/algorithm/chimp_utils.hpp"
+#include "duckdb/common/assert.hpp"
 #include "duckdb.h"
 
 namespace duckdb {

package/src/duckdb/src/include/duckdb/storage/compression/dict_fsst/compression.hpp

@@ -47,7 +47,7 @@ public:
 	DictionaryAppendState TryEncode();
 
 	bool CompressInternal(UnifiedVectorFormat &vector_format, const string_t &str, bool is_null,
-	                      EncodedInput &encoded_input, const idx_t i, idx_t count);
+	                      EncodedInput &encoded_input, const idx_t i, idx_t count, bool fail_on_no_space);
 	void Compress(Vector &scan_vector, idx_t count);
 	void FinalizeCompress();
 	void Flush(bool final);

package/src/duckdb/src/include/duckdb/storage/compression/dict_fsst/decompression.hpp

@@ -28,7 +28,7 @@ public:
 	const SelectionVector &GetSelVec(idx_t start, idx_t scan_count);
 	void Select(Vector &result, idx_t start, const SelectionVector &sel, idx_t sel_count);
 
-	bool AllowDictionaryScan(idx_t start, idx_t scan_count);
+	bool AllowDictionaryScan(idx_t scan_count);
 
 private:
 	string_t FetchStringFromDict(Vector &result, uint32_t dict_offset, idx_t dict_idx);

package/src/duckdb/src/include/duckdb/storage/compression/patas/patas.hpp

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/common/patas/patas.hpp
+// duckdb/storage/compression/patas/patas.hpp
 //
 //
 //===----------------------------------------------------------------------===//

package/src/duckdb/src/include/duckdb/storage/compression/patas/patas_fetch.hpp

@@ -30,7 +30,7 @@ void PatasFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id
 
 	PatasScanState<T> scan_state(segment);
 	scan_state.Skip(segment, UnsafeNumericCast<idx_t>(row_id));
-	auto result_data = FlatVector::GetData<EXACT_TYPE>(result);
+	auto result_data = FlatVector::GetDataUnsafe<EXACT_TYPE>(result);
 	result_data[result_idx] = (EXACT_TYPE)0;
 
 	if (scan_state.GroupFinished() && scan_state.total_value_count < scan_state.count) {

package/src/duckdb/src/include/duckdb/storage/compression/patas/patas_scan.hpp

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/common/storage/compression/chimp/chimp_scan.hpp
+// duckdb/common/storage/compression/patas/patas_scan.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -10,6 +10,11 @@
 
 #include "duckdb/storage/compression/chimp/chimp.hpp"
 #include "duckdb/storage/compression/chimp/algorithm/chimp_utils.hpp"
+#include "duckdb/storage/compression/chimp/algorithm/packed_data.hpp"
+#include "duckdb/storage/compression/chimp/algorithm/byte_reader.hpp"
+#include "duckdb/storage/compression/patas/shared.hpp"
+#include "duckdb/storage/compression/patas/algorithm/patas.hpp"
+#include "duckdb/storage/compression/patas/patas.hpp"
 
 #include "duckdb/common/limits.hpp"
 #include "duckdb/common/types/null_value.hpp"
@@ -214,7 +219,7 @@ void PatasScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan
 	auto &scan_state = (PatasScanState<T> &)*state.scan_state;
 
 	// Get the pointer to the result values
-	auto current_result_ptr = FlatVector::GetData<EXACT_TYPE>(result);
+	auto current_result_ptr = FlatVector::GetDataUnsafe<EXACT_TYPE>(result);
 	result.SetVectorType(VectorType::FLAT_VECTOR);
 	current_result_ptr += result_offset;
 

package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp

@@ -71,6 +71,11 @@ struct RowGroupPointer {
 	vector<MetaBlockPointer> data_pointers;
 	//! Data pointers to the delete information of the row group (if any)
 	vector<MetaBlockPointer> deletes_pointers;
+	//! Whether or not we have all metadata blocks defined in the pointer
+	bool has_metadata_blocks = false;
+	//! Metadata blocks of the columns that are not mentioned in "data_pointers"
+	//! This is often empty - but can be set for wide columns with a lot of metadata
+	vector<idx_t> extra_metadata_blocks;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -96,6 +96,8 @@ public:
 	//! Fetch data from the specific row identifiers from the base table
 	void Fetch(DuckTransaction &transaction, DataChunk &result, const vector<StorageIndex> &column_ids,
 	           const Vector &row_ids, idx_t fetch_count, ColumnFetchState &state);
+	//! Returns true, if the transaction can fetch the row ID.
+	bool CanFetch(DuckTransaction &transaction, const row_t row_id);
 
 	//! Initializes appending to transaction-local storage
 	void InitializeLocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context,
@@ -179,12 +181,15 @@ public:
 	//! Merge a row group collection directly into this table - appending it to the end of the table without copying
 	void MergeStorage(RowGroupCollection &data, TableIndexList &indexes, optional_ptr<StorageCommitState> commit_state);
 
-	//! Append a chunk with the row ids [row_start, ..., row_start + chunk.size()] to all indexes of the table.
-	//! Returns empty ErrorData, if the append was successful.
-	ErrorData AppendToIndexes(optional_ptr<TableIndexList> delete_indexes, DataChunk &chunk, row_t row_start,
-	                          const IndexAppendMode index_append_mode);
+	//! Appends a chunk with the row ids [row_start, ..., row_start + chunk.size()] to all indexes of the table.
+	//! If an index is bound, it appends table_chunk. Else, it buffers index_chunk.
 	static ErrorData AppendToIndexes(TableIndexList &indexes, optional_ptr<TableIndexList> delete_indexes,
-	                                 DataChunk &chunk, row_t row_start, const IndexAppendMode index_append_mode);
+	                                 DataChunk &table_chunk, DataChunk &index_chunk,
+	                                 const vector<StorageIndex> &mapped_column_ids, row_t row_start,
+	                                 const IndexAppendMode index_append_mode);
+	ErrorData AppendToIndexes(optional_ptr<TableIndexList> delete_indexes, DataChunk &table_chunk,
+	                          DataChunk &index_chunk, const vector<StorageIndex> &mapped_column_ids, row_t row_start,
+	                          const IndexAppendMode index_append_mode);
 	//! Remove a chunk with the row ids [row_start, ..., row_start + chunk.size()] from all indexes of the table
 	void RemoveFromIndexes(TableAppendState &state, DataChunk &chunk, row_t row_start);
 	//! Remove the chunk with the specified set of row identifiers from all indexes of the table
@@ -245,7 +250,7 @@ public:
 
 	shared_ptr<DataTableInfo> &GetDataTableInfo();
 
-	void InitializeIndexes(ClientContext &context);
+	void BindIndexes(ClientContext &context);
 	bool HasIndexes() const;
 	bool HasUniqueIndexes() const;
 	bool HasForeignKeyIndex(const vector<PhysicalIndex> &keys, ForeignKeyType type);
@@ -261,9 +266,9 @@ public:
 
 	idx_t GetRowGroupSize() const;
 
-	static void VerifyUniqueIndexes(TableIndexList &indexes, optional_ptr<LocalTableStorage> storage, DataChunk &chunk,
-	                                optional_ptr<ConflictManager> manager);
-
+	//! Verify any unique indexes using optional delete indexes in the local storage.
+	void VerifyUniqueIndexes(TableIndexList &indexes, optional_ptr<LocalTableStorage> storage, DataChunk &chunk,
+	                         optional_ptr<ConflictManager> manager);
 	//! AddIndex initializes an index and adds it to the table's index list.
 	//! It is either empty, or initialized via its index storage information.
 	void AddIndex(const ColumnList &columns, const vector<LogicalIndex> &column_indexes, const IndexConstraintType type,