duckdb 1.0.1-dev21.0 → 1.0.1-dev27.0

package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp

@@ -39,6 +39,8 @@ public:
 	//! Reallocate an in-memory buffer that is pinned.
 	virtual void ReAllocate(shared_ptr<BlockHandle> &handle, idx_t block_size) = 0;
 	virtual BufferHandle Pin(shared_ptr<BlockHandle> &handle) = 0;
+	//! Prefetch a series of blocks. Note that this is a performance suggestion.
+	virtual void Prefetch(vector<shared_ptr<BlockHandle>> &handles) = 0;
 	virtual void Unpin(shared_ptr<BlockHandle> &handle) = 0;
 
 	//! Returns the currently allocated memory
@@ -49,9 +51,16 @@ public:
 	virtual idx_t GetUsedSwap() = 0;
 	//! Returns the maximum swap space that can be used
 	virtual optional_idx GetMaxSwap() const = 0;
+	//! Returns the block allocation size for buffer-managed blocks.
+	virtual idx_t GetBlockAllocSize() const = 0;
+	//! Returns the block size for buffer-managed blocks.
+	virtual idx_t GetBlockSize() const = 0;
+
+	//! Returns a new block of transient memory.
+	virtual shared_ptr<BlockHandle> RegisterTransientMemory(const idx_t size, const idx_t block_size);
+	//! Returns a new block of memory that is smaller than the block size setting.
+	virtual shared_ptr<BlockHandle> RegisterSmallMemory(const idx_t size);
 
-	//! Returns a new block of memory that is smaller than Storage::BLOCK_SIZE
-	virtual shared_ptr<BlockHandle> RegisterSmallMemory(idx_t block_size);
 	virtual DUCKDB_API Allocator &GetBufferAllocator();
 	virtual DUCKDB_API void ReserveMemory(idx_t size);
 	virtual DUCKDB_API void FreeReservedMemory(idx_t size);
@@ -80,8 +89,8 @@ public:
 	DUCKDB_API static const BufferManager &GetBufferManager(const ClientContext &context);
 	DUCKDB_API static BufferManager &GetBufferManager(AttachedDatabase &db);
 
-	static idx_t GetAllocSize(idx_t block_size) {
-		return AlignValue<idx_t, Storage::SECTOR_SIZE>(block_size + Storage::BLOCK_HEADER_SIZE);
+	static idx_t GetAllocSize(const idx_t block_size) {
+		return AlignValue<idx_t, Storage::SECTOR_SIZE>(block_size + Storage::DEFAULT_BLOCK_HEADER_SIZE);
 	}
 	//! Returns the maximum available memory for a given query
 	idx_t GetQueryMaxMemory() const;
@@ -90,7 +99,7 @@ public:
 	virtual TemporaryMemoryManager &GetTemporaryMemoryManager();
 
 protected:
-	virtual void PurgeQueue() = 0;
+	virtual void PurgeQueue(FileBufferType type) = 0;
 	virtual void AddToEvictionQueue(shared_ptr<BlockHandle> &handle);
 	virtual void WriteTemporaryBuffer(MemoryTag tag, block_id_t block_id, FileBuffer &buffer);
 	virtual unique_ptr<FileBuffer> ReadTemporaryBuffer(MemoryTag tag, block_id_t id, unique_ptr<FileBuffer> buffer);

package/src/duckdb/src/include/duckdb/storage/checkpoint/row_group_writer.hpp

@@ -30,8 +30,6 @@ public:
 
 	CompressionType GetColumnCompressionType(idx_t i);
 
-	virtual void WriteColumnDataPointers(ColumnCheckpointState &column_checkpoint_state, Serializer &serializer) = 0;
-
 	virtual CheckpointType GetCheckpointType() const = 0;
 	virtual MetadataWriter &GetPayloadWriter() = 0;
 
@@ -51,8 +49,6 @@ public:
 	                         TableDataWriter &writer, MetadataWriter &table_data_writer);
 
 public:
-	void WriteColumnDataPointers(ColumnCheckpointState &column_checkpoint_state, Serializer &serializer) override;
-
 	CheckpointType GetCheckpointType() const override;
 	MetadataWriter &GetPayloadWriter() override;
 

package/src/duckdb/src/include/duckdb/storage/checkpoint/string_checkpoint_state.hpp

@@ -38,8 +38,9 @@ struct string_location_t { // NOLINT
 	}
 	string_location_t() {
 	}
-	bool IsValid() {
-		return offset < int32_t(Storage::BLOCK_SIZE) && (block_id == INVALID_BLOCK || block_id >= MAXIMUM_BLOCK);
+	bool IsValid(const idx_t block_size) {
+		auto cast_block_size = NumericCast<int32_t>(block_size);
+		return offset < cast_block_size && (block_id == INVALID_BLOCK || block_id >= MAXIMUM_BLOCK);
 	}
 	block_id_t block_id;
 	int32_t offset;

package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp

@@ -38,6 +38,7 @@ public:
 	virtual CheckpointType GetCheckpointType() const = 0;
 
 	TaskScheduler &GetScheduler();
+	DatabaseInstance &GetDatabase();
 
 protected:
 	DuckTableEntry &table;

package/src/duckdb/src/include/duckdb/storage/checkpoint/write_overflow_strings_to_disk.hpp

@@ -14,11 +14,11 @@ namespace duckdb {
 
 class WriteOverflowStringsToDisk : public OverflowStringWriter {
 public:
-	explicit WriteOverflowStringsToDisk(BlockManager &block_manager);
+	explicit WriteOverflowStringsToDisk(PartialBlockManager &partial_block_manager);
 	~WriteOverflowStringsToDisk() override;
 
 	//! The block manager
-	BlockManager &block_manager;
+	PartialBlockManager &partial_block_manager;
 
 	//! Temporary buffer
 	BufferHandle handle;
@@ -27,8 +27,6 @@ public:
 	//! The offset within the current block
 	idx_t offset;
 
-	static constexpr idx_t STRING_SPACE = Storage::BLOCK_SIZE - sizeof(block_id_t);
-
 public:
 	void WriteString(UncompressedStringSegmentState &state, string_t string, block_id_t &result_block,
 	                 int32_t &result_offset) override;
@@ -36,6 +34,7 @@ public:
 
 private:
 	void AllocateNewBlock(UncompressedStringSegmentState &state, block_id_t new_block_id);
+	idx_t GetStringSpace() const;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp

@@ -125,6 +125,8 @@ private:
 	PartialBlockManager partial_block_manager;
 	//! Checkpoint type
 	CheckpointType checkpoint_type;
+	//! Block usage count for verification purposes
+	unordered_map<block_id_t, idx_t> verify_block_usage_count;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp

@@ -107,10 +107,10 @@ struct AlpCompression {
 	 */
 	static int64_t NumberToInt64(T n) {
 		if (IsImpossibleToEncode(n)) {
-			return NumericCast<int64_t>(AlpConstants::ENCODING_UPPER_LIMIT);
+			return ExactNumericCast<int64_t>(AlpConstants::ENCODING_UPPER_LIMIT);
 		}
 		n = n + AlpTypedConstants<T>::MAGIC_NUMBER - AlpTypedConstants<T>::MAGIC_NUMBER;
-		return NumericCast<int64_t>(n);
+		return LossyNumericCast<int64_t>(n);
 	}
 
 	/*
@@ -185,7 +185,7 @@ struct AlpCompression {
 
 		// Evaluate factor/exponent compression size (we optimize for FOR)
 		uint64_t delta = (static_cast<uint64_t>(max_encoded_value) - static_cast<uint64_t>(min_encoded_value));
-		estimated_bits_per_value = NumericCast<uint32_t>(std::ceil(std::log2(delta + 1)));
+		estimated_bits_per_value = ExactNumericCast<uint32_t>(std::ceil(std::log2(delta + 1)));
 		estimated_compression_size += n_values * estimated_bits_per_value;
 		estimated_compression_size +=
 		    exceptions_count * (EXACT_TYPE_BITSIZE + (AlpConstants::EXCEPTION_POSITION_SIZE * 8));
@@ -255,7 +255,7 @@ struct AlpCompression {
 		//! We sample equidistant values within a vector; to do this we skip a fixed number of values
 		vector<T> vector_sample;
 		auto idx_increments = MaxValue<uint32_t>(
-		    1, UnsafeNumericCast<uint32_t>(std::ceil((double)n_values / AlpConstants::SAMPLES_PER_VECTOR)));
+		    1, ExactNumericCast<uint32_t>(std::ceil((double)n_values / AlpConstants::SAMPLES_PER_VECTOR)));
 		for (idx_t i = 0; i < n_values; i += idx_increments) {
 			vector_sample.push_back(input_vector[i]);
 		}

package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_analyze.hpp

@@ -10,9 +10,10 @@
 
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/storage/compression/alp/algorithm/alp.hpp"
-#include "duckdb/storage/compression/alp/alp_utils.hpp"
 #include "duckdb/storage/compression/alp/alp_constants.hpp"
+#include "duckdb/storage/compression/alp/alp_utils.hpp"
 #include "duckdb/storage/compression/patas/patas.hpp"
+#include "duckdb/storage/table/column_data.hpp"
 
 #include <cmath>
 
@@ -23,7 +24,7 @@ struct AlpAnalyzeState : public AnalyzeState {
 public:
 	using EXACT_TYPE = typename FloatingToExact<T>::TYPE;
 
-	AlpAnalyzeState() : state() {
+	explicit AlpAnalyzeState(const CompressionInfo &info) : AnalyzeState(info), state() {
 	}
 
 	idx_t total_bytes_used = 0;
@@ -62,7 +63,7 @@ public:
 		idx_t bytes_to_be_used = AlignValue(current_bytes_used_in_segment + RequiredSpace());
 		// We have enough space if the already used space + the required space for a new vector
 		// does not exceed the space of the block - the segment header (the pointer to the metadata)
-		return bytes_to_be_used <= (Storage::BLOCK_SIZE - AlpConstants::METADATA_POINTER_SIZE);
+		return bytes_to_be_used <= (info.GetBlockSize() - AlpConstants::METADATA_POINTER_SIZE);
 	}
 
 	idx_t TotalUsedBytes() const {
@@ -72,7 +73,8 @@ public:
 
 template <class T>
 unique_ptr<AnalyzeState> AlpInitAnalyze(ColumnData &col_data, PhysicalType type) {
-	return make_uniq<AlpAnalyzeState<T>>();
+	CompressionInfo info(col_data.GetBlockManager().GetBlockSize());
+	return make_uniq<AlpAnalyzeState<T>>(info);
 }
 
 /*

package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_compress.hpp

@@ -8,21 +8,19 @@
 
 #pragma once
 
-#include "duckdb/storage/compression/patas/patas.hpp"
-#include "duckdb/function/compression_function.hpp"
-#include "duckdb/storage/compression/alp/algorithm/alp.hpp"
-#include "duckdb/storage/compression/alp/alp_analyze.hpp"
-
 #include "duckdb/common/helper.hpp"
 #include "duckdb/common/limits.hpp"
+#include "duckdb/common/operator/subtract.hpp"
 #include "duckdb/common/types/null_value.hpp"
 #include "duckdb/function/compression/compression.hpp"
+#include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-
+#include "duckdb/storage/compression/alp/algorithm/alp.hpp"
+#include "duckdb/storage/compression/alp/alp_analyze.hpp"
+#include "duckdb/storage/compression/patas/patas.hpp"
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
-#include "duckdb/common/operator/subtract.hpp"
 
 #include <functional>
 
@@ -33,8 +31,10 @@ struct AlpCompressionState : public CompressionState {
 
 public:
 	using EXACT_TYPE = typename FloatingToExact<T>::TYPE;
-	explicit AlpCompressionState(ColumnDataCheckpointer &checkpointer, AlpAnalyzeState<T> *analyze_state)
-	    : checkpointer(checkpointer), function(checkpointer.GetCompressionFunction(CompressionType::COMPRESSION_ALP)) {
+
+	AlpCompressionState(ColumnDataCheckpointer &checkpointer, AlpAnalyzeState<T> *analyze_state)
+	    : CompressionState(analyze_state->info), checkpointer(checkpointer),
+	      function(checkpointer.GetCompressionFunction(CompressionType::COMPRESSION_ALP)) {
 		CreateEmptySegment(checkpointer.GetRowGroup().start);
 
 		//! Combinations found on the analyze step are needed for compression
@@ -92,17 +92,19 @@ public:
 	void CreateEmptySegment(idx_t row_start) {
 		auto &db = checkpointer.GetDatabase();
 		auto &type = checkpointer.GetType();
-		auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
+
+		auto compressed_segment =
+		    ColumnSegment::CreateTransientSegment(db, type, row_start, info.GetBlockSize(), info.GetBlockSize());
 		current_segment = std::move(compressed_segment);
 		current_segment->function = function;
+
 		auto &buffer_manager = BufferManager::GetBufferManager(current_segment->db);
 		handle = buffer_manager.Pin(current_segment->block);
 
-		// Pointer to the start of the compressed data
+		// The pointer to the start of the compressed data.
 		data_ptr = handle.Ptr() + current_segment->GetBlockOffset() + AlpConstants::HEADER_SIZE;
-		// Pointer to the start of the Metadata
-		metadata_ptr = handle.Ptr() + current_segment->GetBlockOffset() + Storage::BLOCK_SIZE;
-
+		// The pointer to the start of the metadata.
+		metadata_ptr = handle.Ptr() + current_segment->GetBlockOffset() + info.GetBlockSize();
 		next_vector_byte_index_start = AlpConstants::HEADER_SIZE;
 	}
 
@@ -120,7 +122,7 @@ public:
 
 		if (vector_idx != nulls_idx) { //! At least there is one valid value in the vector
 			for (idx_t i = 0; i < vector_idx; i++) {
-				NumericStats::Update<T>(current_segment->stats.statistics, input_vector[i]);
+				current_segment->stats.statistics.UpdateNumericStats<T>(input_vector[i]);
 			}
 		}
 		current_segment->count += vector_idx;
@@ -183,10 +185,10 @@ public:
 		// Verify that the metadata_ptr is not smaller than the space used by the data
 		D_ASSERT(dataptr + metadata_offset <= metadata_ptr);
 
-		auto bytes_used_by_metadata = UnsafeNumericCast<idx_t>(dataptr + Storage::BLOCK_SIZE - metadata_ptr);
+		auto bytes_used_by_metadata = UnsafeNumericCast<idx_t>(dataptr + info.GetBlockSize() - metadata_ptr);
 
 		// Initially the total segment size is the size of the block
-		auto total_segment_size = Storage::BLOCK_SIZE;
+		auto total_segment_size = info.GetBlockSize();
 
 		//! We compact the block if the space used is less than a threshold
 		const auto used_space_percentage =

package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_constants.hpp

@@ -35,8 +35,8 @@ public:
 	static constexpr double COMPACT_BLOCK_THRESHOLD = 0.80;
 
 	// Largest double which fits into an int64
-	static constexpr double ENCODING_UPPER_LIMIT = 9223372036854774784;
-	static constexpr double ENCODING_LOWER_LIMIT = -9223372036854774784;
+	static constexpr double ENCODING_UPPER_LIMIT = 9223372036854774784.0;
+	static constexpr double ENCODING_LOWER_LIMIT = -9223372036854774784.0;
 
 	static constexpr uint8_t MAX_COMBINATIONS = 5;
 

package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_scan.hpp

@@ -130,7 +130,7 @@ public:
 		// Load the offset (metadata) indicating where the vector data starts
 		metadata_ptr -= AlpConstants::METADATA_POINTER_SIZE;
 		auto data_byte_offset = Load<uint32_t>(metadata_ptr);
-		D_ASSERT(data_byte_offset < Storage::BLOCK_SIZE);
+		D_ASSERT(data_byte_offset < segment.GetBlockManager().GetBlockSize());
 
 		idx_t vector_size = MinValue((idx_t)AlpConstants::ALP_VECTOR_SIZE, (count - total_value_count));
 
@@ -177,12 +177,11 @@ public:
 public:
 	//! Skip the next 'skip_count' values, we don't store the values
 	void Skip(ColumnSegment &col_segment, idx_t skip_count) {
-
 		if (total_value_count != 0 && !VectorFinished()) {
 			// Finish skipping the current vector
-			idx_t to_skip = LeftInVector();
-			skip_count -= to_skip;
+			idx_t to_skip = MinValue<idx_t>(skip_count, LeftInVector());
 			ScanVector<T, true>(nullptr, to_skip);
+			skip_count -= to_skip;
 		}
 		// Figure out how many entire vectors we can skip
 		// For these vectors, we don't even need to process the metadata or values

package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_utils.hpp

@@ -41,8 +41,9 @@ public:
 		    NumericCast<uint32_t>(MinValue(current_vector_n_values, (idx_t)AlpConstants::ALP_VECTOR_SIZE));
 		//! We sample equidistant values within a vector; to do this we jump a fixed number of values
 		uint32_t n_sampled_increments = MaxValue<uint32_t>(
-		    1, UnsafeNumericCast<uint32_t>(std::ceil((double)n_lookup_values / AlpConstants::SAMPLES_PER_VECTOR)));
-		uint32_t n_sampled_values = NumericCast<uint32_t>(std::ceil((double)n_lookup_values / n_sampled_increments));
+		    1, ExactNumericCast<uint32_t>(std::ceil((double)n_lookup_values / AlpConstants::SAMPLES_PER_VECTOR)));
+		uint32_t n_sampled_values =
+		    ExactNumericCast<uint32_t>(std::ceil((double)n_lookup_values / n_sampled_increments));
 		D_ASSERT(n_sampled_values < AlpConstants::ALP_VECTOR_SIZE);
 
 		AlpSamplingParameters sampling_params = {n_lookup_values, n_sampled_increments, n_sampled_values};

package/src/duckdb/src/include/duckdb/storage/compression/alprd/algorithm/alprd.hpp

@@ -73,7 +73,8 @@ struct AlpRDCompression {
 	                                      uint64_t sample_count) {
 		double exceptions_size =
 		    exceptions_count * ((AlpRDConstants::EXCEPTION_POSITION_SIZE + AlpRDConstants::EXCEPTION_SIZE) * 8);
-		double estimated_size = right_bit_width + left_bit_width + (exceptions_size / sample_count);
+		double estimated_size =
+		    right_bit_width + left_bit_width + (exceptions_size / static_cast<double>(sample_count));
 		return estimated_size;
 	}
 
@@ -106,7 +107,7 @@ struct AlpRDCompression {
 		uint64_t actual_dictionary_size =
 		    MinValue<uint64_t>(AlpRDConstants::MAX_DICTIONARY_SIZE, left_parts_sorted_repetitions.size());
 		uint8_t left_bit_width =
-		    MaxValue<uint8_t>(1, NumericCast<uint8_t>(std::ceil(std::log2(actual_dictionary_size))));
+		    MaxValue<uint8_t>(1, ExactNumericCast<uint8_t>(std::ceil(std::log2(actual_dictionary_size))));
 
 		if (PERSIST_DICT) {
 			for (idx_t dict_idx = 0; dict_idx < actual_dictionary_size; dict_idx++) {

package/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp

@@ -8,13 +8,14 @@
 
 #pragma once
 
+#include "duckdb/common/numeric_utils.hpp"
 #include "duckdb/function/compression_function.hpp"
-#include "duckdb/storage/compression/patas/patas.hpp"
+#include "duckdb/storage/compression/alp/alp_constants.hpp"
+#include "duckdb/storage/compression/alp/alp_utils.hpp"
 #include "duckdb/storage/compression/alprd/algorithm/alprd.hpp"
 #include "duckdb/storage/compression/alprd/alprd_constants.hpp"
-#include "duckdb/storage/compression/alp/alp_utils.hpp"
-#include "duckdb/storage/compression/alp/alp_constants.hpp"
-#include "duckdb/common/numeric_utils.hpp"
+#include "duckdb/storage/compression/patas/patas.hpp"
+#include "duckdb/storage/table/column_data.hpp"
 
 #include <cmath>
 
@@ -25,7 +26,7 @@ struct AlpRDAnalyzeState : public AnalyzeState {
 public:
 	using EXACT_TYPE = typename FloatingToExact<T>::TYPE;
 
-	AlpRDAnalyzeState() : state() {
+	explicit AlpRDAnalyzeState(const CompressionInfo &info) : AnalyzeState(info), state() {
 	}
 
 	idx_t vectors_count = 0;
@@ -37,7 +38,8 @@ public:
 
 template <class T>
 unique_ptr<AnalyzeState> AlpRDInitAnalyze(ColumnData &col_data, PhysicalType type) {
-	return make_uniq<AlpRDAnalyzeState<T>>();
+	CompressionInfo info(col_data.GetBlockManager().GetBlockSize());
+	return make_uniq<AlpRDAnalyzeState<T>>(info);
 }
 
 /*
@@ -126,15 +128,15 @@ idx_t AlpRDFinalAnalyze(AnalyzeState &state) {
 	//! Overhead per vector: Pointer to data + Exceptions count
 	double per_vector_overhead = AlpRDConstants::METADATA_POINTER_SIZE + AlpRDConstants::EXCEPTIONS_COUNT_SIZE;
 
-	uint32_t n_vectors =
-	    NumericCast<uint32_t>(std::ceil((double)analyze_state.total_values_count / AlpRDConstants::ALP_VECTOR_SIZE));
+	uint32_t n_vectors = LossyNumericCast<uint32_t>(
+	    std::ceil((double)analyze_state.total_values_count / AlpRDConstants::ALP_VECTOR_SIZE));
 
 	auto estimated_size = (estimed_compressed_bytes * factor_of_sampling) + (n_vectors * per_vector_overhead);
-	uint32_t estimated_n_blocks =
-	    NumericCast<uint32_t>(std::ceil(estimated_size / (Storage::BLOCK_SIZE - per_segment_overhead)));
+	uint32_t estimated_n_blocks = LossyNumericCast<uint32_t>(
+	    std::ceil(estimated_size / (static_cast<double>(state.info.GetBlockSize()) - per_segment_overhead)));
 
 	auto final_analyze_size = estimated_size + (estimated_n_blocks * per_segment_overhead);
-	return NumericCast<idx_t>(final_analyze_size);
+	return LossyNumericCast<idx_t>(final_analyze_size);
 }
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_compress.hpp

@@ -8,23 +8,21 @@
 
 #pragma once
 
-#include "duckdb/storage/compression/patas/patas.hpp"
-#include "duckdb/storage/compression/alprd/algorithm/alprd.hpp"
-#include "duckdb/function/compression_function.hpp"
-#include "duckdb/storage/compression/alprd/alprd_analyze.hpp"
-#include "duckdb/storage/compression/alprd/alprd_constants.hpp"
-#include "duckdb/storage/compression/alp/alp_constants.hpp"
-
 #include "duckdb/common/helper.hpp"
 #include "duckdb/common/limits.hpp"
+#include "duckdb/common/operator/subtract.hpp"
 #include "duckdb/common/types/null_value.hpp"
 #include "duckdb/function/compression/compression.hpp"
+#include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-
+#include "duckdb/storage/compression/alp/alp_constants.hpp"
+#include "duckdb/storage/compression/alprd/algorithm/alprd.hpp"
+#include "duckdb/storage/compression/alprd/alprd_analyze.hpp"
+#include "duckdb/storage/compression/alprd/alprd_constants.hpp"
+#include "duckdb/storage/compression/patas/patas.hpp"
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
-#include "duckdb/common/operator/subtract.hpp"
 
 #include <functional>
 
@@ -35,8 +33,9 @@ struct AlpRDCompressionState : public CompressionState {
 
 public:
 	using EXACT_TYPE = typename FloatingToExact<T>::TYPE;
-	explicit AlpRDCompressionState(ColumnDataCheckpointer &checkpointer, AlpRDAnalyzeState<T> *analyze_state)
-	    : checkpointer(checkpointer),
+
+	AlpRDCompressionState(ColumnDataCheckpointer &checkpointer, AlpRDAnalyzeState<T> *analyze_state)
+	    : CompressionState(analyze_state->info), checkpointer(checkpointer),
 	      function(checkpointer.GetCompressionFunction(CompressionType::COMPRESSION_ALPRD)) {
 		//! State variables from the analyze step that are needed for compression
 		state.left_parts_dict_map = std::move(analyze_state->state.left_parts_dict_map);
@@ -103,19 +102,20 @@ public:
 	void CreateEmptySegment(idx_t row_start) {
 		auto &db = checkpointer.GetDatabase();
 		auto &type = checkpointer.GetType();
-		auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
+
+		auto compressed_segment =
+		    ColumnSegment::CreateTransientSegment(db, type, row_start, info.GetBlockSize(), info.GetBlockSize());
 		compressed_segment->function = function;
 		current_segment = std::move(compressed_segment);
 
 		auto &buffer_manager = BufferManager::GetBufferManager(db);
 		handle = buffer_manager.Pin(current_segment->block);
 
-		// Pointer to the start of the compressed data
+		// The pointer to the start of the compressed data.
 		data_ptr = handle.Ptr() + current_segment->GetBlockOffset() + AlpRDConstants::HEADER_SIZE +
 		           actual_dictionary_size_bytes;
-		// Pointer to the start of the Metadata
-		metadata_ptr = handle.Ptr() + current_segment->GetBlockOffset() + Storage::BLOCK_SIZE;
-
+		// The pointer to the start of the metadata.
+		metadata_ptr = handle.Ptr() + current_segment->GetBlockOffset() + info.GetBlockSize();
 		next_vector_byte_index_start = AlpRDConstants::HEADER_SIZE + actual_dictionary_size_bytes;
 	}
 
@@ -134,7 +134,7 @@ public:
 		if (vector_idx != nulls_idx) { //! At least there is one valid value in the vector
 			for (idx_t i = 0; i < vector_idx; i++) {
 				T floating_point_value = Load<T>(const_data_ptr_cast(&input_vector[i]));
-				NumericStats::Update<T>(current_segment->stats.statistics, floating_point_value);
+				current_segment->stats.statistics.UpdateNumericStats<T>(floating_point_value);
 			}
 		}
 		current_segment->count += vector_idx;
@@ -185,10 +185,10 @@ public:
 		// Verify that the metadata_ptr is not smaller than the space used by the data
 		D_ASSERT(dataptr + metadata_offset <= metadata_ptr);
 
-		auto bytes_used_by_metadata = UnsafeNumericCast<idx_t>(dataptr + Storage::BLOCK_SIZE - metadata_ptr);
+		auto bytes_used_by_metadata = UnsafeNumericCast<idx_t>(dataptr + info.GetBlockSize() - metadata_ptr);
 
 		// Initially the total segment size is the size of the block
-		auto total_segment_size = Storage::BLOCK_SIZE;
+		auto total_segment_size = info.GetBlockSize();
 
 		//! We compact the block if the space used is less than a threshold
 		const auto used_space_percentage =

package/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_scan.hpp

@@ -149,7 +149,7 @@ public:
 		// Load the offset (metadata) indicating where the vector data starts
 		metadata_ptr -= AlpRDConstants::METADATA_POINTER_SIZE;
 		auto data_byte_offset = Load<uint32_t>(metadata_ptr);
-		D_ASSERT(data_byte_offset < Storage::BLOCK_SIZE);
+		D_ASSERT(data_byte_offset < segment.GetBlockManager().GetBlockSize());
 
 		idx_t vector_size = MinValue((idx_t)AlpRDConstants::ALP_VECTOR_SIZE, (count - total_value_count));
 
@@ -184,12 +184,11 @@ public:
 public:
 	//! Skip the next 'skip_count' values, we don't store the values
 	void Skip(ColumnSegment &col_segment, idx_t skip_count) {
-
 		if (total_value_count != 0 && !VectorFinished()) {
 			// Finish skipping the current vector
-			idx_t to_skip = LeftInVector();
-			skip_count -= to_skip;
+			idx_t to_skip = MinValue<idx_t>(skip_count, LeftInVector());
 			ScanVector<EXACT_TYPE, true>(nullptr, to_skip);
+			skip_count -= to_skip;
 		}
 		// Figure out how many entire vectors we can skip
 		// For these vectors, we don't even need to process the metadata or values

package/src/duckdb/src/include/duckdb/storage/compression/chimp/chimp_scan.hpp

@@ -193,7 +193,7 @@ public:
 		// Load the offset indicating where a groups data starts
 		metadata_ptr -= sizeof(uint32_t);
 		auto data_byte_offset = Load<uint32_t>(metadata_ptr);
-		D_ASSERT(data_byte_offset < Storage::BLOCK_SIZE);
+		D_ASSERT(data_byte_offset < segment.GetBlockManager().GetBlockSize());
 		//  Only used for point queries
 		(void)data_byte_offset;
 

package/src/duckdb/src/include/duckdb/storage/compression/patas/patas_scan.hpp

@@ -156,7 +156,7 @@ public:
 		// Load the offset indicating where a groups data starts
 		metadata_ptr -= sizeof(uint32_t);
 		auto data_byte_offset = Load<uint32_t>(metadata_ptr);
-		D_ASSERT(data_byte_offset < Storage::BLOCK_SIZE);
+		D_ASSERT(data_byte_offset < segment.GetBlockManager().GetBlockSize());
 
 		// Initialize the byte_reader with the data values for the group
 		group_state.Init(segment_data + data_byte_offset);

package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp

@@ -37,11 +37,19 @@ struct ColumnSegmentState {
 		DynamicCastCheck<TARGET>(this);
 		return reinterpret_cast<const TARGET &>(*this);
 	}
+
+public:
+	vector<block_id_t> blocks;
 };
 
 struct DataPointer {
-	explicit DataPointer(BaseStatistics stats) : statistics(std::move(stats)) {
-	}
+	explicit DataPointer(BaseStatistics stats);
+	// disable copy constructors
+	DataPointer(const DataPointer &other) = delete;
+	DataPointer &operator=(const DataPointer &) = delete;
+	//! enable move constructors
+	DUCKDB_API DataPointer(DataPointer &&other) noexcept;
+	DUCKDB_API DataPointer &operator=(DataPointer &&) noexcept;
 
 	uint64_t row_start;
 	uint64_t tuple_count;

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -149,7 +149,7 @@ public:
 	//! Commit the append
 	void CommitAppend(transaction_t commit_id, idx_t row_start, idx_t count);
 	//! Write a segment of the table to the WAL
-	void WriteToLog(WriteAheadLog &log, idx_t row_start, idx_t count);
+	void WriteToLog(WriteAheadLog &log, idx_t row_start, idx_t count, optional_ptr<StorageCommitState> commit_state);
 	//! Revert a set of appends made by the given AppendState, used to revert appends in the event of an error during
 	//! commit (e.g. because of an I/O exception)
 	void RevertAppend(idx_t start_row, idx_t count);
@@ -158,7 +158,7 @@ public:
 	void ScanTableSegment(idx_t start_row, idx_t count, const std::function<void(DataChunk &chunk)> &function);
 
 	//! Merge a row group collection directly into this table - appending it to the end of the table without copying
-	void MergeStorage(RowGroupCollection &data, TableIndexList &indexes);
+	void MergeStorage(RowGroupCollection &data, TableIndexList &indexes, optional_ptr<StorageCommitState> commit_state);
 
 	//! Append a chunk with the row ids [row_start, ..., row_start + chunk.size()] to all indexes of the table, returns
 	//! whether or not the append succeeded
@@ -219,6 +219,7 @@ public:
 	bool HasForeignKeyIndex(const vector<PhysicalIndex> &keys, ForeignKeyType type);
 	void SetIndexStorageInfo(vector<IndexStorageInfo> index_storage_info);
 	void VacuumIndexes();
+	void CleanupAppend(transaction_t lowest_transaction, idx_t start, idx_t count);
 
 	string GetTableName() const;
 	void SetTableName(string new_name);

package/src/duckdb/src/include/duckdb/storage/in_memory_block_manager.hpp

@@ -29,12 +29,18 @@ public:
 	block_id_t GetFreeBlockId() override {
 		throw InternalException("Cannot perform IO in in-memory database - GetFreeBlockId!");
 	}
+	block_id_t PeekFreeBlockId() override {
+		throw InternalException("Cannot perform IO in in-memory database - PeekFreeBlockId!");
+	}
 	bool IsRootBlock(MetaBlockPointer root) override {
 		throw InternalException("Cannot perform IO in in-memory database - IsRootBlock!");
 	}
 	void MarkBlockAsFree(block_id_t block_id) override {
 		throw InternalException("Cannot perform IO in in-memory database - MarkBlockAsFree!");
 	}
+	void MarkBlockAsUsed(block_id_t block_id) override {
+		throw InternalException("Cannot perform IO in in-memory database - MarkBlockAsUsed!");
+	}
 	void MarkBlockAsModified(block_id_t block_id) override {
 		throw InternalException("Cannot perform IO in in-memory database - MarkBlockAsModified!");
 	}
@@ -47,12 +53,21 @@ public:
 	void Read(Block &block) override {
 		throw InternalException("Cannot perform IO in in-memory database - Read!");
 	}
+	void ReadBlocks(FileBuffer &buffer, block_id_t start_block, idx_t block_count) override {
+		throw InternalException("Cannot perform IO in in-memory database - ReadBlocks!");
+	}
 	void Write(FileBuffer &block, block_id_t block_id) override {
 		throw InternalException("Cannot perform IO in in-memory database - Write!");
 	}
 	void WriteHeader(DatabaseHeader header) override {
 		throw InternalException("Cannot perform IO in in-memory database - WriteHeader!");
 	}
+	bool InMemory() override {
+		return true;
+	}
+	void FileSync() override {
+		throw InternalException("Cannot perform IO in in-memory database - FileSync!");
+	}
 	idx_t TotalBlocks() override {
 		throw InternalException("Cannot perform IO in in-memory database - TotalBlocks!");
 	}