duckdb 0.10.1-dev9.0 → 0.10.1

package/src/duckdb/src/include/duckdb/planner/extension_callback.hpp

@@ -18,6 +18,12 @@ public:
 	virtual ~ExtensionCallback() {
 	}
 
+	//! Called when a new connection is opened
+	virtual void OnConnectionOpened(ClientContext &context) {
+	}
+	//! Called when a connection is closed
+	virtual void OnConnectionClosed(ClientContext &context) {
+	}
 	//! Called after an extension is finished loading
 	virtual void OnExtensionLoaded(DatabaseInstance &db, const string &name) {
 	}

package/src/duckdb/src/include/duckdb/planner/operator/list.hpp

@@ -25,7 +25,6 @@
 #include "duckdb/planner/operator/logical_insert.hpp"
 #include "duckdb/planner/operator/logical_join.hpp"
 #include "duckdb/planner/operator/logical_limit.hpp"
-#include "duckdb/planner/operator/logical_limit_percent.hpp"
 #include "duckdb/planner/operator/logical_materialized_cte.hpp"
 #include "duckdb/planner/operator/logical_order.hpp"
 #include "duckdb/planner/operator/logical_pivot.hpp"

package/src/duckdb/src/include/duckdb/planner/operator/logical_insert.hpp

@@ -53,7 +53,7 @@ public:
 	// The types of the columns targeted by the DO UPDATE SET expressions
 	vector<LogicalType> set_types;
 	// The table_index referring to the column references qualified with 'excluded'
-	idx_t excluded_table_index;
+	idx_t excluded_table_index = 0;
 	// The columns to fetch from the 'destination' table
 	vector<column_t> columns_to_fetch;
 	// The columns to fetch from the 'source' table

package/src/duckdb/src/include/duckdb/planner/operator/logical_limit.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/planner/bound_result_modifier.hpp"
 
 namespace duckdb {
 
@@ -18,15 +19,10 @@ public:
 	static constexpr const LogicalOperatorType TYPE = LogicalOperatorType::LOGICAL_LIMIT;
 
 public:
-	LogicalLimit(int64_t limit_val, int64_t offset_val, unique_ptr<Expression> limit, unique_ptr<Expression> offset);
-
-	//! Limit and offset values in case they are constants, used in optimizations.
-	int64_t limit_val;
-	int64_t offset_val;
-	//! The maximum amount of elements to emit
-	unique_ptr<Expression> limit;
-	//! The offset from the start to begin emitting elements
-	unique_ptr<Expression> offset;
+	LogicalLimit(BoundLimitNode limit_val, BoundLimitNode offset_val);
+
+	BoundLimitNode limit_val;
+	BoundLimitNode offset_val;
 
 public:
 	vector<ColumnBinding> GetColumnBindings() override;

package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp

@@ -26,21 +26,6 @@ protected:
 	unique_ptr<Expression> VisitReplace(BoundColumnRefExpression &expr, unique_ptr<Expression> *expr_ptr) override;
 	unique_ptr<Expression> VisitReplace(BoundSubqueryExpression &expr, unique_ptr<Expression> *expr_ptr) override;
 
-private:
-	//! Helper class used to recursively rewrite correlated expressions within nested subqueries.
-	class RewriteCorrelatedRecursive {
-	public:
-		RewriteCorrelatedRecursive(BoundSubqueryExpression &parent, ColumnBinding base_binding,
-		                           column_binding_map_t<idx_t> &correlated_map);
-		void RewriteJoinRefRecursive(BoundTableRef &ref);
-		void RewriteCorrelatedSubquery(BoundSubqueryExpression &expr);
-		void RewriteCorrelatedExpressions(Expression &child);
-
-		BoundSubqueryExpression &parent;
-		ColumnBinding base_binding;
-		column_binding_map_t<idx_t> &correlated_map;
-	};
-
 private:
 	ColumnBinding base_binding;
 	column_binding_map_t<idx_t> &correlated_map;

package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp

@@ -46,7 +46,10 @@ public:
 	DUCKDB_API ArenaChunk *GetTail();
 
 	DUCKDB_API bool IsEmpty() const;
+	//! Get the total *used* size (not cached)
 	DUCKDB_API idx_t SizeInBytes() const;
+	//! Get the currently allocated size in bytes (cached, read from "allocated_size")
+	DUCKDB_API idx_t AllocationSize() const;
 
 	//! Returns an "Allocator" wrapper for this arena allocator
 	Allocator &GetAllocator() {
@@ -61,6 +64,8 @@ private:
 	ArenaChunk *tail;
 	//! An allocator wrapper using this arena allocator
 	Allocator arena_allocator;
+	//! The total allocated size
+	idx_t allocated_size = 0;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/buffer/buffer_pool.hpp

@@ -29,7 +29,6 @@ struct BufferEvictionNode {
 	idx_t timestamp;
 
 	bool CanUnload(BlockHandle &handle_p);
-
 	shared_ptr<BlockHandle> TryGetBlockHandle();
 };
 
@@ -73,9 +72,24 @@ protected:
 	virtual EvictionResult EvictBlocks(MemoryTag tag, idx_t extra_memory, idx_t memory_limit,
 	                                   unique_ptr<FileBuffer> *buffer = nullptr);
 
-	//! Garbage collect eviction queue
+	//! Tries to dequeue an element from the eviction queue, but only after acquiring the purge queue lock.
+	bool TryDequeueWithLock(BufferEvictionNode &node);
+	//! Bulk purge dead nodes from the eviction queue. Then, enqueue those that are still alive.
+	void PurgeIteration(const idx_t purge_size);
+	//! Garbage collect dead nodes in the eviction queue.
 	void PurgeQueue();
-	void AddToEvictionQueue(shared_ptr<BlockHandle> &handle);
+	//! Add a buffer handle to the eviction queue. Returns true, if the queue is
+	//! ready to be purged, and false otherwise.
+	bool AddToEvictionQueue(shared_ptr<BlockHandle> &handle);
+
+	//! Increment the dead node counter in the purge queue.
+	inline void IncrementDeadNodes() {
+		total_dead_nodes++;
+	}
+	//! Decrement the dead node counter in the purge queue.
+	inline void DecrementDeadNodes() {
+		total_dead_nodes--;
+	}
 
 protected:
 	//! The lock for changing the memory limit
@@ -86,12 +100,33 @@ protected:
 	atomic<idx_t> maximum_memory;
 	//! Eviction queue
 	unique_ptr<EvictionQueue> queue;
-	//! Total number of insertions into the eviction queue. This guides the schedule for calling PurgeQueue.
-	atomic<uint32_t> queue_insertions;
 	//! Memory manager for concurrently used temporary memory, e.g., for physical operators
 	unique_ptr<TemporaryMemoryManager> temporary_memory_manager;
 	//! Memory usage per tag
 	atomic<idx_t> memory_usage_per_tag[MEMORY_TAG_COUNT];
+
+	//! We trigger a purge of the eviction queue every INSERT_INTERVAL insertions
+	constexpr static idx_t INSERT_INTERVAL = 4096;
+	//! We multiply the base purge size by this value.
+	constexpr static idx_t PURGE_SIZE_MULTIPLIER = 2;
+	//! We multiply the purge size by this value to determine early-outs. This is the minimum queue size.
+	//! We never purge below this point.
+	constexpr static idx_t EARLY_OUT_MULTIPLIER = 4;
+	//! We multiply the approximate alive nodes by this value to test whether our total dead nodes
+	//! exceed their allowed ratio. Must be greater than 1.
+	constexpr static idx_t ALIVE_NODE_MULTIPLIER = 4;
+
+	//! Total number of insertions into the eviction queue. This guides the schedule for calling PurgeQueue.
+	atomic<idx_t> evict_queue_insertions;
+	//! Total dead nodes in the eviction queue. There are two scenarios in which a node dies: (1) we destroy its block
+	//! handle, or (2) we insert a newer version into the eviction queue.
+	atomic<idx_t> total_dead_nodes;
+	//! Locked, if a queue purge is currently active or we're trying to forcefully evict a node.
+	//! Only lets a single thread enter the purge phase.
+	mutex purge_lock;
+
+	//! A pre-allocated vector of eviction nodes. We reuse this to keep the allocation overhead of purges small.
+	vector<BufferEvictionNode> purge_nodes;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp

@@ -306,7 +306,7 @@ struct AlpCompression {
 			state.encoded_integers[i] = encoded_value;
 			//! We detect exceptions using a predicated comparison
 			auto is_exception = (decoded_value != actual_value);
-			state.exceptions_positions[exceptions_idx] = i;
+			state.exceptions_positions[exceptions_idx] = UnsafeNumericCast<uint16_t>(i);
 			exceptions_idx += is_exception;
 		}
 

package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_analyze.hpp

@@ -115,7 +115,7 @@ bool AlpAnalyze(AnalyzeState &state, Vector &input, idx_t count) {
 			T value = data[idx];
 			//! We resolve null values with a predicated comparison
 			bool is_null = !vdata.validity.RowIsValid(idx);
-			current_vector_null_positions[nulls_idx] = i;
+			current_vector_null_positions[nulls_idx] = UnsafeNumericCast<uint16_t>(i);
 			nulls_idx += is_null;
 			current_vector_values[i] = value;
 		}
@@ -164,6 +164,10 @@ idx_t AlpFinalAnalyze(AnalyzeState &state) {
 	// Flush last unfinished segment
 	analyze_state.FlushSegment();
 
+	if (compressed_values == 0) {
+		return DConstants::INVALID_INDEX;
+	}
+
 	// We estimate the size by taking into account the portion of the values we took
 	const auto factor_of_sampling = analyze_state.total_values_count / compressed_values;
 	const auto final_analyze_size = analyze_state.TotalUsedBytes() * factor_of_sampling;

package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_compress.hpp

@@ -141,7 +141,7 @@ public:
 		Store<uint64_t>(state.frame_of_reference, data_ptr);
 		data_ptr += AlpConstants::FOR_SIZE;
 
-		Store<uint8_t>(state.bit_width, data_ptr);
+		Store<uint8_t>(UnsafeNumericCast<uint8_t>(state.bit_width), data_ptr);
 		data_ptr += AlpConstants::BIT_WIDTH_SIZE;
 
 		memcpy((void *)data_ptr, (void *)state.values_encoded, state.bp_size);
@@ -166,7 +166,7 @@ public:
 		// Write pointer to the vector data (metadata)
 		metadata_ptr -= sizeof(uint32_t);
 		Store<uint32_t>(next_vector_byte_index_start, metadata_ptr);
-		next_vector_byte_index_start = UsedSpace();
+		next_vector_byte_index_start = NumericCast<uint32_t>(UsedSpace());
 
 		vectors_flushed++;
 		vector_idx = 0;
@@ -206,7 +206,7 @@ public:
 		}
 
 		// Store the offset to the end of metadata (to be used as a backwards pointer in decoding)
-		Store<uint32_t>(total_segment_size, dataptr);
+		Store<uint32_t>(NumericCast<uint32_t>(total_segment_size), dataptr);
 
 		handle.Destroy();
 		checkpoint_state.FlushSegment(std::move(current_segment), total_segment_size);
@@ -244,7 +244,7 @@ public:
 					T value = data[idx];
 					bool is_null = !vdata.validity.RowIsValid(idx);
 					//! We resolve null values with a predicated comparison
-					vector_null_positions[nulls_idx] = vector_idx + i;
+					vector_null_positions[nulls_idx] = UnsafeNumericCast<uint16_t>(vector_idx + i);
 					nulls_idx += is_null;
 					input_vector[vector_idx + i] = value;
 				}

package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_utils.hpp

@@ -37,7 +37,8 @@ public:
 public:
 	static AlpSamplingParameters GetSamplingParameters(idx_t current_vector_n_values) {
 
-		uint32_t n_lookup_values = MinValue(current_vector_n_values, (idx_t)AlpConstants::ALP_VECTOR_SIZE);
+		auto n_lookup_values =
+		    NumericCast<uint32_t>(MinValue(current_vector_n_values, (idx_t)AlpConstants::ALP_VECTOR_SIZE));
 		//! We sample equidistant values within a vector; to do this we jump a fixed number of values
 		uint32_t n_sampled_increments =
 		    MaxValue(1, (int32_t)std::ceil((double)n_lookup_values / AlpConstants::SAMPLES_PER_VECTOR));

package/src/duckdb/src/include/duckdb/storage/compression/alprd/algorithm/alprd.hpp

@@ -16,6 +16,7 @@
 #include "duckdb/common/pair.hpp"
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/common/limits.hpp"
+#include "duckdb/common/numeric_utils.hpp"
 
 #include <cmath>
 
@@ -109,7 +110,8 @@ struct AlpRDCompression {
 		if (PERSIST_DICT) {
 			for (idx_t dict_idx = 0; dict_idx < actual_dictionary_size; dict_idx++) {
 				//! The dict keys are mapped to the left part themselves
-				state.left_parts_dict[dict_idx] = left_parts_sorted_repetitions[dict_idx].hash;
+				state.left_parts_dict[dict_idx] =
+				    UnsafeNumericCast<uint16_t>(left_parts_sorted_repetitions[dict_idx].hash);
 				state.left_parts_dict_map.insert({state.left_parts_dict[dict_idx], dict_idx});
 			}
 			//! Pararelly we store a map of the dictionary to quickly resolve exceptions during encoding
@@ -118,24 +120,24 @@ struct AlpRDCompression {
 			}
 			state.left_bit_width = left_bit_width;
 			state.right_bit_width = right_bit_width;
-			state.actual_dictionary_size = actual_dictionary_size;
+			state.actual_dictionary_size = UnsafeNumericCast<uint8_t>(actual_dictionary_size);
 
 			D_ASSERT(state.left_bit_width > 0 && state.right_bit_width > 0 &&
 			         state.left_bit_width <= AlpRDConstants::MAX_DICTIONARY_BIT_WIDTH &&
 			         state.actual_dictionary_size <= AlpRDConstants::MAX_DICTIONARY_SIZE);
 		}
 
-		double estimated_size =
-		    EstimateCompressionSize(right_bit_width, left_bit_width, exceptions_count, values.size());
+		double estimated_size = EstimateCompressionSize(right_bit_width, left_bit_width,
+		                                                UnsafeNumericCast<uint16_t>(exceptions_count), values.size());
 		return estimated_size;
 	}
 
 	static double FindBestDictionary(const vector<EXACT_TYPE> &values, State &state) {
-		uint8_t right_bit_width;
+		uint8_t right_bit_width = 0;
 		double best_dict_size = NumericLimits<int32_t>::Maximum();
 		//! Finding the best position to CUT the values
 		for (idx_t i = 1; i <= AlpRDConstants::CUTTING_LIMIT; i++) {
-			uint8_t candidate_right_bit_width = EXACT_TYPE_BITSIZE - i;
+			uint8_t candidate_right_bit_width = UnsafeNumericCast<uint8_t>(EXACT_TYPE_BITSIZE - i);
 			double estimated_size = BuildLeftPartsDictionary<false>(values, candidate_right_bit_width, state);
 			if (estimated_size <= best_dict_size) {
 				right_bit_width = candidate_right_bit_width;
@@ -156,7 +158,7 @@ struct AlpRDCompression {
 		for (idx_t i = 0; i < n_values; i++) {
 			EXACT_TYPE tmp = input_vector[i];
 			right_parts[i] = tmp & ((1ULL << state.right_bit_width) - 1);
-			left_parts[i] = (tmp >> state.right_bit_width);
+			left_parts[i] = UnsafeNumericCast<uint16_t>(tmp >> state.right_bit_width);
 		}
 
 		// Dictionary encoding for left parts
@@ -174,7 +176,7 @@ struct AlpRDCompression {
 			//! Left parts not found in the dictionary are stored as exceptions
 			if (dictionary_index >= state.actual_dictionary_size) {
 				state.exceptions[state.exceptions_count] = dictionary_key;
-				state.exceptions_positions[state.exceptions_count] = i;
+				state.exceptions_positions[state.exceptions_count] = UnsafeNumericCast<uint16_t>(i);
 				state.exceptions_count++;
 			}
 		}

package/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp

@@ -14,6 +14,7 @@
 #include "duckdb/storage/compression/alprd/alprd_constants.hpp"
 #include "duckdb/storage/compression/alp/alp_utils.hpp"
 #include "duckdb/storage/compression/alp/alp_constants.hpp"
+#include "duckdb/common/numeric_utils.hpp"
 
 #include <cmath>
 
@@ -82,7 +83,7 @@ bool AlpRDAnalyze(AnalyzeState &state, Vector &input, idx_t count) {
 			current_vector_sample[sample_idx] = value;
 			//! We resolve null values with a predicated comparison
 			bool is_null = !vdata.validity.RowIsValid(idx);
-			current_vector_null_positions[nulls_idx] = sample_idx;
+			current_vector_null_positions[nulls_idx] = UnsafeNumericCast<uint16_t>(sample_idx);
 			nulls_idx += is_null;
 			sample_idx++;
 		}
@@ -108,6 +109,9 @@ bool AlpRDAnalyze(AnalyzeState &state, Vector &input, idx_t count) {
 template <class T>
 idx_t AlpRDFinalAnalyze(AnalyzeState &state) {
 	auto &analyze_state = (AlpRDAnalyzeState<T> &)state;
+	if (analyze_state.total_values_count == 0) {
+		return DConstants::INVALID_INDEX;
+	}
 	double factor_of_sampling = 1 / ((double)analyze_state.rowgroup_sample.size() / analyze_state.total_values_count);
 
 	// Finding which is the best dictionary for the sample

package/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_compress.hpp

@@ -168,7 +168,7 @@ public:
 		// Write pointer to the vector data (metadata)
 		metadata_ptr -= AlpRDConstants::METADATA_POINTER_SIZE;
 		Store<uint32_t>(next_vector_byte_index_start, metadata_ptr);
-		next_vector_byte_index_start = UsedSpace();
+		next_vector_byte_index_start = NumericCast<uint32_t>(UsedSpace());
 
 		vectors_flushed++;
 		vector_idx = 0;
@@ -208,7 +208,7 @@ public:
 		}
 
 		// Store the offset to the end of metadata (to be used as a backwards pointer in decoding)
-		Store<uint32_t>(total_segment_size, dataptr);
+		Store<uint32_t>(NumericCast<uint32_t>(total_segment_size), dataptr);
 		dataptr += AlpRDConstants::METADATA_POINTER_SIZE;
 
 		// Store the right bw for the segment
@@ -261,7 +261,7 @@ public:
 					EXACT_TYPE value = Load<EXACT_TYPE>(const_data_ptr_cast(&data[idx]));
 					bool is_null = !vdata.validity.RowIsValid(idx);
 					//! We resolve null values with a predicated comparison
-					vector_null_positions[nulls_idx] = vector_idx + i;
+					vector_null_positions[nulls_idx] = UnsafeNumericCast<uint16_t>(vector_idx + i);
 					nulls_idx += is_null;
 					input_vector[vector_idx + i] = value;
 				}

package/src/duckdb/src/include/duckdb/storage/compression/chimp/algorithm/bit_reader.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb.h"
+#include "duckdb/common/numeric_utils.hpp"
 
 namespace duckdb {
 
@@ -71,7 +72,7 @@ public:
 	}
 
 	inline uint8_t InnerReadByte(const uint8_t &offset) {
-		uint8_t result = input[ByteIndex() + offset] << BitIndex() |
+		uint8_t result = static_cast<uint8_t>(input[ByteIndex() + offset] << BitIndex()) |
 		                 ((input[ByteIndex() + offset + 1] & REMAINDER_MASKS[8 + BitIndex()]) >> (8 - BitIndex()));
 		return result;
 	}
@@ -114,30 +115,30 @@ public:
 	inline T ReadBytes(const uint8_t &remainder) {
 		T result = 0;
 		if (BYTES > 0) {
-			result = result << 8 | InnerReadByte(0);
+			result = UnsafeNumericCast<T>(result << 8 | InnerReadByte(0));
 		}
 		if (BYTES > 1) {
-			result = result << 8 | InnerReadByte(1);
+			result = UnsafeNumericCast<T>(result << 8 | InnerReadByte(1));
 		}
 		if (BYTES > 2) {
-			result = result << 8 | InnerReadByte(2);
+			result = UnsafeNumericCast<T>(result << 8 | InnerReadByte(2));
 		}
 		if (BYTES > 3) {
-			result = result << 8 | InnerReadByte(3);
+			result = UnsafeNumericCast<T>(result << 8 | InnerReadByte(3));
 		}
 		if (BYTES > 4) {
-			result = result << 8 | InnerReadByte(4);
+			result = UnsafeNumericCast<T>(result << 8 | InnerReadByte(4));
 		}
 		if (BYTES > 5) {
-			result = result << 8 | InnerReadByte(5);
+			result = UnsafeNumericCast<T>(result << 8 | InnerReadByte(5));
 		}
 		if (BYTES > 6) {
-			result = result << 8 | InnerReadByte(6);
+			result = UnsafeNumericCast<T>(result << 8 | InnerReadByte(6));
 		}
 		if (BYTES > 7) {
-			result = result << 8 | InnerReadByte(7);
+			result = UnsafeNumericCast<T>(result << 8 | InnerReadByte(7));
 		}
-		result = result << remainder | InnerRead(remainder, BYTES);
+		result = UnsafeNumericCast<T>(result << remainder | InnerRead(remainder, BYTES));
 		index += (BYTES << 3) + remainder;
 		return result;
 	}

package/src/duckdb/src/include/duckdb/storage/compression/chimp/algorithm/bit_utils.hpp

@@ -8,12 +8,14 @@
 
 #pragma once
 
+#include "duckdb/common/numeric_utils.hpp"
+
 namespace duckdb {
 
 template <class R>
 struct BitUtils {
 	static constexpr R Mask(unsigned int const bits) {
-		return (((uint64_t)(bits < (sizeof(R) * 8))) << (bits & ((sizeof(R) * 8) - 1))) - 1U;
+		return UnsafeNumericCast<R>((((uint64_t)(bits < (sizeof(R) * 8))) << (bits & ((sizeof(R) * 8) - 1))) - 1U);
 	}
 };
 

package/src/duckdb/src/include/duckdb/storage/compression/chimp/algorithm/chimp128.hpp

@@ -257,7 +257,7 @@ public:
 		case ChimpConstants::Flags::VALUE_IDENTICAL: {
 			//! Value is identical to previous value
 			auto index = state.input.template ReadValue<uint8_t, 7>();
-			result = state.ring_buffer.Value(index);
+			result = UnsafeNumericCast<CHIMP_TYPE>(state.ring_buffer.Value(index));
 			break;
 		}
 		case ChimpConstants::Flags::TRAILING_EXCEEDS_THRESHOLD: {

package/src/duckdb/src/include/duckdb/storage/compression/chimp/algorithm/leading_zero_buffer.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb.h"
 #include "duckdb/common/helper.hpp"
+#include "duckdb/common/numeric_utils.hpp"
 #ifdef DEBUG
 #include "duckdb/common/vector.hpp"
 #include "duckdb/common/assert.hpp"
@@ -91,7 +92,8 @@ public:
 public:
 #ifdef DEBUG
 	uint8_t ExtractValue(uint32_t value, uint8_t index) {
-		return (value & LeadingZeroBufferConstants::MASKS[index]) >> LeadingZeroBufferConstants::SHIFTS[index];
+		return NumericCast<uint8_t>((value & LeadingZeroBufferConstants::MASKS[index]) >>
+		                            LeadingZeroBufferConstants::SHIFTS[index]);
 	}
 #endif
 
@@ -140,8 +142,8 @@ public:
 		const auto buffer_idx = BlockIndex();
 		auto const temp = Load<uint32_t>(buffer + buffer_idx);
 
-		const uint8_t result =
-		    (temp & LeadingZeroBufferConstants::MASKS[counter & 7]) >> LeadingZeroBufferConstants::SHIFTS[counter & 7];
+		const uint8_t result = UnsafeNumericCast<uint8_t>((temp & LeadingZeroBufferConstants::MASKS[counter & 7]) >>
+		                                                  LeadingZeroBufferConstants::SHIFTS[counter & 7]);
 		counter++;
 		return result;
 	}

package/src/duckdb/src/include/duckdb/storage/compression/chimp/chimp_scan.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/storage/compression/chimp/algorithm/chimp_utils.hpp"
 
 #include "duckdb/common/limits.hpp"
+#include "duckdb/common/numeric_utils.hpp"
 #include "duckdb/common/types/null_value.hpp"
 #include "duckdb/function/compression/compression.hpp"
 #include "duckdb/function/compression_function.hpp"
@@ -210,7 +211,7 @@ public:
 		auto group_size = MinValue<idx_t>(segment_count - total_value_count, ChimpPrimitives::CHIMP_SEQUENCE_SIZE);
 		// Reduce by one, because the first value of a group does not have a flag
 		auto flag_count = group_size - 1;
-		uint16_t flag_byte_count = (AlignValue<uint16_t, 4>(flag_count) / 4);
+		uint16_t flag_byte_count = AlignValue<uint16_t, 4>(UnsafeNumericCast<uint16_t>(flag_count)) / 4;
 
 		// Load the flags
 		metadata_ptr -= flag_byte_count;

package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp

@@ -29,7 +29,7 @@ struct ColumnSegmentState {
 
 	template <class TARGET>
 	TARGET &Cast() {
-		D_ASSERT(dynamic_cast<TARGET *>(this));
+		DynamicCastCheck<TARGET>(this);
 		return reinterpret_cast<TARGET &>(*this);
 	}
 	template <class TARGET>

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -128,9 +128,11 @@ public:
 	//! Fetches an append lock
 	void AppendLock(TableAppendState &state);
 	//! Begin appending structs to this table, obtaining necessary locks, etc
-	void InitializeAppend(DuckTransaction &transaction, TableAppendState &state, idx_t append_count);
+	void InitializeAppend(DuckTransaction &transaction, TableAppendState &state);
 	//! Append a chunk to the table using the AppendState obtained from InitializeAppend
 	void Append(DataChunk &chunk, TableAppendState &state);
+	//! Finalize an append
+	void FinalizeAppend(DuckTransaction &transaction, TableAppendState &state);
 	//! Commit the append
 	void CommitAppend(transaction_t commit_id, idx_t row_start, idx_t count);
 	//! Write a segment of the table to the WAL

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -154,7 +154,7 @@ private:
 public:
 	template <class TARGET>
 	TARGET &Cast() {
-		D_ASSERT(dynamic_cast<TARGET *>(this));
+		DynamicCastCheck<TARGET>(this);
 		return reinterpret_cast<TARGET &>(*this);
 	}
 

package/src/duckdb/src/include/duckdb/storage/partial_block_manager.hpp

@@ -69,7 +69,7 @@ public:
 public:
 	template <class TARGET>
 	TARGET &Cast() {
-		D_ASSERT(dynamic_cast<TARGET *>(this));
+		DynamicCastCheck<TARGET>(this);
 		return reinterpret_cast<TARGET &>(*this);
 	}
 };

package/src/duckdb/src/include/duckdb/storage/standard_buffer_manager.hpp

@@ -40,6 +40,9 @@ public:
 
 public:
 	static unique_ptr<StandardBufferManager> CreateBufferManager(DatabaseInstance &db, string temp_directory);
+	static unique_ptr<FileBuffer> ReadTemporaryBufferInternal(BufferManager &buffer_manager, FileHandle &handle,
+	                                                          idx_t position, idx_t size,
+	                                                          unique_ptr<FileBuffer> reusable_buffer);
 	//! Registers an in-memory buffer that cannot be unloaded until it is destroyed
 	//! This buffer can be small (smaller than BLOCK_SIZE)
 	//! Unpin and pin are nops on this block of memory

package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp

@@ -91,7 +91,7 @@ protected:
 public:
 	template <class TARGET>
 	TARGET &Cast() {
-		D_ASSERT(dynamic_cast<TARGET *>(this));
+		DynamicCastCheck<TARGET>(this);
 		return reinterpret_cast<TARGET &>(*this);
 	}
 	template <class TARGET>

package/src/duckdb/src/include/duckdb/storage/table/append_state.hpp

@@ -67,8 +67,6 @@ struct TableAppendState {
 	RowGroup *start_row_group;
 	//! The transaction data
 	TransactionData transaction;
-	//! The remaining append count, only if the append count is known beforehand
-	idx_t remaining;
 };
 
 struct LocalAppendState {

package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp

@@ -46,7 +46,7 @@ public:
 public:
 	template <class TARGET>
 	TARGET &Cast() {
-		D_ASSERT(dynamic_cast<TARGET *>(this));
+		DynamicCastCheck<TARGET>(this);
 		return reinterpret_cast<TARGET &>(*this);
 	}
 	template <class TARGET>

package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -68,7 +68,9 @@ public:
 	DataTableInfo &GetTableInfo() const;
 	virtual idx_t GetMaxEntry();
 
-	void IncrementVersion();
+	idx_t GetAllocationSize() const {
+		return allocation_size;
+	}
 
 	virtual void SetStart(idx_t new_start);
 	//! The root type of the column
@@ -164,10 +166,10 @@ protected:
 	mutex update_lock;
 	//! The updates for this column segment
 	unique_ptr<UpdateSegment> updates;
-	//! The internal version of the column data
-	idx_t version;
 	//! The stats of the root segment
 	unique_ptr<SegmentStatistics> stats;
+	//! Total transient allocation size
+	idx_t allocation_size;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp

@@ -68,8 +68,8 @@ public:
 	//! Fetch a value of the specific row id and append it to the result
 	void FetchRow(ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx);
 
-	static idx_t FilterSelection(SelectionVector &sel, Vector &result, const TableFilter &filter,
-	                             idx_t &approved_tuple_count, ValidityMask &mask);
+	static idx_t FilterSelection(SelectionVector &sel, Vector &vector, UnifiedVectorFormat &vdata,
+	                             const TableFilter &filter, idx_t scan_count, idx_t &approved_tuple_count);
 
 	//! Skip a scan forward to the row_index specified in the scan state
 	void Skip(ColumnScanState &state);

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -139,6 +139,10 @@ public:
 
 	void GetColumnSegmentInfo(idx_t row_group_index, vector<ColumnSegmentInfo> &result);
 
+	idx_t GetAllocationSize() const {
+		return allocation_size;
+	}
+
 	void Verify();
 
 	void NextVector(CollectionScanState &state);
@@ -172,6 +176,7 @@ private:
 	unique_ptr<atomic<bool>[]> is_loaded;
 	vector<MetaBlockPointer> deletes_pointers;
 	atomic<bool> deletes_is_loaded;
+	idx_t allocation_size;
 };
 
 } // namespace duckdb