duckdb 0.7.2-dev16.0 → 0.7.2-dev225.0

package/src/duckdb/src/storage/buffer_manager.cpp

@@ -7,6 +7,7 @@
 #include "duckdb/storage/in_memory_block_manager.hpp"
 #include "duckdb/storage/storage_manager.hpp"
 #include "duckdb/main/attached_database.hpp"
+#include "duckdb/main/database.hpp"
 
 namespace duckdb {
 
@@ -71,11 +72,11 @@ BlockHandle::~BlockHandle() { // NOLINT: allow internal exceptions
 		D_ASSERT(memory_charge.size > 0);
 		// the block is still loaded in memory: erase it
 		buffer.reset();
-		memory_charge.Resize(buffer_manager.current_memory, 0);
+		memory_charge.Resize(buffer_manager.buffer_pool.current_memory, 0);
 	} else {
 		D_ASSERT(memory_charge.size == 0);
 	}
-	buffer_manager.PurgeQueue();
+	buffer_manager.buffer_pool.PurgeQueue();
 	block_manager.UnregisterBlock(block_id, can_destroy);
 }
 
@@ -150,7 +151,7 @@ unique_ptr<FileBuffer> BlockHandle::UnloadAndTakeBlock() {
 		// temporary block that cannot be destroyed: write to temporary file
 		block_manager.buffer_manager.WriteTemporaryBuffer(block_id, *buffer);
 	}
-	memory_charge.Resize(block_manager.buffer_manager.current_memory, 0);
+	memory_charge.Resize(block_manager.buffer_manager.buffer_pool.current_memory, 0);
 	state = BlockState::BLOCK_UNLOADED;
 	return std::move(buffer);
 }
@@ -241,9 +242,14 @@ void BufferManager::SetTemporaryDirectory(string new_dir) {
 	this->temp_directory = std::move(new_dir);
 }
 
-BufferManager::BufferManager(DatabaseInstance &db, string tmp, idx_t maximum_memory)
-    : db(db), current_memory(0), maximum_memory(maximum_memory), temp_directory(std::move(tmp)),
-      queue(make_unique<EvictionQueue>()), temporary_id(MAXIMUM_BLOCK), queue_insertions(0),
+BufferPool::BufferPool(idx_t maximum_memory)
+    : current_memory(0), maximum_memory(maximum_memory), queue(make_unique<EvictionQueue>()), queue_insertions(0) {
+}
+BufferPool::~BufferPool() {
+}
+
+BufferManager::BufferManager(DatabaseInstance &db, string tmp)
+    : db(db), buffer_pool(db.GetBufferPool()), temp_directory(std::move(tmp)), temporary_id(MAXIMUM_BLOCK),
       buffer_allocator(BufferAllocatorAllocate, BufferAllocatorFree, BufferAllocatorRealloc,
                        make_unique<BufferAllocatorData>(*this)) {
 	temp_block_manager = make_unique<InMemoryBlockManager>(*this);
@@ -311,15 +317,15 @@ shared_ptr<BlockHandle> BlockManager::ConvertToPersistent(block_id_t block_id, s
 	// persist the new block to disk
 	Write(*new_block->buffer, block_id);
 
-	buffer_manager.AddToEvictionQueue(new_block);
+	buffer_manager.buffer_pool.AddToEvictionQueue(new_block);
 
 	return new_block;
 }
 
 template <typename... ARGS>
-TempBufferPoolReservation BufferManager::EvictBlocksOrThrow(idx_t memory_delta, idx_t limit,
-                                                            unique_ptr<FileBuffer> *buffer, ARGS... args) {
-	auto r = EvictBlocks(memory_delta, limit, buffer);
+TempBufferPoolReservation BufferManager::EvictBlocksOrThrow(idx_t memory_delta, unique_ptr<FileBuffer> *buffer,
+                                                            ARGS... args) {
+	auto r = buffer_pool.EvictBlocks(memory_delta, buffer_pool.maximum_memory, buffer);
 	if (!r.success) {
 		throw OutOfMemoryException(args..., InMemoryWarning());
 	}
@@ -328,9 +334,8 @@ TempBufferPoolReservation BufferManager::EvictBlocksOrThrow(idx_t memory_delta,
 
 shared_ptr<BlockHandle> BufferManager::RegisterSmallMemory(idx_t block_size) {
 	D_ASSERT(block_size < Storage::BLOCK_SIZE);
-	auto res = EvictBlocksOrThrow(block_size, maximum_memory, nullptr,
-	                              "could not allocate block of %lld bytes (%lld/%lld used) %s", block_size,
-	                              GetUsedMemory(), GetMaxMemory());
+	auto res = EvictBlocksOrThrow(block_size, nullptr, "could not allocate block of %lld bytes (%lld/%lld used) %s",
+	                              block_size, GetUsedMemory(), GetMaxMemory());
 
 	auto buffer = ConstructManagedBuffer(block_size, nullptr, FileBufferType::TINY_BUFFER);
 
@@ -344,9 +349,9 @@ shared_ptr<BlockHandle> BufferManager::RegisterMemory(idx_t block_size, bool can
 	auto alloc_size = GetAllocSize(block_size);
 	// first evict blocks until we have enough memory to store this buffer
 	unique_ptr<FileBuffer> reusable_buffer;
-	auto res = EvictBlocksOrThrow(alloc_size, maximum_memory, &reusable_buffer,
-	                              "could not allocate block of %lld bytes (%lld/%lld used) %s", alloc_size,
-	                              GetUsedMemory(), GetMaxMemory());
+	auto res =
+	    EvictBlocksOrThrow(alloc_size, &reusable_buffer, "could not allocate block of %lld bytes (%lld/%lld used) %s",
+	                       alloc_size, GetUsedMemory(), GetMaxMemory());
 
 	auto buffer = ConstructManagedBuffer(block_size, std::move(reusable_buffer));
 
@@ -376,14 +381,13 @@ void BufferManager::ReAllocate(shared_ptr<BlockHandle> &handle, idx_t block_size
 		return;
 	} else if (memory_delta > 0) {
 		// evict blocks until we have space to resize this block
-		auto reservation =
-		    EvictBlocksOrThrow(memory_delta, maximum_memory, nullptr, "failed to resize block from %lld to %lld%s",
-		                       handle->memory_usage, req.alloc_size);
+		auto reservation = EvictBlocksOrThrow(memory_delta, nullptr, "failed to resize block from %lld to %lld%s",
+		                                      handle->memory_usage, req.alloc_size);
 		// EvictBlocks decrements 'current_memory' for us.
 		handle->memory_charge.Merge(std::move(reservation));
 	} else {
 		// no need to evict blocks, but we do need to decrement 'current_memory'.
-		handle->memory_charge.Resize(current_memory, req.alloc_size);
+		handle->memory_charge.Resize(buffer_pool.current_memory, req.alloc_size);
 	}
 
 	// resize and adjust current memory
@@ -407,15 +411,15 @@ BufferHandle BufferManager::Pin(shared_ptr<BlockHandle> &handle) {
 	}
 	// evict blocks until we have space for the current block
 	unique_ptr<FileBuffer> reusable_buffer;
-	auto reservation = EvictBlocksOrThrow(required_memory, maximum_memory, &reusable_buffer,
-	                                      "failed to pin block of size %lld%s", required_memory);
+	auto reservation =
+	    EvictBlocksOrThrow(required_memory, &reusable_buffer, "failed to pin block of size %lld%s", required_memory);
 	// lock the handle again and repeat the check (in case anybody loaded in the mean time)
 	lock_guard<mutex> lock(handle->lock);
 	// check if the block is already loaded
 	if (handle->state == BlockState::BLOCK_LOADED) {
 		// the block is loaded, increment the reader count and return a pointer to the handle
 		handle->readers++;
-		reservation.Resize(current_memory, 0);
+		reservation.Resize(buffer_pool.current_memory, 0);
 		return handle->Load(handle);
 	}
 	// now we can actually load the current block
@@ -428,13 +432,13 @@ BufferHandle BufferManager::Pin(shared_ptr<BlockHandle> &handle) {
 	if (delta) {
 		D_ASSERT(delta < 0);
 		handle->memory_usage += delta;
-		handle->memory_charge.Resize(current_memory, handle->memory_usage);
+		handle->memory_charge.Resize(buffer_pool.current_memory, handle->memory_usage);
 	}
 	D_ASSERT(handle->memory_usage == handle->buffer->AllocSize());
 	return buf;
 }
 
-void BufferManager::AddToEvictionQueue(shared_ptr<BlockHandle> &handle) {
+void BufferPool::AddToEvictionQueue(shared_ptr<BlockHandle> &handle) {
 	constexpr int INSERT_INTERVAL = 1024;
 
 	D_ASSERT(handle->readers == 0);
@@ -465,12 +469,12 @@ void BufferManager::Unpin(shared_ptr<BlockHandle> &handle) {
 	handle->readers--;
 	if (handle->readers == 0) {
 		VerifyZeroReaders(handle);
-		AddToEvictionQueue(handle);
+		buffer_pool.AddToEvictionQueue(handle);
 	}
 }
 
-BufferManager::EvictionResult BufferManager::EvictBlocks(idx_t extra_memory, idx_t memory_limit,
-                                                         unique_ptr<FileBuffer> *buffer) {
+BufferPool::EvictionResult BufferPool::EvictBlocks(idx_t extra_memory, idx_t memory_limit,
+                                                   unique_ptr<FileBuffer> *buffer) {
 	BufferEvictionNode node;
 	TempBufferPoolReservation r(current_memory, extra_memory);
 	while (current_memory > memory_limit) {
@@ -504,7 +508,7 @@ BufferManager::EvictionResult BufferManager::EvictBlocks(idx_t extra_memory, idx
 	return {true, std::move(r)};
 }
 
-void BufferManager::PurgeQueue() {
+void BufferPool::PurgeQueue() {
 	BufferEvictionNode node;
 	while (true) {
 		if (!queue->q.try_dequeue(node)) {
@@ -531,13 +535,13 @@ void BlockManager::UnregisterBlock(block_id_t block_id, bool can_destroy) {
 	}
 }
 
-void BufferManager::SetLimit(idx_t limit) {
+void BufferPool::SetLimit(idx_t limit, const char *exception_postscript) {
 	lock_guard<mutex> l_lock(limit_lock);
 	// try to evict until the limit is reached
 	if (!EvictBlocks(0, limit).success) {
 		throw OutOfMemoryException(
 		    "Failed to change memory limit to %lld: could not free up enough memory for the new limit%s", limit,
-		    InMemoryWarning());
+		    exception_postscript);
 	}
 	idx_t old_limit = maximum_memory;
 	// set the global maximum memory to the new limit if successful
@@ -548,20 +552,16 @@ void BufferManager::SetLimit(idx_t limit) {
 		maximum_memory = old_limit;
 		throw OutOfMemoryException(
 		    "Failed to change memory limit to %lld: could not free up enough memory for the new limit%s", limit,
-		    InMemoryWarning());
+		    exception_postscript);
 	}
 }
 
 void BufferManager::IncreaseUsedMemory(idx_t size) {
-	if (current_memory + size > maximum_memory) {
-		throw OutOfMemoryException("Failed to allocate data of size %lld%s", size, InMemoryWarning());
-	}
-	current_memory += size;
+	ReserveMemory(size);
 }
 
 void BufferManager::DecreaseUsedMemory(idx_t size) {
-	D_ASSERT(current_memory >= size);
-	current_memory -= size;
+	FreeReservedMemory(size);
 }
 
 //===--------------------------------------------------------------------===//
@@ -1044,7 +1044,7 @@ vector<TemporaryFileInformation> BufferManager::GetTemporaryFiles() {
 	return result;
 }
 
-string BufferManager::InMemoryWarning() {
+const char *BufferManager::InMemoryWarning() {
 	if (!temp_directory.empty()) {
 		return "";
 	}
@@ -1058,8 +1058,7 @@ void BufferManager::ReserveMemory(idx_t size) {
 	if (size == 0) {
 		return;
 	}
-	auto reservation =
-	    EvictBlocksOrThrow(size, maximum_memory, nullptr, "failed to reserve memory data of size %lld%s", size);
+	auto reservation = EvictBlocksOrThrow(size, nullptr, "failed to reserve memory data of size %lld%s", size);
 	reservation.size = 0;
 }
 
@@ -1067,7 +1066,7 @@ void BufferManager::FreeReservedMemory(idx_t size) {
 	if (size == 0) {
 		return;
 	}
-	current_memory -= size;
+	buffer_pool.current_memory -= size;
 }
 
 //===--------------------------------------------------------------------===//
@@ -1075,8 +1074,7 @@ void BufferManager::FreeReservedMemory(idx_t size) {
 //===--------------------------------------------------------------------===//
 data_ptr_t BufferManager::BufferAllocatorAllocate(PrivateAllocatorData *private_data, idx_t size) {
 	auto &data = (BufferAllocatorData &)*private_data;
-	auto reservation = data.manager.EvictBlocksOrThrow(size, data.manager.maximum_memory, nullptr,
-	                                                   "failed to allocate data of size %lld%s", size);
+	auto reservation = data.manager.EvictBlocksOrThrow(size, nullptr, "failed to allocate data of size %lld%s", size);
 	// We rely on manual tracking of this one. :(
 	reservation.size = 0;
 	return Allocator::Get(data.manager.db).AllocateData(size);
@@ -1086,7 +1084,7 @@ void BufferManager::BufferAllocatorFree(PrivateAllocatorData *private_data, data
 	auto &data = (BufferAllocatorData &)*private_data;
 	BufferPoolReservation r;
 	r.size = size;
-	r.Resize(data.manager.current_memory, 0);
+	r.Resize(data.manager.buffer_pool.current_memory, 0);
 	return Allocator::Get(data.manager.db).FreeData(pointer, size);
 }
 
@@ -1098,7 +1096,7 @@ data_ptr_t BufferManager::BufferAllocatorRealloc(PrivateAllocatorData *private_d
 	auto &data = (BufferAllocatorData &)*private_data;
 	BufferPoolReservation r;
 	r.size = old_size;
-	r.Resize(data.manager.current_memory, size);
+	r.Resize(data.manager.buffer_pool.current_memory, size);
 	r.size = 0;
 	return Allocator::Get(data.manager.db).ReallocateData(pointer, old_size, size);
 }

package/src/duckdb/src/storage/compression/bitpacking.cpp

@@ -347,7 +347,7 @@ idx_t BitpackingFinalAnalyze(AnalyzeState &state) {
 //===--------------------------------------------------------------------===//
 // Compress
 //===--------------------------------------------------------------------===//
-template <class T, class T_S = typename std::make_signed<T>::type>
+template <class T, bool WRITE_STATISTICS, class T_S = typename std::make_signed<T>::type>
 struct BitpackingCompressState : public CompressionState {
 public:
 	explicit BitpackingCompressState(ColumnDataCheckpointer &checkpointer) : checkpointer(checkpointer) {
@@ -377,7 +377,7 @@ public:
 public:
 	struct BitpackingWriter {
 		static void WriteConstant(T constant, idx_t count, void *data_ptr, bool all_invalid) {
-			auto state = (BitpackingCompressState<T> *)data_ptr;
+			auto state = (BitpackingCompressState<T, WRITE_STATISTICS> *)data_ptr;
 
 			ReserveSpace(state, sizeof(T));
 			WriteMetaData(state, BitpackingMode::CONSTANT);
@@ -388,7 +388,7 @@ public:
 
 		static void WriteConstantDelta(T_S constant, T frame_of_reference, idx_t count, T *values, bool *validity,
 		                               void *data_ptr) {
-			auto state = (BitpackingCompressState<T> *)data_ptr;
+			auto state = (BitpackingCompressState<T, WRITE_STATISTICS> *)data_ptr;
 
 			ReserveSpace(state, 2 * sizeof(T));
 			WriteMetaData(state, BitpackingMode::CONSTANT_DELTA);
@@ -400,7 +400,7 @@ public:
 
 		static void WriteDeltaFor(T *values, bool *validity, bitpacking_width_t width, T frame_of_reference,
 		                          T_S delta_offset, T *original_values, idx_t count, void *data_ptr) {
-			auto state = (BitpackingCompressState<T> *)data_ptr;
+			auto state = (BitpackingCompressState<T, WRITE_STATISTICS> *)data_ptr;
 
 			auto bp_size = BitpackingPrimitives::GetRequiredSize(count, width);
 			ReserveSpace(state, bp_size + 3 * sizeof(T));
@@ -418,7 +418,7 @@ public:
 
 		static void WriteFor(T *values, bool *validity, bitpacking_width_t width, T frame_of_reference, idx_t count,
 		                     void *data_ptr) {
-			auto state = (BitpackingCompressState<T> *)data_ptr;
+			auto state = (BitpackingCompressState<T, WRITE_STATISTICS> *)data_ptr;
 
 			auto bp_size = BitpackingPrimitives::GetRequiredSize(count, width);
 			ReserveSpace(state, bp_size + 2 * sizeof(T));
@@ -439,22 +439,22 @@ public:
 			ptr += sizeof(T_OUT);
 		}
 
-		static void WriteMetaData(BitpackingCompressState<T> *state, BitpackingMode mode) {
+		static void WriteMetaData(BitpackingCompressState<T, WRITE_STATISTICS> *state, BitpackingMode mode) {
 			bitpacking_metadata_t metadata {mode, (uint32_t)(state->data_ptr - state->handle.Ptr())};
 			state->metadata_ptr -= sizeof(bitpacking_metadata_encoded_t);
 			Store<bitpacking_metadata_encoded_t>(EncodeMeta(metadata), state->metadata_ptr);
 		}
 
-		static void ReserveSpace(BitpackingCompressState<T> *state, idx_t data_bytes) {
+		static void ReserveSpace(BitpackingCompressState<T, WRITE_STATISTICS> *state, idx_t data_bytes) {
 			idx_t meta_bytes = sizeof(bitpacking_metadata_encoded_t);
 			state->FlushAndCreateSegmentIfFull(data_bytes, meta_bytes);
 			D_ASSERT(state->CanStore(data_bytes, meta_bytes));
 		}
 
-		static void UpdateStats(BitpackingCompressState<T> *state, idx_t count) {
+		static void UpdateStats(BitpackingCompressState<T, WRITE_STATISTICS> *state, idx_t count) {
 			state->current_segment->count += count;
 
-			if (!state->state.all_invalid) {
+			if (WRITE_STATISTICS && !state->state.all_invalid) {
 				NumericStatistics::Update<T>(state->current_segment->stats, state->state.minimum);
 				NumericStatistics::Update<T>(state->current_segment->stats, state->state.maximum);
 			}
@@ -487,8 +487,8 @@ public:
 
 		for (idx_t i = 0; i < count; i++) {
 			auto idx = vdata.sel->get_index(i);
-			state.template Update<BitpackingCompressState<T, T_S>::BitpackingWriter>(data[idx],
-			                                                                         vdata.validity.RowIsValid(idx));
+			state.template Update<BitpackingCompressState<T, WRITE_STATISTICS, T_S>::BitpackingWriter>(
+			    data[idx], vdata.validity.RowIsValid(idx));
 		}
 	}
 
@@ -524,29 +524,29 @@ public:
 	}
 
 	void Finalize() {
-		state.template Flush<BitpackingCompressState<T, T_S>::BitpackingWriter>();
+		state.template Flush<BitpackingCompressState<T, WRITE_STATISTICS, T_S>::BitpackingWriter>();
 		FlushSegment();
 		current_segment.reset();
 	}
 };
 
-template <class T>
+template <class T, bool WRITE_STATISTICS>
 unique_ptr<CompressionState> BitpackingInitCompression(ColumnDataCheckpointer &checkpointer,
                                                        unique_ptr<AnalyzeState> state) {
-	return make_unique<BitpackingCompressState<T>>(checkpointer);
+	return make_unique<BitpackingCompressState<T, WRITE_STATISTICS>>(checkpointer);
 }
 
-template <class T>
+template <class T, bool WRITE_STATISTICS>
 void BitpackingCompress(CompressionState &state_p, Vector &scan_vector, idx_t count) {
-	auto &state = (BitpackingCompressState<T> &)state_p;
+	auto &state = (BitpackingCompressState<T, WRITE_STATISTICS> &)state_p;
 	UnifiedVectorFormat vdata;
 	scan_vector.ToUnifiedFormat(count, vdata);
 	state.Append(vdata, count);
 }
 
-template <class T>
+template <class T, bool WRITE_STATISTICS>
 void BitpackingFinalizeCompress(CompressionState &state_p) {
-	auto &state = (BitpackingCompressState<T> &)state_p;
+	auto &state = (BitpackingCompressState<T, WRITE_STATISTICS> &)state_p;
 	state.Finalize();
 }
 
@@ -880,11 +880,12 @@ void BitpackingSkip(ColumnSegment &segment, ColumnScanState &state, idx_t skip_c
 //===--------------------------------------------------------------------===//
 // Get Function
 //===--------------------------------------------------------------------===//
-template <class T>
+template <class T, bool WRITE_STATISTICS = true>
 CompressionFunction GetBitpackingFunction(PhysicalType data_type) {
 	return CompressionFunction(CompressionType::COMPRESSION_BITPACKING, data_type, BitpackingInitAnalyze<T>,
-	                           BitpackingAnalyze<T>, BitpackingFinalAnalyze<T>, BitpackingInitCompression<T>,
-	                           BitpackingCompress<T>, BitpackingFinalizeCompress<T>, BitpackingInitScan<T>,
+	                           BitpackingAnalyze<T>, BitpackingFinalAnalyze<T>,
+	                           BitpackingInitCompression<T, WRITE_STATISTICS>, BitpackingCompress<T, WRITE_STATISTICS>,
+	                           BitpackingFinalizeCompress<T, WRITE_STATISTICS>, BitpackingInitScan<T>,
 	                           BitpackingScan<T>, BitpackingScanPartial<T>, BitpackingFetchRow<T>, BitpackingSkip<T>);
 }
 
@@ -907,6 +908,8 @@ CompressionFunction BitpackingFun::GetFunction(PhysicalType type) {
 		return GetBitpackingFunction<uint32_t>(type);
 	case PhysicalType::UINT64:
 		return GetBitpackingFunction<uint64_t>(type);
+	case PhysicalType::LIST:
+		return GetBitpackingFunction<uint64_t, false>(type);
 	default:
 		throw InternalException("Unsupported type for Bitpacking");
 	}
@@ -923,6 +926,7 @@ bool BitpackingFun::TypeIsSupported(PhysicalType type) {
 	case PhysicalType::UINT16:
 	case PhysicalType::UINT32:
 	case PhysicalType::UINT64:
+	case PhysicalType::LIST:
 		return true;
 	default:
 		return false;

package/src/duckdb/src/storage/compression/fixed_size_uncompressed.cpp

@@ -153,13 +153,7 @@ void FixedSizeScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_co
 	auto source_data = data + start * sizeof(T);
 
 	result.SetVectorType(VectorType::FLAT_VECTOR);
-	if (std::is_same<T, list_entry_t>()) {
-		// list columns are modified in-place during the scans to correct the offsets
-		// so we can't do a zero-copy there
-		memcpy(FlatVector::GetData(result), source_data, scan_count * sizeof(T));
-	} else {
-		FlatVector::SetData(result, source_data);
-	}
+	FlatVector::SetData(result, source_data);
 }
 
 //===--------------------------------------------------------------------===//
@@ -186,48 +180,52 @@ static unique_ptr<CompressionAppendState> FixedSizeInitAppend(ColumnSegment &seg
 	return make_unique<CompressionAppendState>(std::move(handle));
 }
 
-template <class T>
-static void AppendLoop(SegmentStatistics &stats, data_ptr_t target, idx_t target_offset, UnifiedVectorFormat &adata,
-                       idx_t offset, idx_t count) {
-	auto sdata = (T *)adata.data;
-	auto tdata = (T *)target;
-	if (!adata.validity.AllValid()) {
-		for (idx_t i = 0; i < count; i++) {
-			auto source_idx = adata.sel->get_index(offset + i);
-			auto target_idx = target_offset + i;
-			bool is_null = !adata.validity.RowIsValid(source_idx);
-			if (!is_null) {
+struct StandardFixedSizeAppend {
+	template <class T>
+	static void Append(SegmentStatistics &stats, data_ptr_t target, idx_t target_offset, UnifiedVectorFormat &adata,
+	                   idx_t offset, idx_t count) {
+		auto sdata = (T *)adata.data;
+		auto tdata = (T *)target;
+		if (!adata.validity.AllValid()) {
+			for (idx_t i = 0; i < count; i++) {
+				auto source_idx = adata.sel->get_index(offset + i);
+				auto target_idx = target_offset + i;
+				bool is_null = !adata.validity.RowIsValid(source_idx);
+				if (!is_null) {
+					NumericStatistics::Update<T>(stats, sdata[source_idx]);
+					tdata[target_idx] = sdata[source_idx];
+				} else {
+					// we insert a NullValue<T> in the null gap for debuggability
+					// this value should never be used or read anywhere
+					tdata[target_idx] = NullValue<T>();
+				}
+			}
+		} else {
+			for (idx_t i = 0; i < count; i++) {
+				auto source_idx = adata.sel->get_index(offset + i);
+				auto target_idx = target_offset + i;
 				NumericStatistics::Update<T>(stats, sdata[source_idx]);
 				tdata[target_idx] = sdata[source_idx];
-			} else {
-				// we insert a NullValue<T> in the null gap for debuggability
-				// this value should never be used or read anywhere
-				tdata[target_idx] = NullValue<T>();
 			}
 		}
-	} else {
+	}
+};
+
+struct ListFixedSizeAppend {
+	template <class T>
+	static void Append(SegmentStatistics &stats, data_ptr_t target, idx_t target_offset, UnifiedVectorFormat &adata,
+	                   idx_t offset, idx_t count) {
+		auto sdata = (uint64_t *)adata.data;
+		auto tdata = (uint64_t *)target;
 		for (idx_t i = 0; i < count; i++) {
 			auto source_idx = adata.sel->get_index(offset + i);
 			auto target_idx = target_offset + i;
-			NumericStatistics::Update<T>(stats, sdata[source_idx]);
 			tdata[target_idx] = sdata[source_idx];
 		}
 	}
-}
-
-template <>
-void AppendLoop<list_entry_t>(SegmentStatistics &stats, data_ptr_t target, idx_t target_offset,
-                              UnifiedVectorFormat &adata, idx_t offset, idx_t count) {
-	auto sdata = (list_entry_t *)adata.data;
-	auto tdata = (list_entry_t *)target;
-	for (idx_t i = 0; i < count; i++) {
-		auto source_idx = adata.sel->get_index(offset + i);
-		auto target_idx = target_offset + i;
-		tdata[target_idx] = sdata[source_idx];
-	}
-}
+};
 
-template <class T>
+template <class T, class OP>
 idx_t FixedSizeAppend(CompressionAppendState &append_state, ColumnSegment &segment, SegmentStatistics &stats,
                       UnifiedVectorFormat &data, idx_t offset, idx_t count) {
 	D_ASSERT(segment.GetBlockOffset() == 0);
@@ -236,7 +234,7 @@ idx_t FixedSizeAppend(CompressionAppendState &append_state, ColumnSegment &segme
 	idx_t max_tuple_count = segment.SegmentSize() / sizeof(T);
 	idx_t copy_count = MinValue<idx_t>(count, max_tuple_count - segment.count);
 
-	AppendLoop<T>(stats, target_ptr, segment.count, data, offset, copy_count);
+	OP::template Append<T>(stats, target_ptr, segment.count, data, offset, copy_count);
 	segment.count += copy_count;
 	return copy_count;
 }
@@ -249,14 +247,14 @@ idx_t FixedSizeFinalizeAppend(ColumnSegment &segment, SegmentStatistics &stats)
 //===--------------------------------------------------------------------===//
 // Get Function
 //===--------------------------------------------------------------------===//
-template <class T>
+template <class T, class APPENDER = StandardFixedSizeAppend>
 CompressionFunction FixedSizeGetFunction(PhysicalType data_type) {
 	return CompressionFunction(CompressionType::COMPRESSION_UNCOMPRESSED, data_type, FixedSizeInitAnalyze,
 	                           FixedSizeAnalyze, FixedSizeFinalAnalyze<T>, UncompressedFunctions::InitCompression,
 	                           UncompressedFunctions::Compress, UncompressedFunctions::FinalizeCompress,
 	                           FixedSizeInitScan, FixedSizeScan<T>, FixedSizeScanPartial<T>, FixedSizeFetchRow<T>,
-	                           UncompressedFunctions::EmptySkip, nullptr, FixedSizeInitAppend, FixedSizeAppend<T>,
-	                           FixedSizeFinalizeAppend<T>, nullptr);
+	                           UncompressedFunctions::EmptySkip, nullptr, FixedSizeInitAppend,
+	                           FixedSizeAppend<T, APPENDER>, FixedSizeFinalizeAppend<T>, nullptr);
 }
 
 CompressionFunction FixedSizeUncompressed::GetFunction(PhysicalType data_type) {
@@ -287,7 +285,7 @@ CompressionFunction FixedSizeUncompressed::GetFunction(PhysicalType data_type) {
 	case PhysicalType::INTERVAL:
 		return FixedSizeGetFunction<interval_t>(data_type);
 	case PhysicalType::LIST:
-		return FixedSizeGetFunction<list_entry_t>(data_type);
+		return FixedSizeGetFunction<uint64_t, ListFixedSizeAppend>(data_type);
 	default:
 		throw InternalException("Unsupported type for FixedSizeUncompressed::GetFunction");
 	}

package/src/duckdb/src/storage/compression/rle.cpp

@@ -118,12 +118,12 @@ struct RLEConstants {
 	static constexpr const idx_t RLE_HEADER_SIZE = sizeof(uint64_t);
 };
 
-template <class T>
+template <class T, bool WRITE_STATISTICS>
 struct RLECompressState : public CompressionState {
 	struct RLEWriter {
 		template <class VALUE_TYPE>
 		static void Operation(VALUE_TYPE value, rle_count_t count, void *dataptr, bool is_null) {
-			auto state = (RLECompressState<T> *)dataptr;
+			auto state = (RLECompressState<T, WRITE_STATISTICS> *)dataptr;
 			state->WriteValue(value, count, is_null);
 		}
 	};
@@ -160,7 +160,7 @@ struct RLECompressState : public CompressionState {
 		auto data = (T *)vdata.data;
 		for (idx_t i = 0; i < count; i++) {
 			auto idx = vdata.sel->get_index(i);
-			state.template Update<RLECompressState<T>::RLEWriter>(data, vdata.validity, idx);
+			state.template Update<RLECompressState<T, WRITE_STATISTICS>::RLEWriter>(data, vdata.validity, idx);
 		}
 	}
 
@@ -174,7 +174,7 @@ struct RLECompressState : public CompressionState {
 		entry_count++;
 
 		// update meta data
-		if (!is_null) {
+		if (WRITE_STATISTICS && !is_null) {
 			NumericStatistics::Update<T>(current_segment->stats, value);
 		}
 		current_segment->count += count;
@@ -206,7 +206,7 @@ struct RLECompressState : public CompressionState {
 	}
 
 	void Finalize() {
-		state.template Flush<RLECompressState<T>::RLEWriter>();
+		state.template Flush<RLECompressState<T, WRITE_STATISTICS>::RLEWriter>();
 
 		FlushSegment();
 		current_segment.reset();
@@ -222,23 +222,23 @@ struct RLECompressState : public CompressionState {
 	idx_t max_rle_count;
 };
 
-template <class T>
+template <class T, bool WRITE_STATISTICS>
 unique_ptr<CompressionState> RLEInitCompression(ColumnDataCheckpointer &checkpointer, unique_ptr<AnalyzeState> state) {
-	return make_unique<RLECompressState<T>>(checkpointer);
+	return make_unique<RLECompressState<T, WRITE_STATISTICS>>(checkpointer);
 }
 
-template <class T>
+template <class T, bool WRITE_STATISTICS>
 void RLECompress(CompressionState &state_p, Vector &scan_vector, idx_t count) {
-	auto &state = (RLECompressState<T> &)state_p;
+	auto &state = (RLECompressState<T, WRITE_STATISTICS> &)state_p;
 	UnifiedVectorFormat vdata;
 	scan_vector.ToUnifiedFormat(count, vdata);
 
 	state.Append(vdata, count);
 }
 
-template <class T>
+template <class T, bool WRITE_STATISTICS>
 void RLEFinalizeCompress(CompressionState &state_p) {
-	auto &state = (RLECompressState<T> &)state_p;
+	auto &state = (RLECompressState<T, WRITE_STATISTICS> &)state_p;
 	state.Finalize();
 }
 
@@ -341,10 +341,11 @@ void RLEFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id,
 //===--------------------------------------------------------------------===//
 // Get Function
 //===--------------------------------------------------------------------===//
-template <class T>
+template <class T, bool WRITE_STATISTICS = true>
 CompressionFunction GetRLEFunction(PhysicalType data_type) {
 	return CompressionFunction(CompressionType::COMPRESSION_RLE, data_type, RLEInitAnalyze<T>, RLEAnalyze<T>,
-	                           RLEFinalAnalyze<T>, RLEInitCompression<T>, RLECompress<T>, RLEFinalizeCompress<T>,
+	                           RLEFinalAnalyze<T>, RLEInitCompression<T, WRITE_STATISTICS>,
+	                           RLECompress<T, WRITE_STATISTICS>, RLEFinalizeCompress<T, WRITE_STATISTICS>,
 	                           RLEInitScan<T>, RLEScan<T>, RLEScanPartial<T>, RLEFetchRow<T>, RLESkip<T>);
 }
 
@@ -373,6 +374,8 @@ CompressionFunction RLEFun::GetFunction(PhysicalType type) {
 		return GetRLEFunction<float>(type);
 	case PhysicalType::DOUBLE:
 		return GetRLEFunction<double>(type);
+	case PhysicalType::LIST:
+		return GetRLEFunction<uint64_t, false>(type);
 	default:
 		throw InternalException("Unsupported type for RLE");
 	}
@@ -392,6 +395,7 @@ bool RLEFun::TypeIsSupported(PhysicalType type) {
 	case PhysicalType::UINT64:
 	case PhysicalType::FLOAT:
 	case PhysicalType::DOUBLE:
+	case PhysicalType::LIST:
 		return true;
 	default:
 		return false;

package/src/duckdb/src/storage/statistics/base_statistics.cpp

@@ -12,7 +12,7 @@
 namespace duckdb {
 
 BaseStatistics::BaseStatistics(LogicalType type, StatisticsType stats_type)
-    : type(std::move(type)), stats_type(stats_type) {
+    : type(std::move(type)), distinct_count(0), stats_type(stats_type) {
 }
 
 BaseStatistics::~BaseStatistics() {
@@ -72,9 +72,9 @@ void BaseStatistics::Merge(const BaseStatistics &other) {
 idx_t BaseStatistics::GetDistinctCount() {
 	if (distinct_stats) {
 		auto &d_stats = (DistinctStatistics &)*distinct_stats;
-		return d_stats.GetCount();
+		distinct_count = d_stats.GetCount();
 	}
-	return 0;
+	return distinct_count;
 }
 
 unique_ptr<BaseStatistics> BaseStatistics::CreateEmpty(LogicalType type, StatisticsType stats_type) {