duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/storage/table/column_checkpoint_state.cpp

@@ -113,7 +113,9 @@ void PartialBlockForCheckpoint::Clear() {
 }
 
 void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_t segment_size) {
-	D_ASSERT(segment_size <= Storage::BLOCK_SIZE);
+	auto block_size = partial_block_manager.GetBlockManager().GetBlockSize();
+	D_ASSERT(segment_size <= block_size);
+
 	auto tuple_count = segment->count.load();
 	if (tuple_count == 0) { // LCOV_EXCL_START
 		return;
@@ -151,11 +153,11 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 			pstate.AddSegmentToTail(column_data, *segment, offset_in_block);
 		} else {
 			// Create a new block for future reuse.
-			if (segment->SegmentSize() != Storage::BLOCK_SIZE) {
+			if (segment->SegmentSize() != block_size) {
 				// the segment is smaller than the block size
 				// allocate a new block and copy the data over
-				D_ASSERT(segment->SegmentSize() < Storage::BLOCK_SIZE);
-				segment->Resize(Storage::BLOCK_SIZE);
+				D_ASSERT(segment->SegmentSize() < block_size);
+				segment->Resize(block_size);
 			}
 			D_ASSERT(offset_in_block == 0);
 			allocation.partial_block = make_uniq<PartialBlockForCheckpoint>(column_data, *segment, allocation.state,
@@ -192,8 +194,10 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 	data_pointers.push_back(std::move(data_pointer));
 }
 
-void ColumnCheckpointState::WriteDataPointers(RowGroupWriter &writer, Serializer &serializer) {
-	writer.WriteColumnDataPointers(*this, serializer);
+PersistentColumnData ColumnCheckpointState::ToPersistentData() {
+	PersistentColumnData data(column_data.type.InternalType());
+	data.pointers = std::move(data_pointers);
+	return data;
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/table/column_data.cpp

@@ -17,6 +17,7 @@
 #include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/common/serializer/read_stream.hpp"
 #include "duckdb/common/serializer/binary_deserializer.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
 
 namespace duckdb {
 
@@ -107,6 +108,31 @@ ScanVectorType ColumnData::GetVectorScanType(ColumnScanState &state, idx_t scan_
 	return ScanVectorType::SCAN_ENTIRE_VECTOR;
 }
 
+void ColumnData::InitializePrefetch(PrefetchState &prefetch_state, ColumnScanState &scan_state, idx_t remaining) {
+	auto current_segment = scan_state.current;
+	if (!current_segment) {
+		return;
+	}
+	if (!scan_state.initialized) {
+		// need to prefetch for the current segment if we have not yet initialized the scan for this segment
+		scan_state.current->InitializePrefetch(prefetch_state, scan_state);
+	}
+	idx_t row_index = scan_state.row_index;
+	while (remaining > 0) {
+		idx_t scan_count = MinValue<idx_t>(remaining, current_segment->start + current_segment->count - row_index);
+		remaining -= scan_count;
+		row_index += scan_count;
+		if (remaining > 0) {
+			auto next = data.GetNextSegment(current_segment);
+			if (!next) {
+				break;
+			}
+			next->InitializePrefetch(prefetch_state, scan_state);
+			current_segment = next;
+		}
+	}
+}
+
 idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remaining, ScanVectorType scan_type) {
 	if (scan_type == ScanVectorType::SCAN_FLAT_VECTOR && result.GetVectorType() != VectorType::FLAT_VECTOR) {
 		throw InternalException("ScanVector called with SCAN_FLAT_VECTOR but result is not a flat vector");
@@ -204,8 +230,12 @@ void ColumnData::UpdateInternal(TransactionData transaction, idx_t column_index,
 template <bool SCAN_COMMITTED, bool ALLOW_UPDATES>
 idx_t ColumnData::ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
                              idx_t target_scan) {
-	auto scan_count = ScanVector(state, result, target_scan, GetVectorScanType(state, target_scan));
-	FetchUpdates(transaction, vector_index, result, scan_count, ALLOW_UPDATES, SCAN_COMMITTED);
+	auto scan_type = GetVectorScanType(state, target_scan);
+	auto scan_count = ScanVector(state, result, target_scan, scan_type);
+	if (scan_type != ScanVectorType::SCAN_ENTIRE_VECTOR) {
+		// if we are scanning an entire vector we cannot have updates
+		FetchUpdates(transaction, vector_index, result, scan_count, ALLOW_UPDATES, SCAN_COMMITTED);
+	}
 	return scan_count;
 }
 
@@ -307,17 +337,42 @@ void ColumnData::Append(ColumnAppendState &state, Vector &vector, idx_t append_c
 	Append(stats->statistics, state, vector, append_count);
 }
 
-bool ColumnData::CheckZonemap(TableFilter &filter) {
+FilterPropagateResult ColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
+	if (state.segment_checked) {
+		return FilterPropagateResult::NO_PRUNING_POSSIBLE;
+	}
+	if (!state.current) {
+		return FilterPropagateResult::NO_PRUNING_POSSIBLE;
+	}
+	state.segment_checked = true;
+	FilterPropagateResult prune_result;
+	{
+		lock_guard<mutex> l(stats_lock);
+		prune_result = filter.CheckStatistics(state.current->stats.statistics);
+		if (prune_result == FilterPropagateResult::NO_PRUNING_POSSIBLE) {
+			return FilterPropagateResult::NO_PRUNING_POSSIBLE;
+		}
+	}
+	lock_guard<mutex> l(update_lock);
+	if (!updates) {
+		// no updates - return original result
+		return prune_result;
+	}
+	auto update_stats = updates->GetStatistics();
+	// combine the update and original prune result
+	FilterPropagateResult update_result = filter.CheckStatistics(*update_stats);
+	if (prune_result == update_result) {
+		return prune_result;
+	}
+	return FilterPropagateResult::NO_PRUNING_POSSIBLE;
+}
+
+FilterPropagateResult ColumnData::CheckZonemap(TableFilter &filter) {
 	if (!stats) {
 		throw InternalException("ColumnData::CheckZonemap called on a column without stats");
 	}
 	lock_guard<mutex> l(stats_lock);
-	auto propagate_result = filter.CheckStatistics(stats->statistics);
-	if (propagate_result == FilterPropagateResult::FILTER_ALWAYS_FALSE ||
-	    propagate_result == FilterPropagateResult::FILTER_FALSE_OR_NULL) {
-		return false;
-	}
-	return true;
+	return filter.CheckStatistics(stats->statistics);
 }
 
 unique_ptr<BaseStatistics> ColumnData::GetStatistics() {
@@ -394,9 +449,9 @@ void ColumnData::RevertAppend(row_t start_row) {
 	auto l = data.Lock();
 	// check if this row is in the segment tree at all
 	auto last_segment = data.GetLastSegment(l);
-	if (idx_t(start_row) >= last_segment->start + last_segment->count) {
+	if (NumericCast<idx_t>(start_row) >= last_segment->start + last_segment->count) {
 		// the start row is equal to the final portion of the column data: nothing was ever appended here
-		D_ASSERT(idx_t(start_row) == last_segment->start + last_segment->count);
+		D_ASSERT(NumericCast<idx_t>(start_row) == last_segment->start + last_segment->count);
 		return;
 	}
 	// find the segment index that the current row belongs to
@@ -415,7 +470,7 @@ void ColumnData::RevertAppend(row_t start_row) {
 
 idx_t ColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &result) {
 	D_ASSERT(row_id >= 0);
-	D_ASSERT(idx_t(row_id) >= start);
+	D_ASSERT(NumericCast<idx_t>(row_id) >= start);
 	// perform the fetch within the segment
 	state.row_index =
 	    start + ((UnsafeNumericCast<idx_t>(row_id) - start) / STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE);
@@ -454,19 +509,22 @@ void ColumnData::UpdateColumn(TransactionData transaction, const vector<column_t
 
 void ColumnData::AppendTransientSegment(SegmentLock &l, idx_t start_row) {
 
-	idx_t vector_segment_size = Storage::BLOCK_SIZE;
-	if (start_row == idx_t(MAX_ROW_ID)) {
+	const auto block_size = block_manager.GetBlockSize();
+	const auto type_size = GetTypeIdSize(type.InternalType());
+	auto vector_segment_size = block_size;
+
+	if (start_row == NumericCast<idx_t>(MAX_ROW_ID)) {
 #if STANDARD_VECTOR_SIZE < 1024
-		vector_segment_size = 1024 * GetTypeIdSize(type.InternalType());
+		vector_segment_size = 1024 * type_size;
 #else
-		vector_segment_size = STANDARD_VECTOR_SIZE * GetTypeIdSize(type.InternalType());
+		vector_segment_size = STANDARD_VECTOR_SIZE * type_size;
 #endif
 	}
 
-	// the segment size is bound by the block size, but can be smaller
-	idx_t segment_size = Storage::BLOCK_SIZE < vector_segment_size ? Storage::BLOCK_SIZE : vector_segment_size;
+	// The segment size is bound by the block size, but can be smaller.
+	idx_t segment_size = block_size < vector_segment_size ? block_size : vector_segment_size;
 	allocation_size += segment_size;
-	auto new_segment = ColumnSegment::CreateTransientSegment(GetDatabase(), type, start_row, segment_size);
+	auto new_segment = ColumnSegment::CreateTransientSegment(GetDatabase(), type, start_row, segment_size, block_size);
 	data.AppendSegment(l, std::move(new_segment));
 }
 
@@ -522,20 +580,15 @@ unique_ptr<ColumnCheckpointState> ColumnData::Checkpoint(RowGroup &row_group, Co
 	return checkpoint_state;
 }
 
-void ColumnData::DeserializeColumn(Deserializer &deserializer, BaseStatistics &target_stats) {
-	// load the data pointers for the column
-	deserializer.Set<DatabaseInstance &>(info.GetDB().GetDatabase());
-	deserializer.Set<LogicalType &>(type);
-
-	vector<DataPointer> data_pointers;
-	deserializer.ReadProperty(100, "data_pointers", data_pointers);
-
-	deserializer.Unset<DatabaseInstance>();
-	deserializer.Unset<LogicalType>();
+void ColumnData::InitializeColumn(PersistentColumnData &column_data) {
+	InitializeColumn(column_data, stats->statistics);
+}
 
+void ColumnData::InitializeColumn(PersistentColumnData &column_data, BaseStatistics &target_stats) {
+	D_ASSERT(type.InternalType() == column_data.physical_type);
 	// construct the segments based on the data pointers
 	this->count = 0;
-	for (auto &data_pointer : data_pointers) {
+	for (auto &data_pointer : column_data.pointers) {
 		// Update the count and statistics
 		this->count += data_pointer.tuple_count;
 
@@ -554,13 +607,181 @@ void ColumnData::DeserializeColumn(Deserializer &deserializer, BaseStatistics &t
 	}
 }
 
+bool ColumnData::IsPersistent() {
+	for (auto &segment : data.Segments()) {
+		if (segment.segment_type != ColumnSegmentType::PERSISTENT) {
+			return false;
+		}
+	}
+	return true;
+}
+
+vector<DataPointer> ColumnData::GetDataPointers() {
+	vector<DataPointer> pointers;
+	for (auto &segment : data.Segments()) {
+		pointers.push_back(segment.GetDataPointer());
+	}
+	return pointers;
+}
+
+PersistentColumnData::PersistentColumnData(PhysicalType physical_type_p) : physical_type(physical_type_p) {
+}
+
+PersistentColumnData::PersistentColumnData(PhysicalType physical_type, vector<DataPointer> pointers_p)
+    : physical_type(physical_type), pointers(std::move(pointers_p)) {
+	D_ASSERT(!pointers.empty());
+}
+
+PersistentColumnData::~PersistentColumnData() {
+}
+
+void PersistentColumnData::Serialize(Serializer &serializer) const {
+	if (has_updates) {
+		throw InternalException("Column data with updates cannot be serialized");
+	}
+	serializer.WritePropertyWithDefault(100, "data_pointers", pointers);
+	if (child_columns.empty()) {
+		// validity column
+		D_ASSERT(physical_type == PhysicalType::BIT);
+		return;
+	}
+	serializer.WriteProperty(101, "validity", child_columns[0]);
+	if (physical_type == PhysicalType::ARRAY || physical_type == PhysicalType::LIST) {
+		D_ASSERT(child_columns.size() == 2);
+		serializer.WriteProperty(102, "child_column", child_columns[1]);
+	} else if (physical_type == PhysicalType::STRUCT) {
+		serializer.WriteList(102, "sub_columns", child_columns.size() - 1,
+		                     [&](Serializer::List &list, idx_t i) { list.WriteElement(child_columns[i + 1]); });
+	}
+}
+
+void PersistentColumnData::DeserializeField(Deserializer &deserializer, field_id_t field_idx, const char *field_name,
+                                            const LogicalType &type) {
+	deserializer.Set<const LogicalType &>(type);
+	child_columns.push_back(deserializer.ReadProperty<PersistentColumnData>(field_idx, field_name));
+	deserializer.Unset<LogicalType>();
+}
+
+PersistentColumnData PersistentColumnData::Deserialize(Deserializer &deserializer) {
+	auto &type = deserializer.Get<const LogicalType &>();
+	auto physical_type = type.InternalType();
+	PersistentColumnData result(physical_type);
+	deserializer.ReadPropertyWithDefault(100, "data_pointers", static_cast<vector<DataPointer> &>(result.pointers));
+	if (result.physical_type == PhysicalType::BIT) {
+		// validity: return
+		return result;
+	}
+	result.DeserializeField(deserializer, 101, "validity", LogicalTypeId::VALIDITY);
+	switch (physical_type) {
+	case PhysicalType::ARRAY:
+		result.DeserializeField(deserializer, 102, "child_column", ArrayType::GetChildType(type));
+		break;
+	case PhysicalType::LIST:
+		result.DeserializeField(deserializer, 102, "child_column", ListType::GetChildType(type));
+		break;
+	case PhysicalType::STRUCT: {
+		auto &child_types = StructType::GetChildTypes(type);
+		deserializer.ReadList(102, "sub_columns", [&](Deserializer::List &list, idx_t i) {
+			deserializer.Set<const LogicalType &>(child_types[i].second);
+			result.child_columns.push_back(list.ReadElement<PersistentColumnData>());
+			deserializer.Unset<LogicalType>();
+		});
+		break;
+	}
+	default:
+		break;
+	}
+	return result;
+}
+
+bool PersistentColumnData::HasUpdates() const {
+	if (has_updates) {
+		return true;
+	}
+	for (auto &child_col : child_columns) {
+		if (child_col.HasUpdates()) {
+			return true;
+		}
+	}
+	return false;
+}
+
+PersistentRowGroupData::PersistentRowGroupData(vector<LogicalType> types_p) : types(std::move(types_p)) {
+}
+
+void PersistentRowGroupData::Serialize(Serializer &serializer) const {
+	serializer.WriteProperty(100, "types", types);
+	serializer.WriteProperty(101, "columns", column_data);
+	serializer.WriteProperty(102, "start", start);
+	serializer.WriteProperty(103, "count", count);
+}
+
+PersistentRowGroupData PersistentRowGroupData::Deserialize(Deserializer &deserializer) {
+	PersistentRowGroupData data;
+	deserializer.ReadProperty(100, "types", data.types);
+	deserializer.ReadList(101, "columns", [&](Deserializer::List &list, idx_t i) {
+		deserializer.Set<const LogicalType &>(data.types[i]);
+		data.column_data.push_back(list.ReadElement<PersistentColumnData>());
+		deserializer.Unset<LogicalType>();
+	});
+	deserializer.ReadProperty(102, "start", data.start);
+	deserializer.ReadProperty(103, "count", data.count);
+	return data;
+}
+
+bool PersistentRowGroupData::HasUpdates() const {
+	for (auto &col : column_data) {
+		if (col.HasUpdates()) {
+			return true;
+		}
+	}
+	return false;
+}
+
+void PersistentCollectionData::Serialize(Serializer &serializer) const {
+	serializer.WriteProperty(100, "row_groups", row_group_data);
+}
+
+PersistentCollectionData PersistentCollectionData::Deserialize(Deserializer &deserializer) {
+	PersistentCollectionData data;
+	deserializer.ReadProperty(100, "row_groups", data.row_group_data);
+	return data;
+}
+
+bool PersistentCollectionData::HasUpdates() const {
+	for (auto &row_group : row_group_data) {
+		if (row_group.HasUpdates()) {
+			return true;
+		}
+	}
+	return false;
+}
+
+PersistentColumnData ColumnData::Serialize() {
+	PersistentColumnData result(type.InternalType(), GetDataPointers());
+	result.has_updates = HasUpdates();
+	return result;
+}
+
 shared_ptr<ColumnData> ColumnData::Deserialize(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
                                                idx_t start_row, ReadStream &source, const LogicalType &type) {
 	auto entry = ColumnData::CreateColumn(block_manager, info, column_index, start_row, type, nullptr);
+
+	// deserialize the persistent column data
 	BinaryDeserializer deserializer(source);
 	deserializer.Begin();
-	entry->DeserializeColumn(deserializer, entry->stats->statistics);
+	deserializer.Set<DatabaseInstance &>(info.GetDB().GetDatabase());
+	CompressionInfo compression_info(block_manager.GetBlockSize());
+	deserializer.Set<const CompressionInfo &>(compression_info);
+	deserializer.Set<const LogicalType &>(type);
+	auto persistent_column_data = PersistentColumnData::Deserialize(deserializer);
+	deserializer.Unset<LogicalType>();
+	deserializer.Unset<const CompressionInfo>();
+	deserializer.Unset<DatabaseInstance>();
 	deserializer.End();
+
+	// initialize the column
+	entry->InitializeColumn(persistent_column_data, entry->stats->statistics);
 	return entry;
 }
 

package/src/duckdb/src/storage/table/column_data_checkpointer.cpp

@@ -13,6 +13,7 @@ ColumnDataCheckpointer::ColumnDataCheckpointer(ColumnData &col_data_p, RowGroup
       is_validity(GetType().id() == LogicalTypeId::VALIDITY),
       intermediate(is_validity ? LogicalType::BOOLEAN : GetType(), true, is_validity),
       checkpoint_info(checkpoint_info_p) {
+
 	auto &config = DBConfig::GetConfig(GetDatabase());
 	auto functions = config.GetCompressionFunctions(GetType().InternalType());
 	for (auto &func : functions) {
@@ -227,18 +228,7 @@ void ColumnDataCheckpointer::WritePersistentSegments() {
 	// we only need to write the metadata
 	for (idx_t segment_idx = 0; segment_idx < nodes.size(); segment_idx++) {
 		auto segment = nodes[segment_idx].node.get();
-		D_ASSERT(segment->segment_type == ColumnSegmentType::PERSISTENT);
-
-		// set up the data pointer directly using the data from the persistent segment
-		DataPointer pointer(segment->stats.statistics.Copy());
-		pointer.block_pointer.block_id = segment->GetBlockId();
-		pointer.block_pointer.offset = NumericCast<uint32_t>(segment->GetBlockOffset());
-		pointer.row_start = segment->start;
-		pointer.tuple_count = segment->count;
-		pointer.compression_type = segment->function.get().type;
-		if (segment->function.get().serialize_state) {
-			pointer.segment_state = segment->function.get().serialize_state(*segment);
-		}
+		auto pointer = segment->GetDataPointer();
 
 		// merge the persistent stats into the global column stats
 		state.global_stats->Merge(segment->stats.statistics);

package/src/duckdb/src/storage/table/column_segment.cpp

@@ -1,16 +1,17 @@
 #include "duckdb/storage/table/column_segment.hpp"
+
 #include "duckdb/common/limits.hpp"
-#include "duckdb/storage/table/update_segment.hpp"
 #include "duckdb/common/types/null_value.hpp"
 #include "duckdb/common/types/vector.hpp"
-#include "duckdb/storage/table/append_state.hpp"
-#include "duckdb/storage/storage_manager.hpp"
+#include "duckdb/main/config.hpp"
 #include "duckdb/planner/filter/conjunction_filter.hpp"
 #include "duckdb/planner/filter/constant_filter.hpp"
 #include "duckdb/planner/filter/struct_filter.hpp"
-#include "duckdb/main/config.hpp"
-#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/data_pointer.hpp"
+#include "duckdb/storage/storage_manager.hpp"
+#include "duckdb/storage/table/append_state.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
+#include "duckdb/storage/table/update_segment.hpp"
 
 #include <cstring>
 
@@ -32,34 +33,29 @@ unique_ptr<ColumnSegment> ColumnSegment::CreatePersistentSegment(DatabaseInstanc
 	shared_ptr<BlockHandle> block;
 
 	if (block_id == INVALID_BLOCK) {
-		// constant segment, no need to allocate an actual block
 		function = config.GetCompressionFunction(CompressionType::COMPRESSION_CONSTANT, type.InternalType());
 	} else {
 		function = config.GetCompressionFunction(compression_type, type.InternalType());
 		block = block_manager.RegisterBlock(block_id);
 	}
 
-	auto segment_size = Storage::BLOCK_SIZE;
+	auto segment_size = block_manager.GetBlockSize();
 	return make_uniq<ColumnSegment>(db, std::move(block), type, ColumnSegmentType::PERSISTENT, start, count, *function,
 	                                std::move(statistics), block_id, offset, segment_size, std::move(segment_state));
 }
 
 unique_ptr<ColumnSegment> ColumnSegment::CreateTransientSegment(DatabaseInstance &db, const LogicalType &type,
-                                                                idx_t start, idx_t segment_size) {
+                                                                const idx_t start, const idx_t segment_size,
+                                                                const idx_t block_size) {
 
-	D_ASSERT(segment_size <= Storage::BLOCK_SIZE);
+	// Allocate a buffer for the uncompressed segment.
+	auto &buffer_manager = BufferManager::GetBufferManager(db);
+	auto block = buffer_manager.RegisterTransientMemory(segment_size, block_size);
 
+	// Get the segment compression function.
 	auto &config = DBConfig::GetConfig(db);
 	auto function = config.GetCompressionFunction(CompressionType::COMPRESSION_UNCOMPRESSED, type.InternalType());
-	auto &buffer_manager = BufferManager::GetBufferManager(db);
-	shared_ptr<BlockHandle> block;
 
-	// transient: allocate a buffer for the uncompressed segment
-	if (segment_size < Storage::BLOCK_SIZE) {
-		block = buffer_manager.RegisterSmallMemory(segment_size);
-	} else {
-		buffer_manager.Allocate(MemoryTag::IN_MEMORY_TABLE, segment_size, false, &block);
-	}
 	return make_uniq<ColumnSegment>(db, std::move(block), type, ColumnSegmentType::TRANSIENT, start, 0U, *function,
 	                                BaseStatistics::CreateEmpty(type), INVALID_BLOCK, 0U, segment_size);
 }
@@ -67,30 +63,33 @@ unique_ptr<ColumnSegment> ColumnSegment::CreateTransientSegment(DatabaseInstance
 //===--------------------------------------------------------------------===//
 // Construct/Destruct
 //===--------------------------------------------------------------------===//
-ColumnSegment::ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, LogicalType type_p,
-                             ColumnSegmentType segment_type, idx_t start, idx_t count, CompressionFunction &function_p,
-                             BaseStatistics statistics, block_id_t block_id_p, idx_t offset_p, idx_t segment_size_p,
-                             unique_ptr<ColumnSegmentState> segment_state)
+ColumnSegment::ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block_p, const LogicalType &type,
+                             const ColumnSegmentType segment_type, const idx_t start, const idx_t count,
+                             CompressionFunction &function_p, BaseStatistics statistics, const block_id_t block_id_p,
+                             const idx_t offset, const idx_t segment_size_p,
+                             const unique_ptr<ColumnSegmentState> segment_state_p)
 
-    : SegmentBase<ColumnSegment>(start, count), db(db), type(std::move(type_p)),
-      type_size(GetTypeIdSize(type.InternalType())), segment_type(segment_type), function(function_p),
-      stats(std::move(statistics)), block(std::move(block)), block_id(block_id_p), offset(offset_p),
-      segment_size(segment_size_p) {
+    : SegmentBase<ColumnSegment>(start, count), db(db), type(type), type_size(GetTypeIdSize(type.InternalType())),
+      segment_type(segment_type), function(function_p), stats(std::move(statistics)), block(std::move(block_p)),
+      block_id(block_id_p), offset(offset), segment_size(segment_size_p) {
 
 	if (function.get().init_segment) {
-		this->segment_state = function.get().init_segment(*this, block_id, segment_state.get());
+		segment_state = function.get().init_segment(*this, block_id, segment_state_p.get());
 	}
-	D_ASSERT(segment_size <= Storage::BLOCK_SIZE);
+
+	// For constant segments (CompressionType::COMPRESSION_CONSTANT) the block is a nullptr.
+	D_ASSERT(!block || segment_size <= GetBlockManager().GetBlockSize());
 }
 
-ColumnSegment::ColumnSegment(ColumnSegment &other, idx_t start)
+ColumnSegment::ColumnSegment(ColumnSegment &other, const idx_t start)
 
     : SegmentBase<ColumnSegment>(start, other.count.load()), db(other.db), type(std::move(other.type)),
       type_size(other.type_size), segment_type(other.segment_type), function(other.function),
       stats(std::move(other.stats)), block(std::move(other.block)), block_id(other.block_id), offset(other.offset),
       segment_size(other.segment_size), segment_state(std::move(other.segment_state)) {
 
-	D_ASSERT(segment_size <= Storage::BLOCK_SIZE);
+	// For constant segments (CompressionType::COMPRESSION_CONSTANT) the block is a nullptr.
+	D_ASSERT(!block || segment_size <= GetBlockManager().GetBlockSize());
 }
 
 ColumnSegment::~ColumnSegment() {
@@ -99,6 +98,18 @@ ColumnSegment::~ColumnSegment() {
 //===--------------------------------------------------------------------===//
 // Scan
 //===--------------------------------------------------------------------===//
+void ColumnSegment::InitializePrefetch(PrefetchState &prefetch_state, ColumnScanState &) {
+	if (!block || block->BlockId() >= MAXIMUM_BLOCK) {
+		// not an on-disk block
+		return;
+	}
+	if (function.get().init_prefetch) {
+		function.get().init_prefetch(*this, prefetch_state);
+	} else {
+		prefetch_state.AddBlock(block);
+	}
+}
+
 void ColumnSegment::InitializeScan(ColumnScanState &state) {
 	state.scan_state = function.get().init_scan(*this);
 }
@@ -144,9 +155,9 @@ idx_t ColumnSegment::SegmentSize() const {
 }
 
 void ColumnSegment::Resize(idx_t new_size) {
-	D_ASSERT(new_size > this->segment_size);
+	D_ASSERT(new_size > segment_size);
 	D_ASSERT(offset == 0);
-	D_ASSERT(new_size <= Storage::BLOCK_SIZE);
+	D_ASSERT(block && new_size <= GetBlockManager().GetBlockSize());
 
 	auto &buffer_manager = BufferManager::GetBufferManager(db);
 	auto old_handle = buffer_manager.Pin(block);
@@ -225,6 +236,23 @@ void ColumnSegment::MarkAsPersistent(shared_ptr<BlockHandle> block_p, uint32_t o
 	block = std::move(block_p);
 }
 
+DataPointer ColumnSegment::GetDataPointer() {
+	if (segment_type != ColumnSegmentType::PERSISTENT) {
+		throw InternalException("Attempting to call ColumnSegment::GetDataPointer on a transient segment");
+	}
+	// set up the data pointer directly using the data from the persistent segment
+	DataPointer pointer(stats.statistics.Copy());
+	pointer.block_pointer.block_id = GetBlockId();
+	pointer.block_pointer.offset = NumericCast<uint32_t>(GetBlockOffset());
+	pointer.row_start = start;
+	pointer.tuple_count = count;
+	pointer.compression_type = function.get().type;
+	if (function.get().serialize_state) {
+		pointer.segment_state = function.get().serialize_state(*this);
+	}
+	return pointer;
+}
+
 //===--------------------------------------------------------------------===//
 // Drop Segment
 //===--------------------------------------------------------------------===//
@@ -253,9 +281,10 @@ static idx_t TemplatedFilterSelection(UnifiedVectorFormat &vdata, T predicate, S
 	for (idx_t i = 0; i < approved_tuple_count; i++) {
 		auto idx = sel.get_index(i);
 		auto vector_idx = vdata.sel->get_index(idx);
-		if ((!HAS_NULL || mask.RowIsValid(vector_idx)) && OP::Operation(vec[vector_idx], predicate)) {
-			result_sel.set_index(result_count++, idx);
-		}
+		bool comparison_result =
+		    (!HAS_NULL || mask.RowIsValid(vector_idx)) && OP::Operation(vec[vector_idx], predicate);
+		result_sel.set_index(result_count, idx);
+		result_count += comparison_result;
 	}
 	return result_count;
 }