duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/src/storage/table/column_segment.cpp

@@ -7,6 +7,7 @@
 #include "duckdb/storage/storage_manager.hpp"
 #include "duckdb/planner/filter/conjunction_filter.hpp"
 #include "duckdb/planner/filter/constant_filter.hpp"
+#include "duckdb/planner/filter/struct_filter.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/data_pointer.hpp"
@@ -15,15 +16,21 @@
 
 namespace duckdb {
 
+//===--------------------------------------------------------------------===//
+// Create
+//===--------------------------------------------------------------------===//
+
 unique_ptr<ColumnSegment> ColumnSegment::CreatePersistentSegment(DatabaseInstance &db, BlockManager &block_manager,
                                                                  block_id_t block_id, idx_t offset,
                                                                  const LogicalType &type, idx_t start, idx_t count,
                                                                  CompressionType compression_type,
                                                                  BaseStatistics statistics,
                                                                  unique_ptr<ColumnSegmentState> segment_state) {
+
 	auto &config = DBConfig::GetConfig(db);
 	optional_ptr<CompressionFunction> function;
 	shared_ptr<BlockHandle> block;
+
 	if (block_id == INVALID_BLOCK) {
 		// constant segment, no need to allocate an actual block
 		function = config.GetCompressionFunction(CompressionType::COMPRESSION_CONSTANT, type.InternalType());
@@ -31,6 +38,7 @@ unique_ptr<ColumnSegment> ColumnSegment::CreatePersistentSegment(DatabaseInstanc
 		function = config.GetCompressionFunction(compression_type, type.InternalType());
 		block = block_manager.RegisterBlock(block_id);
 	}
+
 	auto segment_size = Storage::BLOCK_SIZE;
 	return make_uniq<ColumnSegment>(db, std::move(block), type, ColumnSegmentType::PERSISTENT, start, count, *function,
 	                                std::move(statistics), block_id, offset, segment_size, std::move(segment_state));
@@ -38,42 +46,51 @@ unique_ptr<ColumnSegment> ColumnSegment::CreatePersistentSegment(DatabaseInstanc
 
 unique_ptr<ColumnSegment> ColumnSegment::CreateTransientSegment(DatabaseInstance &db, const LogicalType &type,
                                                                 idx_t start, idx_t segment_size) {
+
+	D_ASSERT(segment_size <= Storage::BLOCK_SIZE);
+
 	auto &config = DBConfig::GetConfig(db);
 	auto function = config.GetCompressionFunction(CompressionType::COMPRESSION_UNCOMPRESSED, type.InternalType());
 	auto &buffer_manager = BufferManager::GetBufferManager(db);
 	shared_ptr<BlockHandle> block;
+
 	// transient: allocate a buffer for the uncompressed segment
 	if (segment_size < Storage::BLOCK_SIZE) {
 		block = buffer_manager.RegisterSmallMemory(segment_size);
 	} else {
-		buffer_manager.Allocate(segment_size, false, &block);
+		buffer_manager.Allocate(MemoryTag::IN_MEMORY_TABLE, segment_size, false, &block);
 	}
 	return make_uniq<ColumnSegment>(db, std::move(block), type, ColumnSegmentType::TRANSIENT, start, 0, *function,
 	                                BaseStatistics::CreateEmpty(type), INVALID_BLOCK, 0, segment_size);
 }
 
-unique_ptr<ColumnSegment> ColumnSegment::CreateSegment(ColumnSegment &other, idx_t start) {
-	return make_uniq<ColumnSegment>(other, start);
-}
-
+//===--------------------------------------------------------------------===//
+// Construct/Destruct
+//===--------------------------------------------------------------------===//
 ColumnSegment::ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, LogicalType type_p,
                              ColumnSegmentType segment_type, idx_t start, idx_t count, CompressionFunction &function_p,
                              BaseStatistics statistics, block_id_t block_id_p, idx_t offset_p, idx_t segment_size_p,
                              unique_ptr<ColumnSegmentState> segment_state)
+
     : SegmentBase<ColumnSegment>(start, count), db(db), type(std::move(type_p)),
       type_size(GetTypeIdSize(type.InternalType())), segment_type(segment_type), function(function_p),
       stats(std::move(statistics)), block(std::move(block)), block_id(block_id_p), offset(offset_p),
       segment_size(segment_size_p) {
+
 	if (function.get().init_segment) {
 		this->segment_state = function.get().init_segment(*this, block_id, segment_state.get());
 	}
+	D_ASSERT(segment_size <= Storage::BLOCK_SIZE);
 }
 
 ColumnSegment::ColumnSegment(ColumnSegment &other, idx_t start)
+
     : SegmentBase<ColumnSegment>(start, other.count.load()), db(other.db), type(std::move(other.type)),
       type_size(other.type_size), segment_type(other.segment_type), function(other.function),
       stats(std::move(other.stats)), block(std::move(other.block)), block_id(other.block_id), offset(other.offset),
       segment_size(other.segment_size), segment_state(std::move(other.segment_state)) {
+
+	D_ASSERT(segment_size <= Storage::BLOCK_SIZE);
 }
 
 ColumnSegment::~ColumnSegment() {
@@ -128,11 +145,14 @@ idx_t ColumnSegment::SegmentSize() const {
 void ColumnSegment::Resize(idx_t new_size) {
 	D_ASSERT(new_size > this->segment_size);
 	D_ASSERT(offset == 0);
+	D_ASSERT(new_size <= Storage::BLOCK_SIZE);
+
 	auto &buffer_manager = BufferManager::GetBufferManager(db);
 	auto old_handle = buffer_manager.Pin(block);
 	shared_ptr<BlockHandle> new_block;
-	auto new_handle = buffer_manager.Allocate(Storage::BLOCK_SIZE, false, &new_block);
+	auto new_handle = buffer_manager.Allocate(MemoryTag::IN_MEMORY_TABLE, new_size, false, &new_block);
 	memcpy(new_handle.Ptr(), old_handle.Ptr(), segment_size);
+
 	this->block_id = new_block->BlockId();
 	this->block = std::move(new_block);
 	this->segment_size = new_size;
@@ -439,6 +459,13 @@ idx_t ColumnSegment::FilterSelection(SelectionVector &sel, Vector &result, const
 			                                 constant_filter.comparison_type, mask);
 			break;
 		}
+		case PhysicalType::UINT128: {
+			auto result_flat = FlatVector::GetData<uhugeint_t>(result);
+			auto predicate = UhugeIntValue::Get(constant_filter.constant);
+			FilterSelectionSwitch<uhugeint_t>(result_flat, predicate, sel, approved_tuple_count,
+			                                  constant_filter.comparison_type, mask);
+			break;
+		}
 		case PhysicalType::FLOAT: {
 			auto result_flat = FlatVector::GetData<float>(result);
 			auto predicate = FloatValue::Get(constant_filter.constant);
@@ -476,6 +503,13 @@ idx_t ColumnSegment::FilterSelection(SelectionVector &sel, Vector &result, const
 		return TemplatedNullSelection<true>(sel, approved_tuple_count, mask);
 	case TableFilterType::IS_NOT_NULL:
 		return TemplatedNullSelection<false>(sel, approved_tuple_count, mask);
+	case TableFilterType::STRUCT_EXTRACT: {
+		auto &struct_filter = filter.Cast<StructFilter>();
+		// Apply the filter on the child vector
+		auto &child_vec = StructVector::GetEntries(result)[struct_filter.child_idx];
+		auto &child_mask = FlatVector::Validity(*child_vec);
+		return FilterSelection(sel, *child_vec, *struct_filter.child_filter, approved_tuple_count, child_mask);
+	}
 	default:
 		throw InternalException("FIXME: unsupported type for filter selection");
 	}

package/src/duckdb/src/storage/table/list_column_data.cpp

@@ -113,6 +113,7 @@ idx_t ListColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t co
 	if (child_scan_count > 0) {
 		auto &child_entry = ListVector::GetEntry(result);
 		if (child_entry.GetType().InternalType() != PhysicalType::STRUCT &&
+		    child_entry.GetType().InternalType() != PhysicalType::ARRAY &&
 		    state.child_states[1].row_index + child_scan_count > child_column->start + child_column->GetMaxEntry()) {
 			throw InternalException("ListColumnData::ScanCount - internal list scan offset is out of range");
 		}
@@ -131,14 +132,14 @@ void ListColumnData::Skip(ColumnScanState &state, idx_t count) {
 	// we need to read the list entries/offsets to figure out how much to skip
 	// note that we only need to read the first and last entry
 	// however, let's just read all "count" entries for now
-	Vector result(LogicalType::UBIGINT, count);
-	idx_t scan_count = ScanVector(state, result, count, false);
-	if (scan_count == 0) {
-		return;
-	}
+	Vector offset_vector(LogicalType::UBIGINT, count);
+	idx_t scan_count = ScanVector(state, offset_vector, count, false);
+	D_ASSERT(scan_count > 0);
 
-	auto data = FlatVector::GetData<uint64_t>(result);
-	auto last_entry = data[scan_count - 1];
+	UnifiedVectorFormat offsets;
+	offset_vector.ToUnifiedFormat(scan_count, offsets);
+	auto data = UnifiedVectorFormat::GetData<uint64_t>(offsets);
+	auto last_entry = data[offsets.sel->get_index(scan_count - 1)];
 	idx_t child_scan_count = last_entry - state.last_offset;
 	if (child_scan_count == 0) {
 		return;
@@ -291,7 +292,7 @@ void ListColumnData::FetchRow(TransactionData transaction, ColumnFetchState &sta
 		auto &child_type = ListType::GetChildType(result.GetType());
 		Vector child_scan(child_type, child_scan_count);
 		// seek the scan towards the specified position and read [length] entries
-		child_state->Initialize(child_type);
+		child_state->Initialize(child_type, nullptr);
 		child_column->InitializeScanWithOffset(*child_state, start + start_offset);
 		D_ASSERT(child_type.InternalType() == PhysicalType::STRUCT ||
 		         child_state->row_index + child_scan_count - this->start <= child_column->GetMaxEntry());
@@ -339,8 +340,8 @@ unique_ptr<ColumnCheckpointState> ListColumnData::CreateCheckpointState(RowGroup
 unique_ptr<ColumnCheckpointState> ListColumnData::Checkpoint(RowGroup &row_group,
                                                              PartialBlockManager &partial_block_manager,
                                                              ColumnCheckpointInfo &checkpoint_info) {
-	auto validity_state = validity.Checkpoint(row_group, partial_block_manager, checkpoint_info);
 	auto base_state = ColumnData::Checkpoint(row_group, partial_block_manager, checkpoint_info);
+	auto validity_state = validity.Checkpoint(row_group, partial_block_manager, checkpoint_info);
 	auto child_state = child_column->Checkpoint(row_group, partial_block_manager, checkpoint_info);
 
 	auto &checkpoint_state = base_state->Cast<ListColumnCheckpointState>();
@@ -349,14 +350,16 @@ unique_ptr<ColumnCheckpointState> ListColumnData::Checkpoint(RowGroup &row_group
 	return base_state;
 }
 
-void ListColumnData::DeserializeColumn(Deserializer &deserializer) {
-	ColumnData::DeserializeColumn(deserializer);
+void ListColumnData::DeserializeColumn(Deserializer &deserializer, BaseStatistics &target_stats) {
+	ColumnData::DeserializeColumn(deserializer, target_stats);
 
-	deserializer.ReadObject(101, "validity",
-	                        [&](Deserializer &deserializer) { validity.DeserializeColumn(deserializer); });
+	deserializer.ReadObject(
+	    101, "validity", [&](Deserializer &deserializer) { validity.DeserializeColumn(deserializer, target_stats); });
 
-	deserializer.ReadObject(102, "child_column",
-	                        [&](Deserializer &deserializer) { child_column->DeserializeColumn(deserializer); });
+	auto &child_stats = ListStats::GetChildStats(target_stats);
+	deserializer.ReadObject(102, "child_column", [&](Deserializer &deserializer) {
+		child_column->DeserializeColumn(deserializer, child_stats);
+	});
 }
 
 void ListColumnData::GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,

package/src/duckdb/src/storage/table/row_group.cpp

@@ -20,6 +20,8 @@
 #include "duckdb/common/serializer/serializer.hpp"
 #include "duckdb/common/serializer/deserializer.hpp"
 #include "duckdb/common/serializer/binary_serializer.hpp"
+#include "duckdb/planner/filter/conjunction_filter.hpp"
+#include "duckdb/planner/filter/struct_filter.hpp"
 
 namespace duckdb {
 
@@ -99,7 +101,7 @@ ColumnData &RowGroup::GetColumn(storage_t c) {
 	auto &block_pointer = column_pointers[c];
 	MetadataReader column_data_reader(metadata_manager, block_pointer);
 	this->columns[c] =
-	    ColumnData::Deserialize(GetBlockManager(), GetTableInfo(), c, start, column_data_reader, types[c], nullptr);
+	    ColumnData::Deserialize(GetBlockManager(), GetTableInfo(), c, start, column_data_reader, types[c]);
 	is_loaded[c] = true;
 	if (this->columns[c]->count != this->count) {
 		throw InternalException("Corrupted database - loaded column with index %llu at row start %llu, count %llu did "
@@ -125,7 +127,10 @@ void RowGroup::InitializeEmpty(const vector<LogicalType> &types) {
 	}
 }
 
-void ColumnScanState::Initialize(const LogicalType &type) {
+void ColumnScanState::Initialize(const LogicalType &type, optional_ptr<TableScanOptions> options) {
+	// Register the options in the state
+	scan_options = options;
+
 	if (type.id() == LogicalTypeId::VALIDITY) {
 		// validity - nothing to initialize
 		return;
@@ -135,15 +140,23 @@ void ColumnScanState::Initialize(const LogicalType &type) {
 		auto &struct_children = StructType::GetChildTypes(type);
 		child_states.resize(struct_children.size() + 1);
 		for (idx_t i = 0; i < struct_children.size(); i++) {
-			child_states[i + 1].Initialize(struct_children[i].second);
+			child_states[i + 1].Initialize(struct_children[i].second, options);
 		}
+		child_states[0].scan_options = options;
 	} else if (type.InternalType() == PhysicalType::LIST) {
 		// validity + list child
 		child_states.resize(2);
-		child_states[1].Initialize(ListType::GetChildType(type));
+		child_states[1].Initialize(ListType::GetChildType(type), options);
+		child_states[0].scan_options = options;
+	} else if (type.InternalType() == PhysicalType::ARRAY) {
+		// validity + array child
+		child_states.resize(2);
+		child_states[0].scan_options = options;
+		child_states[1].Initialize(ArrayType::GetChildType(type), options);
 	} else {
 		// validity
 		child_states.resize(1);
+		child_states[0].scan_options = options;
 	}
 }
 
@@ -154,7 +167,7 @@ void CollectionScanState::Initialize(const vector<LogicalType> &types) {
 		if (column_ids[i] == COLUMN_IDENTIFIER_ROW_ID) {
 			continue;
 		}
-		column_scans[i].Initialize(types[column_ids[i]]);
+		column_scans[i].Initialize(types[column_ids[i]], &GetOptions());
 	}
 }
 
@@ -177,6 +190,7 @@ bool RowGroup::InitializeScanWithOffset(CollectionScanState &state, idx_t vector
 		if (column != COLUMN_IDENTIFIER_ROW_ID) {
 			auto &column_data = GetColumn(column);
 			column_data.InitializeScanWithOffset(state.column_scans[i], start + vector_offset * STANDARD_VECTOR_SIZE);
+			state.column_scans[i].scan_options = &state.GetOptions();
 		} else {
 			state.column_scans[i].current = nullptr;
 		}
@@ -205,6 +219,7 @@ bool RowGroup::InitializeScan(CollectionScanState &state) {
 		if (column != COLUMN_IDENTIFIER_ROW_ID) {
 			auto &column_data = GetColumn(column);
 			column_data.InitializeScan(state.column_scans[i]);
+			state.column_scans[i].scan_options = &state.GetOptions();
 		} else {
 			state.column_scans[i].current = nullptr;
 		}
@@ -349,6 +364,40 @@ bool RowGroup::CheckZonemap(TableFilterSet &filters, const vector<storage_t> &co
 	return true;
 }
 
+static idx_t GetFilterScanCount(ColumnScanState &state, TableFilter &filter) {
+	switch (filter.filter_type) {
+	case TableFilterType::STRUCT_EXTRACT: {
+		auto &struct_filter = filter.Cast<StructFilter>();
+		auto &child_state = state.child_states[1 + struct_filter.child_idx]; // +1 for validity
+		auto &child_filter = struct_filter.child_filter;
+		return GetFilterScanCount(child_state, *child_filter);
+	}
+	case TableFilterType::CONJUNCTION_AND: {
+		auto &conjunction_state = filter.Cast<ConjunctionAndFilter>();
+		idx_t max_count = 0;
+		for (auto &child_filter : conjunction_state.child_filters) {
+			max_count = std::max(GetFilterScanCount(state, *child_filter), max_count);
+		}
+		return max_count;
+	}
+	case TableFilterType::CONJUNCTION_OR: {
+		auto &conjunction_state = filter.Cast<ConjunctionOrFilter>();
+		idx_t max_count = 0;
+		for (auto &child_filter : conjunction_state.child_filters) {
+			max_count = std::max(GetFilterScanCount(state, *child_filter), max_count);
+		}
+		return max_count;
+	}
+	case TableFilterType::IS_NULL:
+	case TableFilterType::IS_NOT_NULL:
+	case TableFilterType::CONSTANT_COMPARISON:
+		return state.current->start + state.current->count;
+	default: {
+		throw NotImplementedException("Unimplemented filter type for zonemap");
+	}
+	}
+}
+
 bool RowGroup::CheckZonemapSegments(CollectionScanState &state) {
 	auto &column_ids = state.GetColumnIds();
 	auto filters = state.GetFilters();
@@ -361,8 +410,9 @@ bool RowGroup::CheckZonemapSegments(CollectionScanState &state) {
 		const auto &base_column_idx = column_ids[column_idx];
 		bool read_segment = GetColumn(base_column_idx).CheckZonemap(state.column_scans[column_idx], *entry.second);
 		if (!read_segment) {
-			idx_t target_row =
-			    state.column_scans[column_idx].current->start + state.column_scans[column_idx].current->count;
+
+			idx_t target_row = GetFilterScanCount(state.column_scans[column_idx], *entry.second);
+
 			D_ASSERT(target_row >= this->start);
 			D_ASSERT(target_row <= this->start + this->count);
 			idx_t target_vector_index = (target_row - this->start) / STANDARD_VECTOR_SIZE;
@@ -664,6 +714,7 @@ void RowGroup::InitializeAppend(RowGroupAppendState &append_state) {
 
 void RowGroup::Append(RowGroupAppendState &state, DataChunk &chunk, idx_t append_count) {
 	// append to the current row_group
+	D_ASSERT(chunk.ColumnCount() == GetColumnCount());
 	for (idx_t i = 0; i < GetColumnCount(); i++) {
 		auto &col_data = GetColumn(i);
 		col_data.Append(state.states[i], chunk.data[i], append_count);
@@ -755,16 +806,12 @@ RowGroupWriteData RowGroup::WriteToDisk(PartialBlockManager &manager,
 	return result;
 }
 
-bool RowGroup::AllDeleted() {
-	if (HasUnloadedDeletes()) {
-		// deletes aren't loaded yet - we know not everything is deleted
-		return false;
-	}
+idx_t RowGroup::GetCommittedRowCount() {
 	auto &vinfo = GetVersionInfo();
 	if (!vinfo) {
-		return false;
+		return count;
 	}
-	return vinfo->GetCommittedDeletedCount(count) == count;
+	return count - vinfo->GetCommittedDeletedCount(count);
 }
 
 bool RowGroup::HasUnloadedDeletes() const {
@@ -776,9 +823,7 @@ bool RowGroup::HasUnloadedDeletes() const {
 	return !deletes_is_loaded;
 }
 
-RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats) {
-	RowGroupPointer row_group_pointer;
-
+RowGroupWriteData RowGroup::WriteToDisk(RowGroupWriter &writer) {
 	vector<CompressionType> compression_types;
 	compression_types.reserve(columns.size());
 	for (idx_t column_idx = 0; column_idx < GetColumnCount(); column_idx++) {
@@ -790,16 +835,23 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, TableStatistics &gl
 		}
 		compression_types.push_back(writer.GetColumnCompressionType(column_idx));
 	}
-	auto result = WriteToDisk(writer.GetPartialBlockManager(), compression_types);
+
+	return WriteToDisk(writer.GetPartialBlockManager(), compression_types);
+}
+
+RowGroupPointer RowGroup::Checkpoint(RowGroupWriteData write_data, RowGroupWriter &writer,
+                                     TableStatistics &global_stats) {
+	RowGroupPointer row_group_pointer;
+
 	for (idx_t column_idx = 0; column_idx < GetColumnCount(); column_idx++) {
-		global_stats.GetStats(column_idx).Statistics().Merge(result.statistics[column_idx]);
+		global_stats.GetStats(column_idx).Statistics().Merge(write_data.statistics[column_idx]);
 	}
 
 	// construct the row group pointer and write the column meta data to disk
-	D_ASSERT(result.states.size() == columns.size());
+	D_ASSERT(write_data.states.size() == columns.size());
 	row_group_pointer.row_start = start;
 	row_group_pointer.tuple_count = count;
-	for (auto &state : result.states) {
+	for (auto &state : write_data.states) {
 		// get the current position of the table data writer
 		auto &data_writer = writer.GetPayloadWriter();
 		auto pointer = data_writer.GetMetaBlockPointer();