duckdb 0.7.2-dev1034.0 → 0.7.2-dev1146.0

package/src/duckdb/src/storage/table/column_data.cpp

@@ -10,27 +10,35 @@
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/list_column_data.hpp"
 #include "duckdb/storage/table/standard_column_data.hpp"
-#include "duckdb/transaction/transaction.hpp"
 
 #include "duckdb/storage/table/struct_column_data.hpp"
 #include "duckdb/storage/table/update_segment.hpp"
 #include "duckdb/storage/table_storage_info.hpp"
+#include "duckdb/storage/table/append_state.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/main/attached_database.hpp"
 
 namespace duckdb {
 
 ColumnData::ColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
-                       LogicalType type, ColumnData *parent)
-    : block_manager(block_manager), info(info), column_index(column_index), start(start_row), type(std::move(type)),
-      parent(parent), version(0) {
+                       LogicalType type_p, ColumnData *parent)
+    : start(start_row), count(0), block_manager(block_manager), info(info), column_index(column_index),
+      type(std::move(type_p)), parent(parent), version(0) {
+	if (!parent) {
+		stats = make_unique<SegmentStatistics>(type);
+	}
 }
 
 ColumnData::ColumnData(ColumnData &other, idx_t start, ColumnData *parent)
-    : block_manager(other.block_manager), info(other.info), column_index(other.column_index), start(start),
-      type(std::move(other.type)), parent(parent), version(parent ? parent->version + 1 : 0) {
+    : start(start), count(other.count), block_manager(other.block_manager), info(other.info),
+      column_index(other.column_index), type(std::move(other.type)), parent(parent),
+      version(parent ? parent->version + 1 : 0) {
 	if (other.updates) {
 		updates = make_unique<UpdateSegment>(*other.updates, *this);
 	}
+	if (other.stats) {
+		stats = make_unique<SegmentStatistics>(other.stats->statistics.Copy());
+	}
 	idx_t offset = 0;
 	for (auto &segment : other.data.Segments()) {
 		this->data.AppendSegment(ColumnSegment::CreateSegment(segment, start + offset));
@@ -61,16 +69,7 @@ void ColumnData::IncrementVersion() {
 }
 
 idx_t ColumnData::GetMaxEntry() {
-	auto l = data.Lock();
-	auto first_segment = data.GetRootSegment(l);
-	auto last_segment = data.GetLastSegment(l);
-	if (!first_segment) {
-		D_ASSERT(!last_segment);
-		return 0;
-	} else {
-		D_ASSERT(last_segment->start >= first_segment->start);
-		return last_segment->start + last_segment->count - first_segment->start;
-	}
+	return count;
 }
 
 void ColumnData::InitializeScan(ColumnScanState &state) {
@@ -81,6 +80,7 @@ void ColumnData::InitializeScan(ColumnScanState &state) {
 	state.initialized = false;
 	state.version = version;
 	state.scan_state.reset();
+	state.last_offset = 0;
 }
 
 void ColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) {
@@ -91,6 +91,7 @@ void ColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx)
 	state.initialized = false;
 	state.version = version;
 	state.scan_state.reset();
+	state.last_offset = 0;
 }
 
 idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remaining) {
@@ -230,6 +231,46 @@ void ColumnData::Append(BaseStatistics &stats, ColumnAppendState &state, Vector
 	AppendData(stats, state, vdata, count);
 }
 
+void ColumnData::Append(ColumnAppendState &state, Vector &vector, idx_t count) {
+	if (parent || !stats) {
+		throw InternalException("ColumnData::Append called on a column with a parent or without stats");
+	}
+	Append(stats->statistics, state, vector, count);
+}
+
+bool ColumnData::CheckZonemap(TableFilter &filter) {
+	if (!stats) {
+		throw InternalException("ColumnData::CheckZonemap called on a column without stats");
+	}
+	auto propagate_result = filter.CheckStatistics(stats->statistics);
+	if (propagate_result == FilterPropagateResult::FILTER_ALWAYS_FALSE ||
+	    propagate_result == FilterPropagateResult::FILTER_FALSE_OR_NULL) {
+		return false;
+	}
+	return true;
+}
+
+unique_ptr<BaseStatistics> ColumnData::GetStatistics() {
+	if (!stats) {
+		throw InternalException("ColumnData::GetStatistics called on a column without stats");
+	}
+	return stats->statistics.ToUnique();
+}
+
+void ColumnData::MergeStatistics(const BaseStatistics &other) {
+	if (!stats) {
+		throw InternalException("ColumnData::MergeStatistics called on a column without stats");
+	}
+	return stats->statistics.Merge(other);
+}
+
+void ColumnData::MergeIntoStatistics(BaseStatistics &other) {
+	if (!stats) {
+		throw InternalException("ColumnData::MergeIntoStatistics called on a column without stats");
+	}
+	return other.Merge(stats->statistics);
+}
+
 void ColumnData::InitializeAppend(ColumnAppendState &state) {
 	auto l = data.Lock();
 	if (data.IsEmpty(l)) {
@@ -253,6 +294,7 @@ void ColumnData::InitializeAppend(ColumnAppendState &state) {
 
 void ColumnData::AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata, idx_t count) {
 	idx_t offset = 0;
+	this->count += count;
 	while (true) {
 		// append the data from the vector
 		idx_t copied_elements = state.current->Append(state, vdata, offset, count);
@@ -292,6 +334,7 @@ void ColumnData::RevertAppend(row_t start_row) {
 	// remove any segments AFTER this segment: they should be deleted entirely
 	data.EraseSegments(l, segment_index);
 
+	this->count = start_row - this->start;
 	segment->next = nullptr;
 	transient.RevertAppend(start_row);
 }
@@ -412,6 +455,7 @@ unique_ptr<ColumnCheckpointState> ColumnData::Checkpoint(RowGroup &row_group,
 
 void ColumnData::DeserializeColumn(Deserializer &source) {
 	// load the data pointers for the column
+	this->count = 0;
 	idx_t data_pointer_count = source.Read<idx_t>();
 	for (idx_t data_ptr = 0; data_ptr < data_pointer_count; data_ptr++) {
 		// read the data pointer
@@ -420,15 +464,20 @@ void ColumnData::DeserializeColumn(Deserializer &source) {
 		auto block_pointer_block_id = source.Read<block_id_t>();
 		auto block_pointer_offset = source.Read<uint32_t>();
 		auto compression_type = source.Read<CompressionType>();
-		auto stats = BaseStatistics::Deserialize(source, type);
+		auto segment_stats = BaseStatistics::Deserialize(source, type);
+		if (stats) {
+			stats->statistics.Merge(segment_stats);
+		}
 
-		DataPointer data_pointer(std::move(stats));
+		DataPointer data_pointer(std::move(segment_stats));
 		data_pointer.row_start = row_start;
 		data_pointer.tuple_count = tuple_count;
 		data_pointer.block_pointer.block_id = block_pointer_block_id;
 		data_pointer.block_pointer.offset = block_pointer_offset;
 		data_pointer.compression_type = compression_type;
 
+		this->count += tuple_count;
+
 		// create a persistent segment
 		auto segment = ColumnSegment::CreatePersistentSegment(
 		    GetDatabase(), block_manager, data_pointer.block_pointer.block_id, data_pointer.block_pointer.offset, type,
@@ -495,14 +544,22 @@ void ColumnData::Verify(RowGroup &parent) {
 #ifdef DEBUG
 	D_ASSERT(this->start == parent.start);
 	data.Verify();
+	if (type.InternalType() == PhysicalType::STRUCT) {
+		// structs don't have segments
+		D_ASSERT(!data.GetRootSegment());
+		return;
+	}
 	idx_t current_index = 0;
 	idx_t current_start = this->start;
+	idx_t total_count = 0;
 	for (auto &segment : data.Segments()) {
 		D_ASSERT(segment.index == current_index);
 		D_ASSERT(segment.start == current_start);
 		current_start += segment.count;
+		total_count += segment.count;
 		current_index++;
 	}
+	D_ASSERT(this->count == total_count);
 #endif
 }
 

package/src/duckdb/src/storage/table/column_data_checkpointer.cpp

@@ -3,6 +3,8 @@
 #include "duckdb/storage/table/update_segment.hpp"
 #include "duckdb/storage/data_table.hpp"
 #include "duckdb/parser/column_definition.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
+
 namespace duckdb {
 
 ColumnDataCheckpointer::ColumnDataCheckpointer(ColumnData &col_data_p, RowGroup &row_group_p,
@@ -161,7 +163,7 @@ void ColumnDataCheckpointer::WriteToDisk() {
 	// first we check the current segments
 	// if there are any persistent segments, we will mark their old block ids as modified
 	// since the segments will be rewritten their old on disk data is no longer required
-	auto &block_manager = col_data.block_manager;
+	auto &block_manager = col_data.GetBlockManager();
 	for (idx_t segment_idx = 0; segment_idx < nodes.size(); segment_idx++) {
 		auto segment = nodes[segment_idx].node.get();
 		if (segment->segment_type == ColumnSegmentType::PERSISTENT) {

package/src/duckdb/src/storage/table/column_segment.cpp

@@ -3,13 +3,12 @@
 #include "duckdb/storage/table/update_segment.hpp"
 #include "duckdb/common/types/null_value.hpp"
 #include "duckdb/common/types/vector.hpp"
-#include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/storage_manager.hpp"
 #include "duckdb/planner/filter/conjunction_filter.hpp"
 #include "duckdb/planner/filter/constant_filter.hpp"
-#include "duckdb/planner/filter/null_filter.hpp"
 #include "duckdb/main/config.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 #include <cstring>
 
@@ -217,12 +216,12 @@ void ColumnSegment::MarkAsPersistent(shared_ptr<BlockHandle> block_p, uint32_t o
 // Filter Selection
 //===--------------------------------------------------------------------===//
 template <class T, class OP, bool HAS_NULL>
-static idx_t TemplatedFilterSelection(T *vec, T *predicate, SelectionVector &sel, idx_t approved_tuple_count,
+static idx_t TemplatedFilterSelection(T *vec, T predicate, SelectionVector &sel, idx_t approved_tuple_count,
                                       ValidityMask &mask, SelectionVector &result_sel) {
 	idx_t result_count = 0;
 	for (idx_t i = 0; i < approved_tuple_count; i++) {
 		auto idx = sel.get_index(i);
-		if ((!HAS_NULL || mask.RowIsValid(idx)) && OP::Operation(vec[idx], *predicate)) {
+		if ((!HAS_NULL || mask.RowIsValid(idx)) && OP::Operation(vec[idx], predicate)) {
 			result_sel.set_index(result_count++, idx);
 		}
 	}
@@ -230,7 +229,7 @@ static idx_t TemplatedFilterSelection(T *vec, T *predicate, SelectionVector &sel
 }
 
 template <class T>
-static void FilterSelectionSwitch(T *vec, T *predicate, SelectionVector &sel, idx_t &approved_tuple_count,
+static void FilterSelectionSwitch(T *vec, T predicate, SelectionVector &sel, idx_t &approved_tuple_count,
                                   ExpressionType comparison_type, ValidityMask &mask) {
 	SelectionVector new_sel(approved_tuple_count);
 	// the inplace loops take the result as the last parameter
@@ -371,104 +370,91 @@ idx_t ColumnSegment::FilterSelection(SelectionVector &sel, Vector &result, const
 		switch (result.GetType().InternalType()) {
 		case PhysicalType::UINT8: {
 			auto result_flat = FlatVector::GetData<uint8_t>(result);
-			Vector predicate_vector(constant_filter.constant);
-			auto predicate = FlatVector::GetData<uint8_t>(predicate_vector);
+			auto predicate = UTinyIntValue::Get(constant_filter.constant);
 			FilterSelectionSwitch<uint8_t>(result_flat, predicate, sel, approved_tuple_count,
 			                               constant_filter.comparison_type, mask);
 			break;
 		}
 		case PhysicalType::UINT16: {
 			auto result_flat = FlatVector::GetData<uint16_t>(result);
-			Vector predicate_vector(constant_filter.constant);
-			auto predicate = FlatVector::GetData<uint16_t>(predicate_vector);
+			auto predicate = USmallIntValue::Get(constant_filter.constant);
 			FilterSelectionSwitch<uint16_t>(result_flat, predicate, sel, approved_tuple_count,
 			                                constant_filter.comparison_type, mask);
 			break;
 		}
 		case PhysicalType::UINT32: {
 			auto result_flat = FlatVector::GetData<uint32_t>(result);
-			Vector predicate_vector(constant_filter.constant);
-			auto predicate = FlatVector::GetData<uint32_t>(predicate_vector);
+			auto predicate = UIntegerValue::Get(constant_filter.constant);
 			FilterSelectionSwitch<uint32_t>(result_flat, predicate, sel, approved_tuple_count,
 			                                constant_filter.comparison_type, mask);
 			break;
 		}
 		case PhysicalType::UINT64: {
 			auto result_flat = FlatVector::GetData<uint64_t>(result);
-			Vector predicate_vector(constant_filter.constant);
-			auto predicate = FlatVector::GetData<uint64_t>(predicate_vector);
+			auto predicate = UBigIntValue::Get(constant_filter.constant);
 			FilterSelectionSwitch<uint64_t>(result_flat, predicate, sel, approved_tuple_count,
 			                                constant_filter.comparison_type, mask);
 			break;
 		}
 		case PhysicalType::INT8: {
 			auto result_flat = FlatVector::GetData<int8_t>(result);
-			Vector predicate_vector(constant_filter.constant);
-			auto predicate = FlatVector::GetData<int8_t>(predicate_vector);
+			auto predicate = TinyIntValue::Get(constant_filter.constant);
 			FilterSelectionSwitch<int8_t>(result_flat, predicate, sel, approved_tuple_count,
 			                              constant_filter.comparison_type, mask);
 			break;
 		}
 		case PhysicalType::INT16: {
 			auto result_flat = FlatVector::GetData<int16_t>(result);
-			Vector predicate_vector(constant_filter.constant);
-			auto predicate = FlatVector::GetData<int16_t>(predicate_vector);
+			auto predicate = SmallIntValue::Get(constant_filter.constant);
 			FilterSelectionSwitch<int16_t>(result_flat, predicate, sel, approved_tuple_count,
 			                               constant_filter.comparison_type, mask);
 			break;
 		}
 		case PhysicalType::INT32: {
 			auto result_flat = FlatVector::GetData<int32_t>(result);
-			Vector predicate_vector(constant_filter.constant);
-			auto predicate = FlatVector::GetData<int32_t>(predicate_vector);
+			auto predicate = IntegerValue::Get(constant_filter.constant);
 			FilterSelectionSwitch<int32_t>(result_flat, predicate, sel, approved_tuple_count,
 			                               constant_filter.comparison_type, mask);
 			break;
 		}
 		case PhysicalType::INT64: {
 			auto result_flat = FlatVector::GetData<int64_t>(result);
-			Vector predicate_vector(constant_filter.constant);
-			auto predicate = FlatVector::GetData<int64_t>(predicate_vector);
+			auto predicate = BigIntValue::Get(constant_filter.constant);
 			FilterSelectionSwitch<int64_t>(result_flat, predicate, sel, approved_tuple_count,
 			                               constant_filter.comparison_type, mask);
 			break;
 		}
 		case PhysicalType::INT128: {
 			auto result_flat = FlatVector::GetData<hugeint_t>(result);
-			Vector predicate_vector(constant_filter.constant);
-			auto predicate = FlatVector::GetData<hugeint_t>(predicate_vector);
+			auto predicate = HugeIntValue::Get(constant_filter.constant);
 			FilterSelectionSwitch<hugeint_t>(result_flat, predicate, sel, approved_tuple_count,
 			                                 constant_filter.comparison_type, mask);
 			break;
 		}
 		case PhysicalType::FLOAT: {
 			auto result_flat = FlatVector::GetData<float>(result);
-			Vector predicate_vector(constant_filter.constant);
-			auto predicate = FlatVector::GetData<float>(predicate_vector);
+			auto predicate = FloatValue::Get(constant_filter.constant);
 			FilterSelectionSwitch<float>(result_flat, predicate, sel, approved_tuple_count,
 			                             constant_filter.comparison_type, mask);
 			break;
 		}
 		case PhysicalType::DOUBLE: {
 			auto result_flat = FlatVector::GetData<double>(result);
-			Vector predicate_vector(constant_filter.constant);
-			auto predicate = FlatVector::GetData<double>(predicate_vector);
+			auto predicate = DoubleValue::Get(constant_filter.constant);
 			FilterSelectionSwitch<double>(result_flat, predicate, sel, approved_tuple_count,
 			                              constant_filter.comparison_type, mask);
 			break;
 		}
 		case PhysicalType::VARCHAR: {
 			auto result_flat = FlatVector::GetData<string_t>(result);
-			Vector predicate_vector(constant_filter.constant);
-			auto predicate = FlatVector::GetData<string_t>(predicate_vector);
+			auto predicate = string_t(StringValue::Get(constant_filter.constant));
 			FilterSelectionSwitch<string_t>(result_flat, predicate, sel, approved_tuple_count,
 			                                constant_filter.comparison_type, mask);
 			break;
 		}
 		case PhysicalType::BOOL: {
 			auto result_flat = FlatVector::GetData<bool>(result);
-			Vector predicate_vector(constant_filter.constant);
-			auto predicate = FlatVector::GetData<bool>(predicate_vector);
+			auto predicate = BooleanValue::Get(constant_filter.constant);
 			FilterSelectionSwitch<bool>(result_flat, predicate, sel, approved_tuple_count,
 			                            constant_filter.comparison_type, mask);
 			break;

package/src/duckdb/src/storage/table/list_column_data.cpp

@@ -2,6 +2,8 @@
 #include "duckdb/storage/statistics/list_stats.hpp"
 #include "duckdb/transaction/transaction.hpp"
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
+#include "duckdb/storage/table/append_state.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 
@@ -30,14 +32,11 @@ void ListColumnData::InitializeScan(ColumnScanState &state) {
 	ColumnData::InitializeScan(state);
 
 	// initialize the validity segment
-	ColumnScanState validity_state;
-	validity.InitializeScan(validity_state);
-	state.child_states.push_back(std::move(validity_state));
+	D_ASSERT(state.child_states.size() == 2);
+	validity.InitializeScan(state.child_states[0]);
 
 	// initialize the child scan
-	ColumnScanState child_state;
-	child_column->InitializeScan(child_state);
-	state.child_states.push_back(std::move(child_state));
+	child_column->InitializeScan(state.child_states[1]);
 }
 
 uint64_t ListColumnData::FetchListOffset(idx_t row_idx) {
@@ -58,19 +57,16 @@ void ListColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_
 	ColumnData::InitializeScanWithOffset(state, row_idx);
 
 	// initialize the validity segment
-	ColumnScanState validity_state;
-	validity.InitializeScanWithOffset(validity_state, row_idx);
-	state.child_states.push_back(std::move(validity_state));
+	D_ASSERT(state.child_states.size() == 2);
+	validity.InitializeScanWithOffset(state.child_states[0], row_idx);
 
 	// we need to read the list at position row_idx to get the correct row offset of the child
 	auto child_offset = row_idx == start ? 0 : FetchListOffset(row_idx - 1);
 
 	D_ASSERT(child_offset <= child_column->GetMaxEntry());
-	ColumnScanState child_state;
 	if (child_offset < child_column->GetMaxEntry()) {
-		child_column->InitializeScanWithOffset(child_state, start + child_offset);
+		child_column->InitializeScanWithOffset(state.child_states[1], start + child_offset);
 	}
-	state.child_states.push_back(std::move(child_state));
 }
 
 idx_t ListColumnData::Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) {
@@ -290,6 +286,7 @@ void ListColumnData::FetchRow(TransactionData transaction, ColumnFetchState &sta
 		auto &child_type = ListType::GetChildType(result.GetType());
 		Vector child_scan(child_type, child_scan_count);
 		// seek the scan towards the specified position and read [length] entries
+		child_state->Initialize(child_type);
 		child_column->InitializeScanWithOffset(*child_state, start + start_offset);
 		D_ASSERT(child_type.InternalType() == PhysicalType::STRUCT ||
 		         child_state->row_index + child_scan_count - this->start <= child_column->GetMaxEntry());