duckdb 0.7.2-dev16.0 → 0.7.2-dev314.0

package/src/duckdb/src/storage/compression/fixed_size_uncompressed.cpp

@@ -5,7 +5,7 @@
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/storage/checkpoint/write_overflow_strings_to_disk.hpp"
 #include "duckdb/storage/segment/uncompressed.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
@@ -153,13 +153,7 @@ void FixedSizeScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_co
 	auto source_data = data + start * sizeof(T);
 
 	result.SetVectorType(VectorType::FLAT_VECTOR);
-	if (std::is_same<T, list_entry_t>()) {
-		// list columns are modified in-place during the scans to correct the offsets
-		// so we can't do a zero-copy there
-		memcpy(FlatVector::GetData(result), source_data, scan_count * sizeof(T));
-	} else {
-		FlatVector::SetData(result, source_data);
-	}
+	FlatVector::SetData(result, source_data);
 }
 
 //===--------------------------------------------------------------------===//
@@ -186,48 +180,52 @@ static unique_ptr<CompressionAppendState> FixedSizeInitAppend(ColumnSegment &seg
 	return make_unique<CompressionAppendState>(std::move(handle));
 }
 
-template <class T>
-static void AppendLoop(SegmentStatistics &stats, data_ptr_t target, idx_t target_offset, UnifiedVectorFormat &adata,
-                       idx_t offset, idx_t count) {
-	auto sdata = (T *)adata.data;
-	auto tdata = (T *)target;
-	if (!adata.validity.AllValid()) {
-		for (idx_t i = 0; i < count; i++) {
-			auto source_idx = adata.sel->get_index(offset + i);
-			auto target_idx = target_offset + i;
-			bool is_null = !adata.validity.RowIsValid(source_idx);
-			if (!is_null) {
-				NumericStatistics::Update<T>(stats, sdata[source_idx]);
+struct StandardFixedSizeAppend {
+	template <class T>
+	static void Append(SegmentStatistics &stats, data_ptr_t target, idx_t target_offset, UnifiedVectorFormat &adata,
+	                   idx_t offset, idx_t count) {
+		auto sdata = (T *)adata.data;
+		auto tdata = (T *)target;
+		if (!adata.validity.AllValid()) {
+			for (idx_t i = 0; i < count; i++) {
+				auto source_idx = adata.sel->get_index(offset + i);
+				auto target_idx = target_offset + i;
+				bool is_null = !adata.validity.RowIsValid(source_idx);
+				if (!is_null) {
+					NumericStats::Update<T>(stats.statistics, sdata[source_idx]);
+					tdata[target_idx] = sdata[source_idx];
+				} else {
+					// we insert a NullValue<T> in the null gap for debuggability
+					// this value should never be used or read anywhere
+					tdata[target_idx] = NullValue<T>();
+				}
+			}
+		} else {
+			for (idx_t i = 0; i < count; i++) {
+				auto source_idx = adata.sel->get_index(offset + i);
+				auto target_idx = target_offset + i;
+				NumericStats::Update<T>(stats.statistics, sdata[source_idx]);
 				tdata[target_idx] = sdata[source_idx];
-			} else {
-				// we insert a NullValue<T> in the null gap for debuggability
-				// this value should never be used or read anywhere
-				tdata[target_idx] = NullValue<T>();
 			}
 		}
-	} else {
+	}
+};
+
+struct ListFixedSizeAppend {
+	template <class T>
+	static void Append(SegmentStatistics &stats, data_ptr_t target, idx_t target_offset, UnifiedVectorFormat &adata,
+	                   idx_t offset, idx_t count) {
+		auto sdata = (uint64_t *)adata.data;
+		auto tdata = (uint64_t *)target;
 		for (idx_t i = 0; i < count; i++) {
 			auto source_idx = adata.sel->get_index(offset + i);
 			auto target_idx = target_offset + i;
-			NumericStatistics::Update<T>(stats, sdata[source_idx]);
 			tdata[target_idx] = sdata[source_idx];
 		}
 	}
-}
-
-template <>
-void AppendLoop<list_entry_t>(SegmentStatistics &stats, data_ptr_t target, idx_t target_offset,
-                              UnifiedVectorFormat &adata, idx_t offset, idx_t count) {
-	auto sdata = (list_entry_t *)adata.data;
-	auto tdata = (list_entry_t *)target;
-	for (idx_t i = 0; i < count; i++) {
-		auto source_idx = adata.sel->get_index(offset + i);
-		auto target_idx = target_offset + i;
-		tdata[target_idx] = sdata[source_idx];
-	}
-}
+};
 
-template <class T>
+template <class T, class OP>
 idx_t FixedSizeAppend(CompressionAppendState &append_state, ColumnSegment &segment, SegmentStatistics &stats,
                       UnifiedVectorFormat &data, idx_t offset, idx_t count) {
 	D_ASSERT(segment.GetBlockOffset() == 0);
@@ -236,7 +234,7 @@ idx_t FixedSizeAppend(CompressionAppendState &append_state, ColumnSegment &segme
 	idx_t max_tuple_count = segment.SegmentSize() / sizeof(T);
 	idx_t copy_count = MinValue<idx_t>(count, max_tuple_count - segment.count);
 
-	AppendLoop<T>(stats, target_ptr, segment.count, data, offset, copy_count);
+	OP::template Append<T>(stats, target_ptr, segment.count, data, offset, copy_count);
 	segment.count += copy_count;
 	return copy_count;
 }
@@ -249,14 +247,14 @@ idx_t FixedSizeFinalizeAppend(ColumnSegment &segment, SegmentStatistics &stats)
 //===--------------------------------------------------------------------===//
 // Get Function
 //===--------------------------------------------------------------------===//
-template <class T>
+template <class T, class APPENDER = StandardFixedSizeAppend>
 CompressionFunction FixedSizeGetFunction(PhysicalType data_type) {
 	return CompressionFunction(CompressionType::COMPRESSION_UNCOMPRESSED, data_type, FixedSizeInitAnalyze,
 	                           FixedSizeAnalyze, FixedSizeFinalAnalyze<T>, UncompressedFunctions::InitCompression,
 	                           UncompressedFunctions::Compress, UncompressedFunctions::FinalizeCompress,
 	                           FixedSizeInitScan, FixedSizeScan<T>, FixedSizeScanPartial<T>, FixedSizeFetchRow<T>,
-	                           UncompressedFunctions::EmptySkip, nullptr, FixedSizeInitAppend, FixedSizeAppend<T>,
-	                           FixedSizeFinalizeAppend<T>, nullptr);
+	                           UncompressedFunctions::EmptySkip, nullptr, FixedSizeInitAppend,
+	                           FixedSizeAppend<T, APPENDER>, FixedSizeFinalizeAppend<T>, nullptr);
 }
 
 CompressionFunction FixedSizeUncompressed::GetFunction(PhysicalType data_type) {
@@ -287,7 +285,7 @@ CompressionFunction FixedSizeUncompressed::GetFunction(PhysicalType data_type) {
 	case PhysicalType::INTERVAL:
 		return FixedSizeGetFunction<interval_t>(data_type);
 	case PhysicalType::LIST:
-		return FixedSizeGetFunction<list_entry_t>(data_type);
+		return FixedSizeGetFunction<uint64_t, ListFixedSizeAppend>(data_type);
 	default:
 		throw InternalException("Unsupported type for FixedSizeUncompressed::GetFunction");
 	}

package/src/duckdb/src/storage/compression/numeric_constant.cpp

@@ -1,8 +1,7 @@
 #include "duckdb/function/compression/compression.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/common/types/vector.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
-#include "duckdb/storage/statistics/validity_statistics.hpp"
+
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/storage/segment/uncompressed.hpp"
@@ -20,8 +19,8 @@ unique_ptr<SegmentScanState> ConstantInitScan(ColumnSegment &segment) {
 // Scan Partial
 //===--------------------------------------------------------------------===//
 void ConstantFillFunctionValidity(ColumnSegment &segment, Vector &result, idx_t start_idx, idx_t count) {
-	auto &validity = (ValidityStatistics &)*segment.stats.statistics;
-	if (validity.has_null) {
+	auto &stats = segment.stats.statistics;
+	if (stats.CanHaveNull()) {
 		auto &mask = FlatVector::Validity(result);
 		for (idx_t i = 0; i < count; i++) {
 			mask.SetInvalid(start_idx + i);
@@ -31,10 +30,10 @@ void ConstantFillFunctionValidity(ColumnSegment &segment, Vector &result, idx_t
 
 template <class T>
 void ConstantFillFunction(ColumnSegment &segment, Vector &result, idx_t start_idx, idx_t count) {
-	auto &nstats = (NumericStatistics &)*segment.stats.statistics;
+	auto &nstats = segment.stats.statistics;
 
 	auto data = FlatVector::GetData<T>(result);
-	auto constant_value = nstats.min.GetValueUnsafe<T>();
+	auto constant_value = NumericStats::GetMinUnsafe<T>(nstats);
 	for (idx_t i = 0; i < count; i++) {
 		data[start_idx + i] = constant_value;
 	}
@@ -55,8 +54,8 @@ void ConstantScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t s
 // Scan base data
 //===--------------------------------------------------------------------===//
 void ConstantScanFunctionValidity(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result) {
-	auto &validity = (ValidityStatistics &)*segment.stats.statistics;
-	if (validity.has_null) {
+	auto &stats = segment.stats.statistics;
+	if (stats.CanHaveNull()) {
 		if (result.GetVectorType() == VectorType::CONSTANT_VECTOR) {
 			result.SetVectorType(VectorType::CONSTANT_VECTOR);
 			ConstantVector::SetNull(result, true);
@@ -69,10 +68,10 @@ void ConstantScanFunctionValidity(ColumnSegment &segment, ColumnScanState &state
 
 template <class T>
 void ConstantScanFunction(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result) {
-	auto &nstats = (NumericStatistics &)*segment.stats.statistics;
+	auto &nstats = segment.stats.statistics;
 
 	auto data = FlatVector::GetData<T>(result);
-	data[0] = nstats.min.GetValueUnsafe<T>();
+	data[0] = NumericStats::GetMinUnsafe<T>(nstats);
 	result.SetVectorType(VectorType::CONSTANT_VECTOR);
 }
 

package/src/duckdb/src/storage/compression/patas.cpp

@@ -10,7 +10,7 @@
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/operator/subtract.hpp"

package/src/duckdb/src/storage/compression/rle.cpp

@@ -1,5 +1,5 @@
 #include "duckdb/function/compression/compression.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/config.hpp"
@@ -118,12 +118,12 @@ struct RLEConstants {
 	static constexpr const idx_t RLE_HEADER_SIZE = sizeof(uint64_t);
 };
 
-template <class T>
+template <class T, bool WRITE_STATISTICS>
 struct RLECompressState : public CompressionState {
 	struct RLEWriter {
 		template <class VALUE_TYPE>
 		static void Operation(VALUE_TYPE value, rle_count_t count, void *dataptr, bool is_null) {
-			auto state = (RLECompressState<T> *)dataptr;
+			auto state = (RLECompressState<T, WRITE_STATISTICS> *)dataptr;
 			state->WriteValue(value, count, is_null);
 		}
 	};
@@ -160,7 +160,7 @@ struct RLECompressState : public CompressionState {
 		auto data = (T *)vdata.data;
 		for (idx_t i = 0; i < count; i++) {
 			auto idx = vdata.sel->get_index(i);
-			state.template Update<RLECompressState<T>::RLEWriter>(data, vdata.validity, idx);
+			state.template Update<RLECompressState<T, WRITE_STATISTICS>::RLEWriter>(data, vdata.validity, idx);
 		}
 	}
 
@@ -174,8 +174,8 @@ struct RLECompressState : public CompressionState {
 		entry_count++;
 
 		// update meta data
-		if (!is_null) {
-			NumericStatistics::Update<T>(current_segment->stats, value);
+		if (WRITE_STATISTICS && !is_null) {
+			NumericStats::Update<T>(current_segment->stats.statistics, value);
 		}
 		current_segment->count += count;
 
@@ -206,7 +206,7 @@ struct RLECompressState : public CompressionState {
 	}
 
 	void Finalize() {
-		state.template Flush<RLECompressState<T>::RLEWriter>();
+		state.template Flush<RLECompressState<T, WRITE_STATISTICS>::RLEWriter>();
 
 		FlushSegment();
 		current_segment.reset();
@@ -222,23 +222,23 @@ struct RLECompressState : public CompressionState {
 	idx_t max_rle_count;
 };
 
-template <class T>
+template <class T, bool WRITE_STATISTICS>
 unique_ptr<CompressionState> RLEInitCompression(ColumnDataCheckpointer &checkpointer, unique_ptr<AnalyzeState> state) {
-	return make_unique<RLECompressState<T>>(checkpointer);
+	return make_unique<RLECompressState<T, WRITE_STATISTICS>>(checkpointer);
 }
 
-template <class T>
+template <class T, bool WRITE_STATISTICS>
 void RLECompress(CompressionState &state_p, Vector &scan_vector, idx_t count) {
-	auto &state = (RLECompressState<T> &)state_p;
+	auto &state = (RLECompressState<T, WRITE_STATISTICS> &)state_p;
 	UnifiedVectorFormat vdata;
 	scan_vector.ToUnifiedFormat(count, vdata);
 
 	state.Append(vdata, count);
 }
 
-template <class T>
+template <class T, bool WRITE_STATISTICS>
 void RLEFinalizeCompress(CompressionState &state_p) {
-	auto &state = (RLECompressState<T> &)state_p;
+	auto &state = (RLECompressState<T, WRITE_STATISTICS> &)state_p;
 	state.Finalize();
 }
 
@@ -341,10 +341,11 @@ void RLEFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id,
 //===--------------------------------------------------------------------===//
 // Get Function
 //===--------------------------------------------------------------------===//
-template <class T>
+template <class T, bool WRITE_STATISTICS = true>
 CompressionFunction GetRLEFunction(PhysicalType data_type) {
 	return CompressionFunction(CompressionType::COMPRESSION_RLE, data_type, RLEInitAnalyze<T>, RLEAnalyze<T>,
-	                           RLEFinalAnalyze<T>, RLEInitCompression<T>, RLECompress<T>, RLEFinalizeCompress<T>,
+	                           RLEFinalAnalyze<T>, RLEInitCompression<T, WRITE_STATISTICS>,
+	                           RLECompress<T, WRITE_STATISTICS>, RLEFinalizeCompress<T, WRITE_STATISTICS>,
 	                           RLEInitScan<T>, RLEScan<T>, RLEScanPartial<T>, RLEFetchRow<T>, RLESkip<T>);
 }
 
@@ -373,6 +374,8 @@ CompressionFunction RLEFun::GetFunction(PhysicalType type) {
 		return GetRLEFunction<float>(type);
 	case PhysicalType::DOUBLE:
 		return GetRLEFunction<double>(type);
+	case PhysicalType::LIST:
+		return GetRLEFunction<uint64_t, false>(type);
 	default:
 		throw InternalException("Unsupported type for RLE");
 	}
@@ -392,6 +395,7 @@ bool RLEFun::TypeIsSupported(PhysicalType type) {
 	case PhysicalType::UINT64:
 	case PhysicalType::FLOAT:
 	case PhysicalType::DOUBLE:
+	case PhysicalType::LIST:
 		return true;
 	default:
 		return false;

package/src/duckdb/src/storage/compression/validity_uncompressed.cpp

@@ -2,7 +2,7 @@
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/storage/table/append_state.hpp"
-#include "duckdb/storage/statistics/validity_statistics.hpp"
+
 #include "duckdb/common/types/null_value.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/function/compression_function.hpp"
@@ -410,14 +410,14 @@ unique_ptr<CompressedSegmentState> ValidityInitSegment(ColumnSegment &segment, b
 idx_t ValidityAppend(CompressionAppendState &append_state, ColumnSegment &segment, SegmentStatistics &stats,
                      UnifiedVectorFormat &data, idx_t offset, idx_t vcount) {
 	D_ASSERT(segment.GetBlockOffset() == 0);
-	auto &validity_stats = (ValidityStatistics &)*stats.statistics;
+	auto &validity_stats = stats.statistics;
 
 	auto max_tuples = segment.SegmentSize() / ValidityMask::STANDARD_MASK_SIZE * STANDARD_VECTOR_SIZE;
 	idx_t append_count = MinValue<idx_t>(vcount, max_tuples - segment.count);
 	if (data.validity.AllValid()) {
 		// no null values: skip append
 		segment.count += append_count;
-		validity_stats.has_no_null = true;
+		validity_stats.SetHasNoNull();
 		return append_count;
 	}
 
@@ -426,9 +426,9 @@ idx_t ValidityAppend(CompressionAppendState &append_state, ColumnSegment &segmen
 		auto idx = data.sel->get_index(offset + i);
 		if (!data.validity.RowIsValidUnsafe(idx)) {
 			mask.SetInvalidUnsafe(segment.count + i);
-			validity_stats.has_null = true;
+			validity_stats.SetHasNull();
 		} else {
-			validity_stats.has_no_null = true;
+			validity_stats.SetHasNoNull();
 		}
 	}
 	segment.count += append_count;

package/src/duckdb/src/storage/data_table.cpp

@@ -1223,9 +1223,9 @@ unique_ptr<BaseStatistics> DataTable::GetStatistics(ClientContext &context, colu
 	return row_groups->CopyStats(column_id);
 }
 
-void DataTable::SetStatistics(column_t column_id, const std::function<void(BaseStatistics &)> &set_fun) {
+void DataTable::SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats) {
 	D_ASSERT(column_id != COLUMN_IDENTIFIER_ROW_ID);
-	row_groups->SetStatistics(column_id, set_fun);
+	row_groups->SetDistinct(column_id, std::move(distinct_stats));
 }
 
 //===--------------------------------------------------------------------===//
@@ -1234,10 +1234,8 @@ void DataTable::SetStatistics(column_t column_id, const std::function<void(BaseS
 void DataTable::Checkpoint(TableDataWriter &writer) {
 	// checkpoint each individual row group
 	// FIXME: we might want to combine adjacent row groups in case they have had deletions...
-	vector<unique_ptr<BaseStatistics>> global_stats;
-	for (idx_t i = 0; i < column_definitions.size(); i++) {
-		global_stats.push_back(row_groups->CopyStats(i));
-	}
+	TableStatistics global_stats;
+	row_groups->CopyStats(global_stats);
 
 	row_groups->Checkpoint(writer, global_stats);