duckdb 0.7.2-dev225.0 → 0.7.2-dev314.0

package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp

@@ -5,6 +5,7 @@
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/common/serializer/buffered_serializer.hpp"
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
+#include "duckdb/storage/table/table_statistics.hpp"
 
 namespace duckdb {
 
@@ -40,14 +41,13 @@ unique_ptr<RowGroupWriter> SingleFileTableDataWriter::GetRowGroupWriter(RowGroup
 	return make_unique<SingleFileRowGroupWriter>(table, checkpoint_manager.partial_block_manager, table_data_writer);
 }
 
-void SingleFileTableDataWriter::FinalizeTable(vector<unique_ptr<BaseStatistics>> &&global_stats, DataTableInfo *info) {
+void SingleFileTableDataWriter::FinalizeTable(TableStatistics &&global_stats, DataTableInfo *info) {
 	// store the current position in the metadata writer
 	// this is where the row groups for this table start
 	auto pointer = table_data_writer.GetBlockPointer();
 
-	for (auto &stats : global_stats) {
-		stats->Serialize(table_data_writer);
-	}
+	global_stats.Serialize(table_data_writer);
+
 	// now start writing the row group pointers to disk
 	table_data_writer.Write<uint64_t>(row_group_pointers.size());
 	for (auto &row_group_pointer : row_group_pointers) {

package/src/duckdb/src/storage/compression/bitpacking.cpp

@@ -6,7 +6,7 @@
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/operator/subtract.hpp"
@@ -455,8 +455,8 @@ public:
 			state->current_segment->count += count;
 
 			if (WRITE_STATISTICS && !state->state.all_invalid) {
-				NumericStatistics::Update<T>(state->current_segment->stats, state->state.minimum);
-				NumericStatistics::Update<T>(state->current_segment->stats, state->state.maximum);
+				NumericStats::Update<T>(state->current_segment->stats.statistics, state->state.minimum);
+				NumericStats::Update<T>(state->current_segment->stats.statistics, state->state.maximum);
 			}
 		}
 	};

package/src/duckdb/src/storage/compression/fixed_size_uncompressed.cpp

@@ -5,7 +5,7 @@
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/storage/checkpoint/write_overflow_strings_to_disk.hpp"
 #include "duckdb/storage/segment/uncompressed.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
@@ -192,7 +192,7 @@ struct StandardFixedSizeAppend {
 				auto target_idx = target_offset + i;
 				bool is_null = !adata.validity.RowIsValid(source_idx);
 				if (!is_null) {
-					NumericStatistics::Update<T>(stats, sdata[source_idx]);
+					NumericStats::Update<T>(stats.statistics, sdata[source_idx]);
 					tdata[target_idx] = sdata[source_idx];
 				} else {
 					// we insert a NullValue<T> in the null gap for debuggability
@@ -204,7 +204,7 @@ struct StandardFixedSizeAppend {
 			for (idx_t i = 0; i < count; i++) {
 				auto source_idx = adata.sel->get_index(offset + i);
 				auto target_idx = target_offset + i;
-				NumericStatistics::Update<T>(stats, sdata[source_idx]);
+				NumericStats::Update<T>(stats.statistics, sdata[source_idx]);
 				tdata[target_idx] = sdata[source_idx];
 			}
 		}

package/src/duckdb/src/storage/compression/numeric_constant.cpp

@@ -1,8 +1,7 @@
 #include "duckdb/function/compression/compression.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/common/types/vector.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
-#include "duckdb/storage/statistics/validity_statistics.hpp"
+
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/storage/segment/uncompressed.hpp"
@@ -20,8 +19,8 @@ unique_ptr<SegmentScanState> ConstantInitScan(ColumnSegment &segment) {
 // Scan Partial
 //===--------------------------------------------------------------------===//
 void ConstantFillFunctionValidity(ColumnSegment &segment, Vector &result, idx_t start_idx, idx_t count) {
-	auto &validity = (ValidityStatistics &)*segment.stats.statistics;
-	if (validity.has_null) {
+	auto &stats = segment.stats.statistics;
+	if (stats.CanHaveNull()) {
 		auto &mask = FlatVector::Validity(result);
 		for (idx_t i = 0; i < count; i++) {
 			mask.SetInvalid(start_idx + i);
@@ -31,10 +30,10 @@ void ConstantFillFunctionValidity(ColumnSegment &segment, Vector &result, idx_t
 
 template <class T>
 void ConstantFillFunction(ColumnSegment &segment, Vector &result, idx_t start_idx, idx_t count) {
-	auto &nstats = (NumericStatistics &)*segment.stats.statistics;
+	auto &nstats = segment.stats.statistics;
 
 	auto data = FlatVector::GetData<T>(result);
-	auto constant_value = nstats.min.GetValueUnsafe<T>();
+	auto constant_value = NumericStats::GetMinUnsafe<T>(nstats);
 	for (idx_t i = 0; i < count; i++) {
 		data[start_idx + i] = constant_value;
 	}
@@ -55,8 +54,8 @@ void ConstantScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t s
 // Scan base data
 //===--------------------------------------------------------------------===//
 void ConstantScanFunctionValidity(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result) {
-	auto &validity = (ValidityStatistics &)*segment.stats.statistics;
-	if (validity.has_null) {
+	auto &stats = segment.stats.statistics;
+	if (stats.CanHaveNull()) {
 		if (result.GetVectorType() == VectorType::CONSTANT_VECTOR) {
 			result.SetVectorType(VectorType::CONSTANT_VECTOR);
 			ConstantVector::SetNull(result, true);
@@ -69,10 +68,10 @@ void ConstantScanFunctionValidity(ColumnSegment &segment, ColumnScanState &state
 
 template <class T>
 void ConstantScanFunction(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result) {
-	auto &nstats = (NumericStatistics &)*segment.stats.statistics;
+	auto &nstats = segment.stats.statistics;
 
 	auto data = FlatVector::GetData<T>(result);
-	data[0] = nstats.min.GetValueUnsafe<T>();
+	data[0] = NumericStats::GetMinUnsafe<T>(nstats);
 	result.SetVectorType(VectorType::CONSTANT_VECTOR);
 }
 

package/src/duckdb/src/storage/compression/patas.cpp

@@ -10,7 +10,7 @@
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/operator/subtract.hpp"

package/src/duckdb/src/storage/compression/rle.cpp

@@ -1,5 +1,5 @@
 #include "duckdb/function/compression/compression.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/config.hpp"
@@ -175,7 +175,7 @@ struct RLECompressState : public CompressionState {
 
 		// update meta data
 		if (WRITE_STATISTICS && !is_null) {
-			NumericStatistics::Update<T>(current_segment->stats, value);
+			NumericStats::Update<T>(current_segment->stats.statistics, value);
 		}
 		current_segment->count += count;
 

package/src/duckdb/src/storage/compression/validity_uncompressed.cpp

@@ -2,7 +2,7 @@
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/storage/table/append_state.hpp"
-#include "duckdb/storage/statistics/validity_statistics.hpp"
+
 #include "duckdb/common/types/null_value.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/function/compression_function.hpp"
@@ -410,14 +410,14 @@ unique_ptr<CompressedSegmentState> ValidityInitSegment(ColumnSegment &segment, b
 idx_t ValidityAppend(CompressionAppendState &append_state, ColumnSegment &segment, SegmentStatistics &stats,
                      UnifiedVectorFormat &data, idx_t offset, idx_t vcount) {
 	D_ASSERT(segment.GetBlockOffset() == 0);
-	auto &validity_stats = (ValidityStatistics &)*stats.statistics;
+	auto &validity_stats = stats.statistics;
 
 	auto max_tuples = segment.SegmentSize() / ValidityMask::STANDARD_MASK_SIZE * STANDARD_VECTOR_SIZE;
 	idx_t append_count = MinValue<idx_t>(vcount, max_tuples - segment.count);
 	if (data.validity.AllValid()) {
 		// no null values: skip append
 		segment.count += append_count;
-		validity_stats.has_no_null = true;
+		validity_stats.SetHasNoNull();
 		return append_count;
 	}
 
@@ -426,9 +426,9 @@ idx_t ValidityAppend(CompressionAppendState &append_state, ColumnSegment &segmen
 		auto idx = data.sel->get_index(offset + i);
 		if (!data.validity.RowIsValidUnsafe(idx)) {
 			mask.SetInvalidUnsafe(segment.count + i);
-			validity_stats.has_null = true;
+			validity_stats.SetHasNull();
 		} else {
-			validity_stats.has_no_null = true;
+			validity_stats.SetHasNoNull();
 		}
 	}
 	segment.count += append_count;

package/src/duckdb/src/storage/data_table.cpp

@@ -1223,9 +1223,9 @@ unique_ptr<BaseStatistics> DataTable::GetStatistics(ClientContext &context, colu
 	return row_groups->CopyStats(column_id);
 }
 
-void DataTable::SetStatistics(column_t column_id, const std::function<void(BaseStatistics &)> &set_fun) {
+void DataTable::SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats) {
 	D_ASSERT(column_id != COLUMN_IDENTIFIER_ROW_ID);
-	row_groups->SetStatistics(column_id, set_fun);
+	row_groups->SetDistinct(column_id, std::move(distinct_stats));
 }
 
 //===--------------------------------------------------------------------===//
@@ -1234,10 +1234,8 @@ void DataTable::SetStatistics(column_t column_id, const std::function<void(BaseS
 void DataTable::Checkpoint(TableDataWriter &writer) {
 	// checkpoint each individual row group
 	// FIXME: we might want to combine adjacent row groups in case they have had deletions...
-	vector<unique_ptr<BaseStatistics>> global_stats;
-	for (idx_t i = 0; i < column_definitions.size(); i++) {
-		global_stats.push_back(row_groups->CopyStats(i));
-	}
+	TableStatistics global_stats;
+	row_groups->CopyStats(global_stats);
 
 	row_groups->Checkpoint(writer, global_stats);