duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -1,20 +1,21 @@
 #include "duckdb/storage/table/row_group_collection.hpp"
-#include "duckdb/storage/table/persistent_table_data.hpp"
+
+#include "duckdb/common/serializer/binary_deserializer.hpp"
 #include "duckdb/execution/expression_executor.hpp"
+#include "duckdb/execution/index/bound_index.hpp"
+#include "duckdb/execution/task_error_manager.hpp"
 #include "duckdb/main/client_context.hpp"
-#include "duckdb/storage/data_table.hpp"
+#include "duckdb/parallel/task_executor.hpp"
 #include "duckdb/planner/constraints/bound_not_null_constraint.hpp"
 #include "duckdb/storage/checkpoint/table_data_writer.hpp"
-#include "duckdb/storage/table/row_group_segment_tree.hpp"
+#include "duckdb/storage/data_table.hpp"
 #include "duckdb/storage/metadata/metadata_reader.hpp"
 #include "duckdb/storage/table/append_state.hpp"
+#include "duckdb/storage/table/column_checkpoint_state.hpp"
+#include "duckdb/storage/table/persistent_table_data.hpp"
+#include "duckdb/storage/table/row_group_segment_tree.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/table_storage_info.hpp"
-#include "duckdb/common/serializer/binary_deserializer.hpp"
-#include "duckdb/parallel/task_scheduler.hpp"
-#include "duckdb/execution/task_error_manager.hpp"
-#include "duckdb/storage/table/column_checkpoint_state.hpp"
-#include "duckdb/execution/index/bound_index.hpp"
 
 namespace duckdb {
 
@@ -90,6 +91,17 @@ void RowGroupCollection::Initialize(PersistentTableData &data) {
 	stats.Initialize(types, data);
 }
 
+void RowGroupCollection::Initialize(PersistentCollectionData &data) {
+	stats.InitializeEmpty(types);
+	auto l = row_groups->Lock();
+	for (auto &row_group_data : data.row_group_data) {
+		auto row_group = make_uniq<RowGroup>(*this, row_group_data);
+		row_group->MergeIntoStatistics(stats);
+		total_rows += row_group->count;
+		row_groups->AppendSegment(l, std::move(row_group));
+	}
+}
+
 void RowGroupCollection::InitializeEmpty() {
 	stats.InitializeEmpty(types);
 }
@@ -321,6 +333,10 @@ void RowGroupCollection::InitializeAppend(TransactionData transaction, TableAppe
 	D_ASSERT(this->row_start + total_rows == state.start_row_group->start + state.start_row_group->count);
 	state.start_row_group->InitializeAppend(state.row_group_append_state);
 	state.transaction = transaction;
+
+	// initialize thread-local stats so we have less lock contention when updating distinct statistics
+	state.stats = TableStatistics();
+	state.stats.InitializeEmpty(types);
 }
 
 void RowGroupCollection::InitializeAppend(TableAppendState &state) {
@@ -346,10 +362,7 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
 			current_row_group->Append(state.row_group_append_state, chunk, append_count);
 			allocation_size += current_row_group->GetAllocationSize() - previous_allocation_size;
 			// merge the stats
-			auto stats_lock = stats.GetLock();
-			for (idx_t i = 0; i < types.size(); i++) {
-				current_row_group->MergeIntoStatistics(i, stats.GetStats(*stats_lock, i).Statistics());
-			}
+			current_row_group->MergeIntoStatistics(stats);
 		}
 		remaining -= append_count;
 		if (remaining > 0) {
@@ -375,9 +388,9 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
 		}
 	}
 	state.current_row += row_t(total_append_count);
-	auto stats_lock = stats.GetLock();
+	auto local_stats_lock = state.stats.GetLock();
 	for (idx_t col_idx = 0; col_idx < types.size(); col_idx++) {
-		stats.GetStats(*stats_lock, col_idx).UpdateDistinctStatistics(chunk.data[col_idx], chunk.size());
+		state.stats.GetStats(*local_stats_lock, col_idx).UpdateDistinctStatistics(chunk.data[col_idx], chunk.size());
 	}
 	return new_row_group;
 }
@@ -396,6 +409,20 @@ void RowGroupCollection::FinalizeAppend(TransactionData transaction, TableAppend
 	state.total_append_count = 0;
 	state.start_row_group = nullptr;
 
+	auto global_stats_lock = stats.GetLock();
+	auto local_stats_lock = state.stats.GetLock();
+	for (idx_t col_idx = 0; col_idx < types.size(); col_idx++) {
+		auto &global_stats = stats.GetStats(*global_stats_lock, col_idx);
+		if (!global_stats.HasDistinctStats()) {
+			continue;
+		}
+		auto &local_stats = state.stats.GetStats(*local_stats_lock, col_idx);
+		if (!local_stats.HasDistinctStats()) {
+			continue;
+		}
+		global_stats.DistinctStats().Merge(local_stats.DistinctStats());
+	}
+
 	Verify();
 }
 
@@ -443,16 +470,75 @@ void RowGroupCollection::RevertAppendInternal(idx_t start_row) {
 	segment.RevertAppend(start_row);
 }
 
-void RowGroupCollection::MergeStorage(RowGroupCollection &data) {
+void RowGroupCollection::CleanupAppend(transaction_t lowest_transaction, idx_t start, idx_t count) {
+	auto row_group = row_groups->GetSegment(start);
+	D_ASSERT(row_group);
+	idx_t current_row = start;
+	idx_t remaining = count;
+	while (true) {
+		idx_t start_in_row_group = current_row - row_group->start;
+		idx_t append_count = MinValue<idx_t>(row_group->count - start_in_row_group, remaining);
+
+		row_group->CleanupAppend(lowest_transaction, start_in_row_group, append_count);
+
+		current_row += append_count;
+		remaining -= append_count;
+		if (remaining == 0) {
+			break;
+		}
+		row_group = row_groups->GetNextSegment(row_group);
+	}
+}
+
+bool RowGroupCollection::IsPersistent() const {
+	for (auto &row_group : row_groups->Segments()) {
+		if (!row_group.IsPersistent()) {
+			return false;
+		}
+	}
+	return true;
+}
+
+void RowGroupCollection::MergeStorage(RowGroupCollection &data, optional_ptr<DataTable> table,
+                                      optional_ptr<StorageCommitState> commit_state) {
 	D_ASSERT(data.types == types);
-	auto index = row_start + total_rows.load();
+	auto start_index = row_start + total_rows.load();
+	auto index = start_index;
 	auto segments = data.row_groups->MoveSegments();
+
+	// check if the row groups we are merging are optimistically written
+	// if all row groups are optimistically written we keep around the block pointers
+	unique_ptr<PersistentCollectionData> row_group_data;
+	idx_t optimistically_written_count = 0;
+	if (commit_state) {
+		for (auto &entry : segments) {
+			auto &row_group = *entry.node;
+			if (!row_group.IsPersistent()) {
+				break;
+			}
+			optimistically_written_count += row_group.count;
+		}
+		if (optimistically_written_count > 0) {
+			row_group_data = make_uniq<PersistentCollectionData>();
+		}
+	}
 	for (auto &entry : segments) {
 		auto &row_group = entry.node;
 		row_group->MoveToCollection(*this, index);
+
+		if (commit_state && (index - start_index) < optimistically_written_count) {
+			// serialize the block pointers of this row group
+			auto persistent_data = row_group->SerializeRowGroupInfo();
+			persistent_data.types = types;
+			row_group_data->row_group_data.push_back(std::move(persistent_data));
+		}
 		index += row_group->count;
 		row_groups->AppendSegment(std::move(row_group));
 	}
+	if (commit_state && optimistically_written_count > 0) {
+		// if we have serialized the row groups - push the serialized block pointers into the commit state
+		commit_state->AddRowGroupData(*table, start_index, optimistically_written_count, std::move(row_group_data));
+	}
 	stats.MergeStats(data.stats);
 	total_rows += data.total_rows.load();
 }
@@ -609,104 +695,26 @@ void RowGroupCollection::UpdateColumn(TransactionData transaction, Vector &row_i
 struct CollectionCheckpointState {
 	CollectionCheckpointState(RowGroupCollection &collection, TableDataWriter &writer,
 	                          vector<SegmentNode<RowGroup>> &segments, TableStatistics &global_stats)
-	    : collection(collection), writer(writer), scheduler(writer.GetScheduler()), segments(segments),
-	      global_stats(global_stats), token(scheduler.CreateProducer()), completed_tasks(0), total_tasks(0) {
+	    : collection(collection), writer(writer), executor(writer.GetScheduler()), segments(segments),
+	      global_stats(global_stats) {
 		writers.resize(segments.size());
 		write_data.resize(segments.size());
 	}
 
 	RowGroupCollection &collection;
 	TableDataWriter &writer;
-	TaskScheduler &scheduler;
+	TaskExecutor executor;
 	vector<SegmentNode<RowGroup>> &segments;
 	vector<unique_ptr<RowGroupWriter>> writers;
 	vector<RowGroupWriteData> write_data;
 	TableStatistics &global_stats;
 	mutex write_lock;
-
-public:
-	void PushError(ErrorData error) {
-		error_manager.PushError(std::move(error));
-	}
-	bool HasError() {
-		return error_manager.HasError();
-	}
-	void ThrowError() {
-		error_manager.ThrowException();
-	}
-
-	void ScheduleTask(unique_ptr<Task> task) {
-		++total_tasks;
-		scheduler.ScheduleTask(*token, std::move(task));
-	}
-	void FinishTask() {
-		++completed_tasks;
-	}
-	bool TasksFinished() {
-		if (completed_tasks == total_tasks) {
-			return true;
-		}
-		if (HasError()) {
-			return true;
-		}
-		return false;
-	}
-	void CancelTasks() {
-		// This should only be called after an error has occurred, no other mechanism to cancel checkpoint tasks exists
-		// currently
-		D_ASSERT(error_manager.HasError());
-		// Give every pending task the chance to cancel
-		WorkOnTasks();
-		// Wait for all active tasks to realize they have been canceled
-		while (completed_tasks != total_tasks) {
-		}
-	}
-
-	void WorkOnTasks() {
-		shared_ptr<Task> task_from_producer;
-		while (scheduler.GetTaskFromProducer(*token, task_from_producer)) {
-			auto res = task_from_producer->Execute(TaskExecutionMode::PROCESS_ALL);
-			(void)res;
-			D_ASSERT(res != TaskExecutionResult::TASK_BLOCKED);
-			task_from_producer.reset();
-		}
-	}
-
-	bool GetTask(shared_ptr<Task> &task) {
-		return scheduler.GetTaskFromProducer(*token, task);
-	}
-
-private:
-	TaskErrorManager error_manager;
-	unique_ptr<ProducerToken> token;
-	atomic<idx_t> completed_tasks;
-	atomic<idx_t> total_tasks;
 };
 
-class BaseCheckpointTask : public Task {
+class BaseCheckpointTask : public BaseExecutorTask {
 public:
-	explicit BaseCheckpointTask(CollectionCheckpointState &checkpoint_state) : checkpoint_state(checkpoint_state) {
-	}
-
-	virtual void ExecuteTask() = 0;
-	TaskExecutionResult Execute(TaskExecutionMode mode) override {
-		(void)mode;
-		D_ASSERT(mode == TaskExecutionMode::PROCESS_ALL);
-		if (checkpoint_state.HasError()) {
-			checkpoint_state.FinishTask();
-			return TaskExecutionResult::TASK_FINISHED;
-		}
-		try {
-			ExecuteTask();
-			checkpoint_state.FinishTask();
-			return TaskExecutionResult::TASK_FINISHED;
-		} catch (std::exception &ex) {
-			checkpoint_state.PushError(ErrorData(ex));
-		} catch (...) { // LCOV_EXCL_START
-			checkpoint_state.PushError(ErrorData("Unknown exception during Checkpoint!"));
-		} // LCOV_EXCL_STOP
-		checkpoint_state.FinishTask();
-		return TaskExecutionResult::TASK_ERROR;
+	explicit BaseCheckpointTask(CollectionCheckpointState &checkpoint_state)
+	    : BaseExecutorTask(checkpoint_state.executor), checkpoint_state(checkpoint_state) {
 	}
 
 protected:
@@ -804,6 +812,7 @@ public:
 				if (scan_chunk.size() == 0) {
 					break;
 				}
+				scan_chunk.Flatten();
 				idx_t remaining = scan_chunk.size();
 				while (remaining > 0) {
 					idx_t append_count =
@@ -894,7 +903,7 @@ bool RowGroupCollection::ScheduleVacuumTasks(CollectionCheckpointState &checkpoi
 		return false;
 	}
 	idx_t merge_rows;
-	idx_t next_idx;
+	idx_t next_idx = 0;
 	idx_t merge_count;
 	idx_t target_count;
 	bool perform_merge = false;
@@ -933,7 +942,7 @@ bool RowGroupCollection::ScheduleVacuumTasks(CollectionCheckpointState &checkpoi
 	// schedule the vacuum task
 	auto vacuum_task = make_uniq<VacuumTask>(checkpoint_state, state, segment_idx, merge_count, target_count,
 	                                         merge_rows, state.row_start);
-	checkpoint_state.ScheduleTask(std::move(vacuum_task));
+	checkpoint_state.executor.ScheduleTask(std::move(vacuum_task));
 	// skip vacuuming by the row groups we have merged
 	state.next_vacuum_idx = next_idx;
 	state.row_start += merge_rows;
@@ -945,7 +954,7 @@ bool RowGroupCollection::ScheduleVacuumTasks(CollectionCheckpointState &checkpoi
 //===--------------------------------------------------------------------===//
 void RowGroupCollection::ScheduleCheckpointTask(CollectionCheckpointState &checkpoint_state, idx_t segment_idx) {
 	auto checkpoint_task = make_uniq<CheckpointTask>(checkpoint_state, segment_idx);
-	checkpoint_state.ScheduleTask(std::move(checkpoint_task));
+	checkpoint_state.executor.ScheduleTask(std::move(checkpoint_task));
 }
 
 void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &global_stats) {
@@ -957,12 +966,17 @@ void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &gl
 	VacuumState vacuum_state;
 	InitializeVacuumState(checkpoint_state, vacuum_state, segments);
 	// schedule tasks
+	idx_t total_vacuum_tasks = 0;
+	auto &config = DBConfig::GetConfig(writer.GetDatabase());
 	for (idx_t segment_idx = 0; segment_idx < segments.size(); segment_idx++) {
 		auto &entry = segments[segment_idx];
-		auto vacuum_tasks = ScheduleVacuumTasks(checkpoint_state, vacuum_state, segment_idx);
-		if (vacuum_tasks) {
-			// vacuum tasks were scheduled - don't schedule a checkpoint task yet
-			continue;
+		if (total_vacuum_tasks < config.options.max_vacuum_tasks) {
+			auto vacuum_tasks = ScheduleVacuumTasks(checkpoint_state, vacuum_state, segment_idx);
+			if (vacuum_tasks) {
+				// vacuum tasks were scheduled - don't schedule a checkpoint task yet
+				total_vacuum_tasks++;
+				continue;
+			}
 		}
 		if (!entry.node) {
 			// row group was vacuumed/dropped - skip
@@ -974,19 +988,7 @@ void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &gl
 		vacuum_state.row_start += entry.node->count;
 	}
 	// all tasks have been scheduled - execute tasks until we are done
-	do {
-		shared_ptr<Task> task;
-		while (checkpoint_state.GetTask(task)) {
-			task->Execute(TaskExecutionMode::PROCESS_ALL);
-			task.reset();
-		}
-	} while (!checkpoint_state.TasksFinished());
-	// check if we ran into any errors while checkpointing
-	if (checkpoint_state.HasError()) {
-		// throw the error
-		checkpoint_state.CancelTasks();
-		checkpoint_state.ThrowError();
-	}
+	checkpoint_state.executor.WorkOnTasks();
 
 	// no errors - finalize the row groups
 	idx_t new_total_rows = 0;

package/src/duckdb/src/storage/table/row_version_manager.cpp

@@ -7,7 +7,7 @@
 
 namespace duckdb {
 
-RowVersionManager::RowVersionManager(idx_t start) : start(start), has_changes(false) {
+RowVersionManager::RowVersionManager(idx_t start) noexcept : start(start), has_changes(false) {
 }
 
 void RowVersionManager::SetStart(idx_t new_start) {
@@ -127,6 +127,32 @@ void RowVersionManager::CommitAppend(transaction_t commit_id, idx_t row_group_st
 	}
 }
 
+void RowVersionManager::CleanupAppend(transaction_t lowest_active_transaction, idx_t row_group_start, idx_t count) {
+	if (count == 0) {
+		return;
+	}
+	idx_t row_group_end = row_group_start + count;
+
+	lock_guard<mutex> lock(version_lock);
+	idx_t start_vector_idx = row_group_start / STANDARD_VECTOR_SIZE;
+	idx_t end_vector_idx = (row_group_end - 1) / STANDARD_VECTOR_SIZE;
+	for (idx_t vector_idx = start_vector_idx; vector_idx <= end_vector_idx; vector_idx++) {
+		idx_t vcount =
+		    vector_idx == end_vector_idx ? row_group_end - end_vector_idx * STANDARD_VECTOR_SIZE : STANDARD_VECTOR_SIZE;
+		auto &info = *vector_info[vector_idx];
+		if (vcount != STANDARD_VECTOR_SIZE) {
+			// not written fully - skip
+			continue;
+		}
+		// if we wrote the entire chunk info try to compress it
+		unique_ptr<ChunkInfo> new_info;
+		auto cleanup = info.Cleanup(lowest_active_transaction, new_info);
+		if (cleanup) {
+			vector_info[vector_idx] = std::move(new_info);
+		}
+	}
+}
+
 void RowVersionManager::RevertAppend(idx_t start_row) {
 	lock_guard<mutex> lock(version_lock);
 	idx_t start_vector_idx = (start_row + (STANDARD_VECTOR_SIZE - 1)) / STANDARD_VECTOR_SIZE;

package/src/duckdb/src/storage/table/scan_state.cpp

@@ -1,19 +1,25 @@
 #include "duckdb/storage/table/scan_state.hpp"
-#include "duckdb/storage/table/row_group.hpp"
-#include "duckdb/storage/table/column_segment.hpp"
-#include "duckdb/transaction/duck_transaction.hpp"
+
+#include "duckdb/execution/adaptive_filter.hpp"
 #include "duckdb/storage/table/column_data.hpp"
+#include "duckdb/storage/table/column_segment.hpp"
+#include "duckdb/storage/table/row_group.hpp"
 #include "duckdb/storage/table/row_group_collection.hpp"
 #include "duckdb/storage/table/row_group_segment_tree.hpp"
+#include "duckdb/transaction/duck_transaction.hpp"
 
 namespace duckdb {
 
-void TableScanState::Initialize(vector<column_t> column_ids, TableFilterSet *table_filters) {
-	this->column_ids = std::move(column_ids);
-	this->table_filters = table_filters;
+TableScanState::TableScanState() : table_state(*this), local_state(*this) {
+}
+
+TableScanState::~TableScanState() {
+}
+
+void TableScanState::Initialize(vector<column_t> column_ids_p, optional_ptr<TableFilterSet> table_filters) {
+	this->column_ids = std::move(column_ids_p);
 	if (table_filters) {
-		D_ASSERT(table_filters->filters.size() > 0);
-		this->adaptive_filter = make_uniq<AdaptiveFilter>(table_filters);
+		filters.Initialize(*table_filters, column_ids);
 	}
 }
 
@@ -22,15 +28,87 @@ const vector<column_t> &TableScanState::GetColumnIds() {
 	return column_ids;
 }
 
-TableFilterSet *TableScanState::GetFilters() {
-	D_ASSERT(!table_filters || adaptive_filter.get());
-	return table_filters;
+ScanFilterInfo::~ScanFilterInfo() {
+}
+
+ScanFilterInfo &TableScanState::GetFilterInfo() {
+	return filters;
+}
+
+ScanFilter::ScanFilter(idx_t index, const vector<column_t> &column_ids, TableFilter &filter)
+    : scan_column_index(index), table_column_index(column_ids[index]), filter(filter), always_true(false) {
+}
+
+void ScanFilterInfo::Initialize(TableFilterSet &filters, const vector<column_t> &column_ids) {
+	D_ASSERT(!filters.filters.empty());
+	table_filters = &filters;
+	adaptive_filter = make_uniq<AdaptiveFilter>(filters);
+	filter_list.reserve(filters.filters.size());
+	for (auto &entry : filters.filters) {
+		filter_list.emplace_back(entry.first, column_ids, *entry.second);
+	}
+	column_has_filter.reserve(column_ids.size());
+	for (idx_t col_idx = 0; col_idx < column_ids.size(); col_idx++) {
+		bool has_filter = table_filters->filters.find(col_idx) != table_filters->filters.end();
+		column_has_filter.push_back(has_filter);
+	}
+	base_column_has_filter = column_has_filter;
+}
+
+bool ScanFilterInfo::ColumnHasFilters(idx_t column_idx) {
+	if (column_idx < column_has_filter.size()) {
+		return column_has_filter[column_idx];
+	} else {
+		return false;
+	}
 }
 
-AdaptiveFilter *TableScanState::GetAdaptiveFilter() {
+bool ScanFilterInfo::HasFilters() const {
+	if (!table_filters) {
+		// no filters
+		return false;
+	}
+	// if we have filters - check if we need to check any of them
+	return always_true_filters < filter_list.size();
+}
+
+void ScanFilterInfo::CheckAllFilters() {
+	always_true_filters = 0;
+	// reset the "column_has_filter" bitmask to the original
+	for (idx_t col_idx = 0; col_idx < column_has_filter.size(); col_idx++) {
+		column_has_filter[col_idx] = base_column_has_filter[col_idx];
+	}
+	// set "always_true" in the individual filters to false
+	for (auto &filter : filter_list) {
+		filter.always_true = false;
+	}
+}
+
+void ScanFilterInfo::SetFilterAlwaysTrue(idx_t filter_idx) {
+	auto &filter = filter_list[filter_idx];
+	filter.always_true = true;
+	column_has_filter[filter.scan_column_index] = false;
+	always_true_filters++;
+}
+
+optional_ptr<AdaptiveFilter> ScanFilterInfo::GetAdaptiveFilter() {
 	return adaptive_filter.get();
 }
 
+AdaptiveFilterState ScanFilterInfo::BeginFilter() const {
+	if (!adaptive_filter) {
+		return AdaptiveFilterState();
+	}
+	return adaptive_filter->BeginFilter();
+}
+
+void ScanFilterInfo::EndFilter(AdaptiveFilterState state) {
+	if (!adaptive_filter) {
+		return;
+	}
+	adaptive_filter->EndFilter(state);
+}
+
 void ColumnScanState::NextInternal(idx_t count) {
 	if (!current) {
 		//! There is no column segment
@@ -59,12 +137,10 @@ const vector<storage_t> &CollectionScanState::GetColumnIds() {
 	return parent.GetColumnIds();
 }
 
-TableFilterSet *CollectionScanState::GetFilters() {
-	return parent.GetFilters();
-}
+TableFilterSet &GetFilters();
 
-AdaptiveFilter *CollectionScanState::GetAdaptiveFilter() {
-	return parent.GetAdaptiveFilter();
+ScanFilterInfo &CollectionScanState::GetFilterInfo() {
+	return parent.GetFilterInfo();
 }
 
 TableScanOptions &CollectionScanState::GetOptions() {
@@ -77,7 +153,7 @@ ParallelCollectionScanState::ParallelCollectionScanState()
 
 CollectionScanState::CollectionScanState(TableScanState &parent_p)
     : row_group(nullptr), vector_index(0), max_row_group_row(0), row_groups(nullptr), max_row(0), batch_index(0),
-      parent(parent_p) {
+      valid_sel(STANDARD_VECTOR_SIZE), parent(parent_p) {
 }
 
 bool CollectionScanState::Scan(DuckTransaction &transaction, DataChunk &result) {
@@ -138,4 +214,11 @@ bool CollectionScanState::ScanCommitted(DataChunk &result, TableScanType type) {
 	return false;
 }
 
+PrefetchState::~PrefetchState() {
+}
+
+void PrefetchState::AddBlock(shared_ptr<BlockHandle> block) {
+	blocks.push_back(std::move(block));
+}
+
 } // namespace duckdb

package/src/duckdb/src/storage/table/standard_column_data.cpp

@@ -33,31 +33,9 @@ ScanVectorType StandardColumnData::GetVectorScanType(ColumnScanState &state, idx
 	return validity.GetVectorScanType(state.child_states[0], scan_count);
 }
 
-bool StandardColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
-	if (!state.segment_checked) {
-		if (!state.current) {
-			return true;
-		}
-		state.segment_checked = true;
-		FilterPropagateResult prune_result;
-		{
-			lock_guard<mutex> l(stats_lock);
-			prune_result = filter.CheckStatistics(state.current->stats.statistics);
-			if (prune_result != FilterPropagateResult::FILTER_ALWAYS_FALSE) {
-				return true;
-			}
-		}
-		lock_guard<mutex> l(update_lock);
-		if (updates) {
-			auto update_stats = updates->GetStatistics();
-			prune_result = filter.CheckStatistics(*update_stats);
-			return prune_result != FilterPropagateResult::FILTER_ALWAYS_FALSE;
-		} else {
-			return false;
-		}
-	} else {
-		return true;
-	}
+void StandardColumnData::InitializePrefetch(PrefetchState &prefetch_state, ColumnScanState &scan_state, idx_t rows) {
+	ColumnData::InitializePrefetch(prefetch_state, scan_state, rows);
+	validity.InitializePrefetch(prefetch_state, scan_state.child_states[0], rows);
 }
 
 void StandardColumnData::InitializeScan(ColumnScanState &state) {
@@ -100,7 +78,6 @@ idx_t StandardColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_
 
 void StandardColumnData::InitializeAppend(ColumnAppendState &state) {
 	ColumnData::InitializeAppend(state);
-
 	ColumnAppendState child_append;
 	validity.InitializeAppend(child_append);
 	state.child_appends.push_back(std::move(child_append));
@@ -192,10 +169,10 @@ public:
 		return std::move(global_stats);
 	}
 
-	void WriteDataPointers(RowGroupWriter &writer, Serializer &serializer) override {
-		ColumnCheckpointState::WriteDataPointers(writer, serializer);
-		serializer.WriteObject(101, "validity",
-		                       [&](Serializer &serializer) { validity_state->WriteDataPointers(writer, serializer); });
+	PersistentColumnData ToPersistentData() override {
+		auto data = ColumnCheckpointState::ToPersistentData();
+		data.child_columns.push_back(validity_state->ToPersistentData());
+		return data;
 	}
 };
 
@@ -226,10 +203,19 @@ void StandardColumnData::CheckpointScan(ColumnSegment &segment, ColumnScanState
 	validity.ScanCommittedRange(row_group_start, offset_in_row_group, count, scan_vector);
 }
 
-void StandardColumnData::DeserializeColumn(Deserializer &deserializer, BaseStatistics &target_stats) {
-	ColumnData::DeserializeColumn(deserializer, target_stats);
-	deserializer.ReadObject(
-	    101, "validity", [&](Deserializer &deserializer) { validity.DeserializeColumn(deserializer, target_stats); });
+bool StandardColumnData::IsPersistent() {
+	return ColumnData::IsPersistent() && validity.IsPersistent();
+}
+
+PersistentColumnData StandardColumnData::Serialize() {
+	auto persistent_data = ColumnData::Serialize();
+	persistent_data.child_columns.push_back(validity.Serialize());
+	return persistent_data;
+}
+
+void StandardColumnData::InitializeColumn(PersistentColumnData &column_data, BaseStatistics &target_stats) {
+	ColumnData::InitializeColumn(column_data, target_stats);
+	validity.InitializeColumn(column_data.child_columns[0], target_stats);
 }
 
 void StandardColumnData::GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,