duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -11,6 +11,9 @@
 #include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/table_storage_info.hpp"
 #include "duckdb/common/serializer/binary_deserializer.hpp"
+#include "duckdb/parallel/task_scheduler.hpp"
+#include "duckdb/execution/task_error_manager.hpp"
+#include "duckdb/storage/table/column_checkpoint_state.hpp"
 
 namespace duckdb {
 
@@ -359,11 +362,7 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
 			D_ASSERT(chunk.size() == remaining + append_count);
 			// slice the input chunk
 			if (remaining < chunk.size()) {
-				SelectionVector sel(remaining);
-				for (idx_t i = 0; i < remaining; i++) {
-					sel.set_index(i, append_count + i);
-				}
-				chunk.Slice(sel, remaining);
+				chunk.Slice(append_count, remaining);
 			}
 			// append a new row_group
 			new_row_group = true;
@@ -436,14 +435,13 @@ void RowGroupCollection::RevertAppendInternal(idx_t start_row) {
 	auto l = row_groups->Lock();
 	// find the segment index that the current row belongs to
 	idx_t segment_index = row_groups->GetSegmentIndex(l, start_row);
-	auto segment = row_groups->GetSegmentByIndex(l, segment_index);
-	auto &info = *segment;
+	auto &segment = *row_groups->GetSegmentByIndex(l, segment_index);
 
 	// remove any segments AFTER this segment: they should be deleted entirely
 	row_groups->EraseSegments(l, segment_index);
 
-	info.next = nullptr;
-	info.RevertAppend(start_row);
+	segment.next = nullptr;
+	segment.RevertAppend(start_row);
 }
 
 void RowGroupCollection::MergeStorage(RowGroupCollection &data) {
@@ -596,28 +594,405 @@ void RowGroupCollection::UpdateColumn(TransactionData transaction, Vector &row_i
 	row_group->MergeIntoStatistics(primary_column_idx, stats.GetStats(primary_column_idx).Statistics());
 }
 
+//===--------------------------------------------------------------------===//
+// Checkpoint State
+//===--------------------------------------------------------------------===//
+struct CollectionCheckpointState {
+	CollectionCheckpointState(RowGroupCollection &collection, TableDataWriter &writer,
+	                          vector<SegmentNode<RowGroup>> &segments, TableStatistics &global_stats)
+	    : collection(collection), writer(writer), scheduler(writer.GetScheduler()), segments(segments),
+	      global_stats(global_stats), token(scheduler.CreateProducer()), completed_tasks(0), total_tasks(0) {
+		writers.resize(segments.size());
+		write_data.resize(segments.size());
+	}
+
+	RowGroupCollection &collection;
+	TableDataWriter &writer;
+	TaskScheduler &scheduler;
+	vector<SegmentNode<RowGroup>> &segments;
+	vector<unique_ptr<RowGroupWriter>> writers;
+	vector<RowGroupWriteData> write_data;
+	TableStatistics &global_stats;
+	mutex write_lock;
+
+public:
+	void PushError(ErrorData error) {
+		error_manager.PushError(std::move(error));
+	}
+	bool HasError() {
+		return error_manager.HasError();
+	}
+	void ThrowError() {
+		error_manager.ThrowException();
+	}
+
+	void ScheduleTask(unique_ptr<Task> task) {
+		++total_tasks;
+		scheduler.ScheduleTask(*token, std::move(task));
+	}
+	void FinishTask() {
+		++completed_tasks;
+	}
+	bool TasksFinished() {
+		if (completed_tasks == total_tasks) {
+			return true;
+		}
+		if (HasError()) {
+			return true;
+		}
+		return false;
+	}
+	void CancelTasks() {
+		// This should only be called after an error has occurred, no other mechanism to cancel checkpoint tasks exists
+		// currently
+		D_ASSERT(error_manager.HasError());
+		// Give every pending task the chance to cancel
+		WorkOnTasks();
+		// Wait for all active tasks to realize they have been canceled
+		while (completed_tasks != total_tasks) {
+		}
+	}
+
+	void WorkOnTasks() {
+		shared_ptr<Task> task_from_producer;
+		while (scheduler.GetTaskFromProducer(*token, task_from_producer)) {
+			auto res = task_from_producer->Execute(TaskExecutionMode::PROCESS_ALL);
+			(void)res;
+			D_ASSERT(res != TaskExecutionResult::TASK_BLOCKED);
+			task_from_producer.reset();
+		}
+	}
+
+	bool GetTask(shared_ptr<Task> &task) {
+		return scheduler.GetTaskFromProducer(*token, task);
+	}
+
+private:
+	TaskErrorManager error_manager;
+	unique_ptr<ProducerToken> token;
+	atomic<idx_t> completed_tasks;
+	atomic<idx_t> total_tasks;
+};
+
+class BaseCheckpointTask : public Task {
+public:
+	explicit BaseCheckpointTask(CollectionCheckpointState &checkpoint_state) : checkpoint_state(checkpoint_state) {
+	}
+
+	virtual void ExecuteTask() = 0;
+	TaskExecutionResult Execute(TaskExecutionMode mode) override {
+		(void)mode;
+		D_ASSERT(mode == TaskExecutionMode::PROCESS_ALL);
+		if (checkpoint_state.HasError()) {
+			checkpoint_state.FinishTask();
+			return TaskExecutionResult::TASK_FINISHED;
+		}
+		try {
+			ExecuteTask();
+			checkpoint_state.FinishTask();
+			return TaskExecutionResult::TASK_FINISHED;
+		} catch (std::exception &ex) {
+			checkpoint_state.PushError(ErrorData(ex));
+		} catch (...) { // LCOV_EXCL_START
+			checkpoint_state.PushError(ErrorData("Unknown exception during Checkpoint!"));
+		} // LCOV_EXCL_STOP
+		checkpoint_state.FinishTask();
+		return TaskExecutionResult::TASK_ERROR;
+	}
+
+protected:
+	CollectionCheckpointState &checkpoint_state;
+};
+
+class CheckpointTask : public BaseCheckpointTask {
+public:
+	CheckpointTask(CollectionCheckpointState &checkpoint_state, idx_t index)
+	    : BaseCheckpointTask(checkpoint_state), index(index) {
+	}
+
+	void ExecuteTask() override {
+		auto &entry = checkpoint_state.segments[index];
+		auto &row_group = *entry.node;
+		checkpoint_state.writers[index] = checkpoint_state.writer.GetRowGroupWriter(*entry.node);
+		checkpoint_state.write_data[index] = row_group.WriteToDisk(*checkpoint_state.writers[index]);
+	}
+
+private:
+	idx_t index;
+};
+
+//===--------------------------------------------------------------------===//
+// Vacuum
+//===--------------------------------------------------------------------===//
+struct VacuumState {
+	bool can_vacuum_deletes = false;
+	idx_t row_start = 0;
+	idx_t next_vacuum_idx = 0;
+	vector<idx_t> row_group_counts;
+};
+
+class VacuumTask : public BaseCheckpointTask {
+public:
+	VacuumTask(CollectionCheckpointState &checkpoint_state, VacuumState &vacuum_state, idx_t segment_idx,
+	           idx_t merge_count, idx_t target_count, idx_t merge_rows, idx_t row_start)
+	    : BaseCheckpointTask(checkpoint_state), vacuum_state(vacuum_state), segment_idx(segment_idx),
+	      merge_count(merge_count), target_count(target_count), merge_rows(merge_rows), row_start(row_start) {
+	}
+
+	void ExecuteTask() override {
+		auto &collection = checkpoint_state.collection;
+		auto &types = collection.GetTypes();
+		// create the new set of target row groups (initially empty)
+		vector<unique_ptr<RowGroup>> new_row_groups;
+		vector<idx_t> append_counts;
+		idx_t row_group_rows = merge_rows;
+		idx_t start = row_start;
+		for (idx_t target_idx = 0; target_idx < target_count; target_idx++) {
+			idx_t current_row_group_rows = MinValue<idx_t>(row_group_rows, Storage::ROW_GROUP_SIZE);
+			auto new_row_group = make_uniq<RowGroup>(collection, start, current_row_group_rows);
+			new_row_group->InitializeEmpty(types);
+			new_row_groups.push_back(std::move(new_row_group));
+			append_counts.push_back(0);
+
+			row_group_rows -= current_row_group_rows;
+			start += current_row_group_rows;
+		}
+
+		DataChunk scan_chunk;
+		scan_chunk.Initialize(Allocator::DefaultAllocator(), types);
+
+		vector<column_t> column_ids;
+		for (idx_t c = 0; c < types.size(); c++) {
+			column_ids.push_back(c);
+		}
+
+		idx_t current_append_idx = 0;
+
+		// fill the new row group with the merged rows
+		TableAppendState append_state;
+		new_row_groups[current_append_idx]->InitializeAppend(append_state.row_group_append_state);
+
+		TableScanState scan_state;
+		scan_state.Initialize(column_ids);
+		scan_state.table_state.Initialize(types);
+		scan_state.table_state.max_row = idx_t(-1);
+		idx_t next_idx = segment_idx + merge_count;
+		for (idx_t c_idx = segment_idx; c_idx < next_idx; c_idx++) {
+			if (vacuum_state.row_group_counts[c_idx] == 0) {
+				continue;
+			}
+			auto &current_row_group = *checkpoint_state.segments[c_idx].node;
+
+			current_row_group.InitializeScan(scan_state.table_state);
+			while (true) {
+				scan_chunk.Reset();
+
+				current_row_group.ScanCommitted(scan_state.table_state, scan_chunk,
+				                                TableScanType::TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED);
+				if (scan_chunk.size() == 0) {
+					break;
+				}
+				idx_t remaining = scan_chunk.size();
+				while (remaining > 0) {
+					idx_t append_count =
+					    MinValue<idx_t>(remaining, Storage::ROW_GROUP_SIZE - append_counts[current_append_idx]);
+					new_row_groups[current_append_idx]->Append(append_state.row_group_append_state, scan_chunk,
+					                                           append_count);
+					append_counts[current_append_idx] += append_count;
+					remaining -= append_count;
+					const bool row_group_full = append_counts[current_append_idx] == Storage::ROW_GROUP_SIZE;
+					const bool last_row_group = current_append_idx + 1 >= new_row_groups.size();
+					if (remaining > 0 || (row_group_full && !last_row_group)) {
+						// move to the next row group
+						current_append_idx++;
+						new_row_groups[current_append_idx]->InitializeAppend(append_state.row_group_append_state);
+						// slice chunk for the next append
+						scan_chunk.Slice(append_count, remaining);
+					}
+				}
+			}
+			// drop the row group after merging
+			current_row_group.CommitDrop();
+			checkpoint_state.segments[c_idx].node.reset();
+		}
+		idx_t total_append_count = 0;
+		for (idx_t target_idx = 0; target_idx < target_count; target_idx++) {
+			auto &row_group = new_row_groups[target_idx];
+			row_group->Verify();
+
+			// assign the new row group to the current segment
+			checkpoint_state.segments[segment_idx + target_idx].node = std::move(row_group);
+			total_append_count += append_counts[target_idx];
+		}
+		if (total_append_count != merge_rows) {
+			throw InternalException("Mismatch in row group count vs verify count in RowGroupCollection::Checkpoint");
+		}
+		// merging is complete - schedule checkpoint tasks of the target row groups
+		for (idx_t i = 0; i < target_count; i++) {
+			collection.ScheduleCheckpointTask(checkpoint_state, segment_idx + i);
+		}
+	}
+
+private:
+	VacuumState &vacuum_state;
+	idx_t segment_idx;
+	idx_t merge_count;
+	idx_t target_count;
+	idx_t merge_rows;
+	idx_t row_start;
+};
+
+void RowGroupCollection::InitializeVacuumState(VacuumState &state, vector<SegmentNode<RowGroup>> &segments) {
+	state.can_vacuum_deletes = info->indexes.Empty();
+	if (!state.can_vacuum_deletes) {
+		return;
+	}
+	// obtain the set of committed row counts for each row group
+	state.row_group_counts.reserve(segments.size());
+	for (auto &entry : segments) {
+		auto &row_group = *entry.node;
+		auto row_group_count = row_group.GetCommittedRowCount();
+		if (row_group_count == 0) {
+			// empty row group - we can drop it entirely
+			row_group.CommitDrop();
+			entry.node.reset();
+		}
+		state.row_group_counts.push_back(row_group_count);
+	}
+}
+
+bool RowGroupCollection::ScheduleVacuumTasks(CollectionCheckpointState &checkpoint_state, VacuumState &state,
+                                             idx_t segment_idx) {
+	static constexpr const idx_t MAX_MERGE_COUNT = 3;
+
+	if (!state.can_vacuum_deletes) {
+		// we cannot vacuum deletes - cannot vacuum
+		return false;
+	}
+	if (segment_idx < state.next_vacuum_idx) {
+		// this segment is being vacuumed by a previously scheduled task
+		return true;
+	}
+	if (state.row_group_counts[segment_idx] == 0) {
+		// segment was already dropped - skip
+		D_ASSERT(!checkpoint_state.segments[segment_idx].node);
+		return false;
+	}
+	idx_t merge_rows;
+	idx_t next_idx;
+	idx_t merge_count;
+	idx_t target_count;
+	bool perform_merge = false;
+	// check if we can merge row groups adjacent to the current segment_idx
+	// we try merging row groups into batches of 1-3 row groups
+	// our goal is to reduce the amount of row groups
+	// hence we target_count should be less than merge_count for a marge to be worth it
+	// we greedily prefer to merge to the lowest target_count
+	// i.e. we prefer to merge 2 row groups into 1, than 3 row groups into 2
+	for (target_count = 1; target_count <= MAX_MERGE_COUNT; target_count++) {
+		auto total_target_size = target_count * Storage::ROW_GROUP_SIZE;
+		merge_count = 0;
+		merge_rows = 0;
+		for (next_idx = segment_idx; next_idx < checkpoint_state.segments.size(); next_idx++) {
+			if (state.row_group_counts[next_idx] == 0) {
+				continue;
+			}
+			if (merge_rows + state.row_group_counts[next_idx] > total_target_size) {
+				// does not fit
+				break;
+			}
+			// we can merge this row group together with the other row group
+			merge_rows += state.row_group_counts[next_idx];
+			merge_count++;
+		}
+		if (target_count < merge_count) {
+			// we can reduce "merge_count" row groups to "target_count"
+			// perform the merge at this level
+			perform_merge = true;
+			break;
+		}
+	}
+	if (!perform_merge) {
+		return false;
+	}
+	// schedule the vacuum task
+	auto vacuum_task = make_uniq<VacuumTask>(checkpoint_state, state, segment_idx, merge_count, target_count,
+	                                         merge_rows, state.row_start);
+	checkpoint_state.ScheduleTask(std::move(vacuum_task));
+	// skip vacuuming by the row groups we have merged
+	state.next_vacuum_idx = segment_idx + merge_count;
+	state.row_start += merge_rows;
+	return true;
+}
+
 //===--------------------------------------------------------------------===//
 // Checkpoint
 //===--------------------------------------------------------------------===//
+void RowGroupCollection::ScheduleCheckpointTask(CollectionCheckpointState &checkpoint_state, idx_t segment_idx) {
+	auto checkpoint_task = make_uniq<CheckpointTask>(checkpoint_state, segment_idx);
+	checkpoint_state.ScheduleTask(std::move(checkpoint_task));
+}
+
 void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &global_stats) {
-	bool can_vacuum_deletes = info->indexes.Empty();
-	idx_t start = this->row_start;
 	auto segments = row_groups->MoveSegments();
 	auto l = row_groups->Lock();
-	for (auto &entry : segments) {
-		auto &row_group = *entry.node;
-		if (can_vacuum_deletes && row_group.AllDeleted()) {
-			row_group.CommitDrop();
+
+	CollectionCheckpointState checkpoint_state(*this, writer, segments, global_stats);
+
+	VacuumState vacuum_state;
+	InitializeVacuumState(vacuum_state, segments);
+	// schedule tasks
+	for (idx_t segment_idx = 0; segment_idx < segments.size(); segment_idx++) {
+		auto &entry = segments[segment_idx];
+		auto vacuum_tasks = ScheduleVacuumTasks(checkpoint_state, vacuum_state, segment_idx);
+		if (vacuum_tasks) {
+			// vacuum tasks were scheduled - don't schedule a checkpoint task yet
+			continue;
+		}
+		if (!entry.node) {
+			// row group was vacuumed/dropped - skip
 			continue;
 		}
-		row_group.MoveToCollection(*this, start);
-		auto row_group_writer = writer.GetRowGroupWriter(row_group);
-		auto pointer = row_group.Checkpoint(*row_group_writer, global_stats);
+		// schedule a checkpoint task for this row group
+		entry.node->MoveToCollection(*this, vacuum_state.row_start);
+		ScheduleCheckpointTask(checkpoint_state, segment_idx);
+		vacuum_state.row_start += entry.node->count;
+	}
+	// all tasks have been scheduled - execute tasks until we are done
+	do {
+		shared_ptr<Task> task;
+		while (checkpoint_state.GetTask(task)) {
+			task->Execute(TaskExecutionMode::PROCESS_ALL);
+			task.reset();
+		}
+	} while (!checkpoint_state.TasksFinished());
+	// check if we ran into any errors while checkpointing
+	if (checkpoint_state.HasError()) {
+		// throw the error
+		checkpoint_state.CancelTasks();
+		checkpoint_state.ThrowError();
+	}
+
+	// no errors - finalize the row groups
+	idx_t new_total_rows = 0;
+	for (idx_t segment_idx = 0; segment_idx < segments.size(); segment_idx++) {
+		auto &entry = segments[segment_idx];
+		if (!entry.node) {
+			// row group was vacuumed/dropped - skip
+			continue;
+		}
+		auto &row_group = *entry.node;
+		auto row_group_writer = std::move(checkpoint_state.writers[segment_idx]);
+		if (!row_group_writer) {
+			throw InternalException("Missing row group writer for index %llu", segment_idx);
+		}
+		auto pointer =
+		    row_group.Checkpoint(std::move(checkpoint_state.write_data[segment_idx]), *row_group_writer, global_stats);
 		writer.AddRowGroup(std::move(pointer), std::move(row_group_writer));
 		row_groups->AppendSegment(l, std::move(entry.node));
-		start += row_group.count;
+		new_total_rows += row_group.count;
 	}
-	total_rows = start;
+	total_rows = new_total_rows;
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/storage/table/row_version_manager.cpp

@@ -219,7 +219,8 @@ shared_ptr<RowVersionManager> RowVersionManager::Deserialize(MetaBlockPointer de
 	for (idx_t i = 0; i < chunk_count; i++) {
 		idx_t vector_index = source.Read<idx_t>();
 		if (vector_index >= Storage::ROW_GROUP_VECTOR_COUNT) {
-			throw Exception("In DeserializeDeletes, vector_index is out of range for the row group. Corrupted file?");
+			throw InternalException(
+			    "In DeserializeDeletes, vector_index is out of range for the row group. Corrupted file?");
 		}
 		version_info->vector_info[vector_index] = ChunkInfo::Read(source);
 	}

package/src/duckdb/src/storage/table/scan_state.cpp

@@ -67,6 +67,10 @@ AdaptiveFilter *CollectionScanState::GetAdaptiveFilter() {
 	return parent.GetAdaptiveFilter();
 }
 
+TableScanOptions &CollectionScanState::GetOptions() {
+	return parent.options;
+}
+
 ParallelCollectionScanState::ParallelCollectionScanState()
     : collection(nullptr), current_row_group(nullptr), processed_rows(0) {
 }

package/src/duckdb/src/storage/table/standard_column_data.cpp

@@ -105,6 +105,7 @@ idx_t StandardColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &re
 	// fetch validity mask
 	if (state.child_states.empty()) {
 		ColumnScanState child_state;
+		child_state.scan_options = state.scan_options;
 		state.child_states.push_back(std::move(child_state));
 	}
 	auto scan_count = ColumnData::Fetch(state, row_id, result);
@@ -189,8 +190,13 @@ StandardColumnData::CreateCheckpointState(RowGroup &row_group, PartialBlockManag
 unique_ptr<ColumnCheckpointState> StandardColumnData::Checkpoint(RowGroup &row_group,
                                                                  PartialBlockManager &partial_block_manager,
                                                                  ColumnCheckpointInfo &checkpoint_info) {
-	auto validity_state = validity.Checkpoint(row_group, partial_block_manager, checkpoint_info);
+	// we need to checkpoint the main column data first
+	// that is because the checkpointing of the main column data ALSO scans the validity data
+	// to prevent reading the validity data immediately after it is checkpointed we first checkpoint the main column
+	// this is necessary for concurrent checkpointing as due to the partial block manager checkpointed data might be
+	// flushed to disk by a different thread than the one that wrote it, causing a data race
 	auto base_state = ColumnData::Checkpoint(row_group, partial_block_manager, checkpoint_info);
+	auto validity_state = validity.Checkpoint(row_group, partial_block_manager, checkpoint_info);
 	auto &checkpoint_state = base_state->Cast<StandardColumnCheckpointState>();
 	checkpoint_state.validity_state = std::move(validity_state);
 	return base_state;
@@ -204,10 +210,10 @@ void StandardColumnData::CheckpointScan(ColumnSegment &segment, ColumnScanState
 	validity.ScanCommittedRange(row_group_start, offset_in_row_group, count, scan_vector);
 }
 
-void StandardColumnData::DeserializeColumn(Deserializer &deserializer) {
-	ColumnData::DeserializeColumn(deserializer);
-	deserializer.ReadObject(101, "validity",
-	                        [&](Deserializer &deserializer) { validity.DeserializeColumn(deserializer); });
+void StandardColumnData::DeserializeColumn(Deserializer &deserializer, BaseStatistics &target_stats) {
+	ColumnData::DeserializeColumn(deserializer, target_stats);
+	deserializer.ReadObject(
+	    101, "validity", [&](Deserializer &deserializer) { validity.DeserializeColumn(deserializer, target_stats); });
 }
 
 void StandardColumnData::GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,

package/src/duckdb/src/storage/table/struct_column_data.cpp

@@ -5,6 +5,7 @@
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
+#include "duckdb/storage/table/update_segment.hpp"
 
 namespace duckdb {
 
@@ -14,7 +15,7 @@ StructColumnData::StructColumnData(BlockManager &block_manager, DataTableInfo &i
       validity(block_manager, info, 0, start_row, *this) {
 	D_ASSERT(type.InternalType() == PhysicalType::STRUCT);
 	auto &child_types = StructType::GetChildTypes(type);
-	D_ASSERT(child_types.size() > 0);
+	D_ASSERT(!child_types.empty());
 	if (type.id() != LogicalTypeId::UNION && StructType::IsUnnamed(type)) {
 		throw InvalidInputException("A table cannot be created from an unnamed struct");
 	}
@@ -36,8 +37,25 @@ void StructColumnData::SetStart(idx_t new_start) {
 }
 
 bool StructColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
-	// table filters are not supported yet for struct columns
-	return false;
+	if (!state.segment_checked) {
+		if (!state.current) {
+			return true;
+		}
+		state.segment_checked = true;
+		auto prune_result = filter.CheckStatistics(state.current->stats.statistics);
+		if (prune_result != FilterPropagateResult::FILTER_ALWAYS_FALSE) {
+			return true;
+		}
+		if (updates) {
+			auto update_stats = updates->GetStatistics();
+			prune_result = filter.CheckStatistics(*update_stats);
+			return prune_result != FilterPropagateResult::FILTER_ALWAYS_FALSE;
+		} else {
+			return false;
+		}
+	} else {
+		return true;
+	}
 }
 
 idx_t StructColumnData::GetMaxEntry() {
@@ -148,6 +166,7 @@ idx_t StructColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &resu
 	// insert any child states that are required
 	for (idx_t i = state.child_states.size(); i < child_entries.size() + 1; i++) {
 		ColumnScanState child_state;
+		child_state.scan_options = state.scan_options;
 		state.child_states.push_back(std::move(child_state));
 	}
 	// fetch the validity state
@@ -239,11 +258,11 @@ struct StructColumnCheckpointState : public ColumnCheckpointState {
 
 public:
 	unique_ptr<BaseStatistics> GetStatistics() override {
-		auto stats = StructStats::CreateEmpty(column_data.type);
+		D_ASSERT(global_stats);
 		for (idx_t i = 0; i < child_states.size(); i++) {
-			StructStats::SetChildStats(stats, i, child_states[i]->GetStatistics());
+			StructStats::SetChildStats(*global_stats, i, child_states[i]->GetStatistics());
 		}
-		return stats.ToUnique();
+		return std::move(global_stats);
 	}
 
 	void WriteDataPointers(RowGroupWriter &writer, Serializer &serializer) override {
@@ -273,12 +292,13 @@ unique_ptr<ColumnCheckpointState> StructColumnData::Checkpoint(RowGroup &row_gro
 	return std::move(checkpoint_state);
 }
 
-void StructColumnData::DeserializeColumn(Deserializer &deserializer) {
-	deserializer.ReadObject(101, "validity",
-	                        [&](Deserializer &deserializer) { validity.DeserializeColumn(deserializer); });
+void StructColumnData::DeserializeColumn(Deserializer &deserializer, BaseStatistics &target_stats) {
+	deserializer.ReadObject(
+	    101, "validity", [&](Deserializer &deserializer) { validity.DeserializeColumn(deserializer, target_stats); });
 
 	deserializer.ReadList(102, "sub_columns", [&](Deserializer::List &list, idx_t i) {
-		list.ReadObject([&](Deserializer &item) { sub_columns[i]->DeserializeColumn(item); });
+		auto &child_stats = StructStats::GetChildStats(target_stats, i);
+		list.ReadObject([&](Deserializer &item) { sub_columns[i]->DeserializeColumn(item, child_stats); });
 	});
 
 	this->count = validity.count;

package/src/duckdb/src/storage/table/table_statistics.cpp

@@ -2,6 +2,7 @@
 #include "duckdb/storage/table/persistent_table_data.hpp"
 #include "duckdb/common/serializer/serializer.hpp"
 #include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/execution/reservoir_sample.hpp"
 
 namespace duckdb {
 
@@ -69,7 +70,10 @@ void TableStatistics::MergeStats(TableStatistics &other) {
 	auto l = GetLock();
 	D_ASSERT(column_stats.size() == other.column_stats.size());
 	for (idx_t i = 0; i < column_stats.size(); i++) {
-		column_stats[i]->Merge(*other.column_stats[i]);
+		if (column_stats[i]) {
+			D_ASSERT(other.column_stats[i]);
+			column_stats[i]->Merge(*other.column_stats[i]);
+		}
 	}
 }
 
@@ -103,6 +107,7 @@ void TableStatistics::CopyStats(TableStatistics &other) {
 
 void TableStatistics::Serialize(Serializer &serializer) const {
 	serializer.WriteProperty(100, "column_stats", column_stats);
+	serializer.WritePropertyWithDefault<unique_ptr<BlockingSample>>(101, "table_sample", table_sample, nullptr);
 }
 
 void TableStatistics::Deserialize(Deserializer &deserializer, ColumnList &columns) {
@@ -120,6 +125,7 @@ void TableStatistics::Deserialize(Deserializer &deserializer, ColumnList &column
 
 		deserializer.Unset<LogicalType>();
 	});
+	table_sample = deserializer.ReadPropertyWithDefault<unique_ptr<BlockingSample>>(101, "sample", nullptr);
 }
 
 unique_ptr<TableStatisticsLock> TableStatistics::GetLock() {