duckdb 0.7.2-dev2867.0 → 0.7.2-dev2995.0

package/src/duckdb/src/storage/table/column_data_checkpointer.cpp

@@ -43,16 +43,16 @@ ColumnCheckpointState &ColumnDataCheckpointer::GetCheckpointState() {
 void ColumnDataCheckpointer::ScanSegments(const std::function<void(Vector &, idx_t)> &callback) {
 	Vector scan_vector(intermediate.GetType(), nullptr);
 	for (idx_t segment_idx = 0; segment_idx < nodes.size(); segment_idx++) {
-		auto segment = nodes[segment_idx].node.get();
+		auto &segment = *nodes[segment_idx].node;
 		ColumnScanState scan_state;
-		scan_state.current = segment;
-		segment->InitializeScan(scan_state);
+		scan_state.current = &segment;
+		segment.InitializeScan(scan_state);
 
-		for (idx_t base_row_index = 0; base_row_index < segment->count; base_row_index += STANDARD_VECTOR_SIZE) {
+		for (idx_t base_row_index = 0; base_row_index < segment.count; base_row_index += STANDARD_VECTOR_SIZE) {
 			scan_vector.Reference(intermediate);
 
-			idx_t count = MinValue<idx_t>(segment->count - base_row_index, STANDARD_VECTOR_SIZE);
-			scan_state.row_index = segment->start + base_row_index;
+			idx_t count = MinValue<idx_t>(segment.count - base_row_index, STANDARD_VECTOR_SIZE);
+			scan_state.row_index = segment.start + base_row_index;
 
 			col_data.CheckpointScan(segment, scan_state, row_group.start, count, scan_vector);
 
@@ -250,7 +250,7 @@ void ColumnDataCheckpointer::Checkpoint(vector<SegmentNode<ColumnSegment>> nodes
 		// no changes: only need to write the metadata for this column
 		WritePersistentSegments();
 	} else {
-		// there are changes: rewrite the set of columns
+		// there are changes: rewrite the set of columns);
 		WriteToDisk();
 	}
 }

package/src/duckdb/src/storage/table/column_segment.cpp

@@ -172,7 +172,7 @@ void ColumnSegment::RevertAppend(idx_t start_row) {
 //===--------------------------------------------------------------------===//
 // Convert To Persistent
 //===--------------------------------------------------------------------===//
-void ColumnSegment::ConvertToPersistent(BlockManager *block_manager, block_id_t block_id_p) {
+void ColumnSegment::ConvertToPersistent(optional_ptr<BlockManager> block_manager, block_id_t block_id_p) {
 	D_ASSERT(segment_type == ColumnSegmentType::TRANSIENT);
 	segment_type = ColumnSegmentType::PERSISTENT;
 

package/src/duckdb/src/storage/table/list_column_data.cpp

@@ -8,19 +8,19 @@
 namespace duckdb {
 
 ListColumnData::ListColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
-                               LogicalType type_p, ColumnData *parent)
+                               LogicalType type_p, optional_ptr<ColumnData> parent)
     : ColumnData(block_manager, info, column_index, start_row, std::move(type_p), parent),
-      validity(block_manager, info, 0, start_row, this) {
+      validity(block_manager, info, 0, start_row, *this) {
 	D_ASSERT(type.InternalType() == PhysicalType::LIST);
 	auto &child_type = ListType::GetChildType(type);
 	// the child column, with column index 1 (0 is the validity mask)
 	child_column = ColumnData::CreateColumnUnique(block_manager, info, 1, start_row, child_type, this);
 }
 
-ListColumnData::ListColumnData(ColumnData &original, idx_t start_row, ColumnData *parent)
-    : ColumnData(original, start_row, parent), validity(((ListColumnData &)original).validity, start_row, this) {
-	auto &list_data = (ListColumnData &)original;
-	child_column = ColumnData::CreateColumnUnique(*list_data.child_column, start_row, this);
+void ListColumnData::SetStart(idx_t new_start) {
+	ColumnData::SetStart(new_start);
+	child_column->SetStart(new_start);
+	validity.SetStart(new_start);
 }
 
 bool ListColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
@@ -322,11 +322,6 @@ public:
 		validity_state->WriteDataPointers(writer);
 		child_state->WriteDataPointers(writer);
 	}
-	void GetBlockIds(unordered_set<block_id_t> &result) override {
-		ColumnCheckpointState::GetBlockIds(result);
-		validity_state->GetBlockIds(result);
-		child_state->GetBlockIds(result);
-	}
 };
 
 unique_ptr<ColumnCheckpointState> ListColumnData::CreateCheckpointState(RowGroup &row_group,

package/src/duckdb/src/storage/table/row_group.cpp

@@ -45,16 +45,15 @@ RowGroup::RowGroup(RowGroupCollection &collection, RowGroupPointer &&pointer)
 	Verify();
 }
 
-RowGroup::RowGroup(RowGroup &row_group, RowGroupCollection &collection, idx_t start)
-    : SegmentBase<RowGroup>(start, row_group.count.load()), collection(collection),
-      version_info(std::move(row_group.version_info)) {
-	for (auto &column : row_group.GetColumns()) {
-		this->columns.push_back(ColumnData::CreateColumn(*column, start));
+void RowGroup::MoveToCollection(RowGroupCollection &collection, idx_t new_start) {
+	this->collection = collection;
+	this->start = new_start;
+	for (auto &column : GetColumns()) {
+		column->SetStart(new_start);
 	}
 	if (version_info) {
-		version_info->SetStart(start);
+		version_info->SetStart(new_start);
 	}
-	Verify();
 }
 
 void VersionNode::SetStart(idx_t start) {
@@ -101,8 +100,8 @@ ColumnData &RowGroup::GetColumn(idx_t c) {
 	if (column_pointers.size() != columns.size()) {
 		throw InternalException("Lazy loading a column but the pointer was not set");
 	}
-	auto &block_manager = collection.GetBlockManager();
-	auto &types = collection.GetTypes();
+	auto &block_manager = GetCollection().GetBlockManager();
+	auto &types = GetCollection().GetTypes();
 	auto &block_pointer = column_pointers[c];
 	MetaBlockReader column_data_reader(block_manager, block_pointer.block_id);
 	column_data_reader.offset = block_pointer.offset;
@@ -113,14 +112,14 @@ ColumnData &RowGroup::GetColumn(idx_t c) {
 }
 
 DatabaseInstance &RowGroup::GetDatabase() {
-	return collection.GetDatabase();
+	return GetCollection().GetDatabase();
 }
 
 BlockManager &RowGroup::GetBlockManager() {
-	return collection.GetBlockManager();
+	return GetCollection().GetBlockManager();
 }
 DataTableInfo &RowGroup::GetTableInfo() {
-	return collection.GetTableInfo();
+	return GetCollection().GetTableInfo();
 }
 
 void RowGroup::InitializeEmpty(const vector<LogicalType> &types) {
@@ -231,7 +230,7 @@ unique_ptr<RowGroup> RowGroup::AlterType(RowGroupCollection &new_collection, con
 	column_data->InitializeAppend(append_state);
 
 	// scan the original table, and fill the new column with the transformed value
-	scan_state.Initialize(collection.GetTypes());
+	scan_state.Initialize(GetCollection().GetTypes());
 	InitializeScan(scan_state);
 
 	DataChunk append_chunk;
@@ -537,7 +536,7 @@ void RowGroup::Scan(TransactionData transaction, CollectionScanState &state, Dat
 }
 
 void RowGroup::ScanCommitted(CollectionScanState &state, DataChunk &result, TableScanType type) {
-	auto &transaction_manager = DuckTransactionManager::Get(collection.GetAttached());
+	auto &transaction_manager = DuckTransactionManager::Get(GetCollection().GetAttached());
 
 	auto lowest_active_start = transaction_manager.LowestActiveStart();
 	auto lowest_active_id = transaction_manager.LowestActiveId();

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -96,6 +96,10 @@ RowGroup *RowGroupCollection::GetRowGroup(int64_t index) {
 	return (RowGroup *)row_groups->GetSegmentByIndex(index);
 }
 
+idx_t RowGroupCollection::RowGroupCount() {
+	return row_groups->GetSegmentCount();
+}
+
 void RowGroupCollection::Verify() {
 #ifdef DEBUG
 	idx_t current_total_rows = 0;
@@ -444,10 +448,12 @@ void RowGroupCollection::RevertAppendInternal(idx_t start_row, idx_t count) {
 void RowGroupCollection::MergeStorage(RowGroupCollection &data) {
 	D_ASSERT(data.types == types);
 	auto index = row_start + total_rows.load();
-	for (auto &row_group : data.row_groups->Segments()) {
-		auto new_group = make_uniq<RowGroup>(row_group, *this, index);
-		index += new_group->count;
-		row_groups->AppendSegment(std::move(new_group));
+	auto segments = data.row_groups->MoveSegments();
+	for (auto &entry : segments) {
+		auto &row_group = entry.node;
+		row_group->MoveToCollection(*this, index);
+		index += row_group->count;
+		row_groups->AppendSegment(std::move(row_group));
 	}
 	stats.MergeStats(data.stats);
 	total_rows += data.total_rows.load();

package/src/duckdb/src/storage/table/standard_column_data.cpp

@@ -10,13 +10,14 @@
 namespace duckdb {
 
 StandardColumnData::StandardColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
-                                       idx_t start_row, LogicalType type, ColumnData *parent)
+                                       idx_t start_row, LogicalType type, optional_ptr<ColumnData> parent)
     : ColumnData(block_manager, info, column_index, start_row, std::move(type), parent),
-      validity(block_manager, info, 0, start_row, this) {
+      validity(block_manager, info, 0, start_row, *this) {
 }
 
-StandardColumnData::StandardColumnData(ColumnData &original, idx_t start_row, ColumnData *parent)
-    : ColumnData(original, start_row, parent), validity(((StandardColumnData &)original).validity, start_row, this) {
+void StandardColumnData::SetStart(idx_t new_start) {
+	ColumnData::SetStart(new_start);
+	validity.SetStart(new_start);
 }
 
 bool StandardColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
@@ -176,11 +177,6 @@ public:
 		ColumnCheckpointState::WriteDataPointers(writer);
 		validity_state->WriteDataPointers(writer);
 	}
-
-	void GetBlockIds(unordered_set<block_id_t> &result) override {
-		ColumnCheckpointState::GetBlockIds(result);
-		validity_state->GetBlockIds(result);
-	}
 };
 
 unique_ptr<ColumnCheckpointState>
@@ -198,7 +194,7 @@ unique_ptr<ColumnCheckpointState> StandardColumnData::Checkpoint(RowGroup &row_g
 	return base_state;
 }
 
-void StandardColumnData::CheckpointScan(ColumnSegment *segment, ColumnScanState &state, idx_t row_group_start,
+void StandardColumnData::CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start,
                                         idx_t count, Vector &scan_vector) {
 	ColumnData::CheckpointScan(segment, state, row_group_start, count, scan_vector);
 

package/src/duckdb/src/storage/table/struct_column_data.cpp

@@ -8,9 +8,9 @@
 namespace duckdb {
 
 StructColumnData::StructColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
-                                   idx_t start_row, LogicalType type_p, ColumnData *parent)
+                                   idx_t start_row, LogicalType type_p, optional_ptr<ColumnData> parent)
     : ColumnData(block_manager, info, column_index, start_row, std::move(type_p), parent),
-      validity(block_manager, info, 0, start_row, this) {
+      validity(block_manager, info, 0, start_row, *this) {
 	D_ASSERT(type.InternalType() == PhysicalType::STRUCT);
 	auto &child_types = StructType::GetChildTypes(type);
 	D_ASSERT(child_types.size() > 0);
@@ -23,12 +23,12 @@ StructColumnData::StructColumnData(BlockManager &block_manager, DataTableInfo &i
 	}
 }
 
-StructColumnData::StructColumnData(ColumnData &original, idx_t start_row, ColumnData *parent)
-    : ColumnData(original, start_row, parent), validity(((StructColumnData &)original).validity, start_row, this) {
-	auto &struct_data = (StructColumnData &)original;
-	for (auto &child_col : struct_data.sub_columns) {
-		sub_columns.push_back(ColumnData::CreateColumnUnique(*child_col, start_row, this));
+void StructColumnData::SetStart(idx_t new_start) {
+	this->start = new_start;
+	for (auto &sub_column : sub_columns) {
+		sub_column->SetStart(new_start);
 	}
+	validity.SetStart(new_start);
 }
 
 bool StructColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
@@ -246,12 +246,6 @@ public:
 			state->WriteDataPointers(writer);
 		}
 	}
-	void GetBlockIds(unordered_set<block_id_t> &result) override {
-		validity_state->GetBlockIds(result);
-		for (auto &state : child_states) {
-			state->GetBlockIds(result);
-		}
-	}
 };
 
 unique_ptr<ColumnCheckpointState> StructColumnData::CreateCheckpointState(RowGroup &row_group,

package/src/duckdb/src/storage/table/update_segment.cpp

@@ -37,31 +37,6 @@ UpdateSegment::UpdateSegment(ColumnData &column_data)
 	this->statistics_update_function = GetStatisticsUpdateFunction(physical_type);
 }
 
-UpdateSegment::UpdateSegment(UpdateSegment &other, ColumnData &owner)
-    : column_data(owner), root(std::move(other.root)), stats(std::move(other.stats)), type_size(other.type_size) {
-
-	this->heap.Move(other.heap);
-	// update the segment links
-	if (root) {
-		for (idx_t i = 0; i < RowGroup::ROW_GROUP_VECTOR_COUNT; i++) {
-			if (!root->info[i]) {
-				continue;
-			}
-			for (auto info = root->info[i]->info.get(); info; info = info->next) {
-				info->segment = this;
-			}
-		}
-	}
-	initialize_update_function = other.initialize_update_function;
-	merge_update_function = other.merge_update_function;
-	fetch_update_function = other.fetch_update_function;
-	fetch_committed_function = other.fetch_committed_function;
-	fetch_committed_range = other.fetch_committed_range;
-	fetch_row_function = other.fetch_row_function;
-	rollback_update_function = other.rollback_update_function;
-	statistics_update_function = other.statistics_update_function;
-}
-
 UpdateSegment::~UpdateSegment() {
 }
 

package/src/duckdb/src/storage/table/validity_column_data.cpp

@@ -5,12 +5,8 @@
 namespace duckdb {
 
 ValidityColumnData::ValidityColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
-                                       idx_t start_row, ColumnData *parent)
-    : ColumnData(block_manager, info, column_index, start_row, LogicalType(LogicalTypeId::VALIDITY), parent) {
-}
-
-ValidityColumnData::ValidityColumnData(ColumnData &original, idx_t start_row, ColumnData *parent)
-    : ColumnData(original, start_row, parent) {
+                                       idx_t start_row, ColumnData &parent)
+    : ColumnData(block_manager, info, column_index, start_row, LogicalType(LogicalTypeId::VALIDITY), &parent) {
 }
 
 bool ValidityColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {

package/src/duckdb/src/transaction/commit_state.cpp

@@ -210,10 +210,10 @@ void CommitState::WriteUpdate(UpdateInfo &info) {
 
 	// construct the column index path
 	vector<column_t> column_indexes;
-	auto column_data_ptr = &column_data;
-	while (column_data_ptr->parent) {
-		column_indexes.push_back(column_data_ptr->column_index);
-		column_data_ptr = column_data_ptr->parent;
+	reference<ColumnData> current_column_data = column_data;
+	while (current_column_data.get().parent) {
+		column_indexes.push_back(current_column_data.get().column_index);
+		current_column_data = *current_column_data.get().parent;
 	}
 	column_indexes.push_back(info.column_index);
 	std::reverse(column_indexes.begin(), column_indexes.end());

package/src/duckdb/src/verification/no_operator_caching_verifier.cpp

@@ -0,0 +1,13 @@
+#include "duckdb/verification/no_operator_caching_verifier.hpp"
+
+namespace duckdb {
+
+NoOperatorCachingVerifier::NoOperatorCachingVerifier(unique_ptr<SQLStatement> statement_p)
+    : StatementVerifier(VerificationType::NO_OPERATOR_CACHING, "No operator caching", std::move(statement_p)) {
+}
+
+unique_ptr<StatementVerifier> NoOperatorCachingVerifier::Create(const SQLStatement &statement_p) {
+	return make_uniq<NoOperatorCachingVerifier>(statement_p.Copy());
+}
+
+} // namespace duckdb

package/src/duckdb/src/verification/statement_verifier.cpp

@@ -9,6 +9,7 @@
 #include "duckdb/verification/parsed_statement_verifier.hpp"
 #include "duckdb/verification/prepared_statement_verifier.hpp"
 #include "duckdb/verification/unoptimized_statement_verifier.hpp"
+#include "duckdb/verification/no_operator_caching_verifier.hpp"
 
 namespace duckdb {
 
@@ -35,6 +36,8 @@ unique_ptr<StatementVerifier> StatementVerifier::Create(VerificationType type, c
 		return ParsedStatementVerifier::Create(statement_p);
 	case VerificationType::UNOPTIMIZED:
 		return UnoptimizedStatementVerifier::Create(statement_p);
+	case VerificationType::NO_OPERATOR_CACHING:
+		return NoOperatorCachingVerifier::Create(statement_p);
 	case VerificationType::PREPARED:
 		return PreparedStatementVerifier::Create(statement_p);
 	case VerificationType::EXTERNAL:
@@ -104,6 +107,7 @@ bool StatementVerifier::Run(
 
 	context.interrupted = false;
 	context.config.enable_optimizer = !DisableOptimizer();
+	context.config.enable_caching_operators = !DisableOperatorCaching();
 	context.config.force_external = ForceExternal();
 	try {
 		auto result = run(query, std::move(statement));

package/src/duckdb/ub_src_core_functions_scalar_string.cpp

@@ -6,6 +6,8 @@
 
 #include "src/core_functions/scalar/string/damerau_levenshtein.cpp"
 
+#include "src/core_functions/scalar/string/format_bytes.cpp"
+
 #include "src/core_functions/scalar/string/hamming.cpp"
 
 #include "src/core_functions/scalar/string/hex.cpp"

package/src/duckdb/ub_src_parallel.cpp

@@ -8,6 +8,8 @@
 
 #include "src/parallel/event.cpp"
 
+#include "src/parallel/interrupt.cpp"
+
 #include "src/parallel/pipeline.cpp"
 
 #include "src/parallel/pipeline_complete_event.cpp"

package/src/duckdb/ub_src_storage.cpp

@@ -26,6 +26,8 @@
 
 #include "src/storage/write_ahead_log.cpp"
 
+#include "src/storage/optimistic_data_writer.cpp"
+
 #include "src/storage/partial_block_manager.cpp"
 
 #include "src/storage/single_file_block_manager.cpp"