duckdb 0.10.3-dev0.0 → 0.10.3-dev6.0

package/src/duckdb/src/planner/planner.cpp

@@ -13,6 +13,7 @@
 #include "duckdb/planner/expression/bound_parameter_expression.hpp"
 #include "duckdb/transaction/meta_transaction.hpp"
 #include "duckdb/execution/column_binding_resolver.hpp"
+#include "duckdb/main/attached_database.hpp"
 
 namespace duckdb {
 
@@ -76,7 +77,7 @@ void Planner::CreatePlan(SQLStatement &statement) {
 			throw;
 		}
 	}
-	this->properties = binder->properties;
+	this->properties = binder->GetStatementProperties();
 	this->properties.parameter_count = parameter_count;
 	properties.bound_all_parameters = !bound_parameters.rebind && parameters_resolved;
 
@@ -101,7 +102,7 @@ shared_ptr<PreparedStatementData> Planner::PrepareSQLStatement(unique_ptr<SQLSta
 	// create a plan of the underlying statement
 	CreatePlan(std::move(statement));
 	// now create the logical prepare
-	auto prepared_data = make_shared<PreparedStatementData>(copied_statement->type);
+	auto prepared_data = make_shared_ptr<PreparedStatementData>(copied_statement->type);
 	prepared_data->unbound_statement = std::move(copied_statement);
 	prepared_data->names = names;
 	prepared_data->types = types;
@@ -138,6 +139,7 @@ void Planner::CreatePlan(unique_ptr<SQLStatement> statement) {
 	case StatementType::ATTACH_STATEMENT:
 	case StatementType::DETACH_STATEMENT:
 	case StatementType::COPY_DATABASE_STATEMENT:
+	case StatementType::UPDATE_EXTENSIONS_STATEMENT:
 		CreatePlan(*statement);
 		break;
 	default:
@@ -156,9 +158,20 @@ static bool OperatorSupportsSerialization(LogicalOperator &op) {
 
 void Planner::VerifyPlan(ClientContext &context, unique_ptr<LogicalOperator> &op,
                          optional_ptr<bound_parameter_map_t> map) {
+	auto &config = DBConfig::GetConfig(context);
 #ifdef DUCKDB_ALTERNATIVE_VERIFY
-	// if alternate verification is enabled we run the original operator
-	return;
+	{
+		auto &serialize_comp = config.options.serialization_compatibility;
+		auto latest_version = SerializationCompatibility::Latest();
+		if (serialize_comp.manually_set &&
+		    serialize_comp.serialization_version != latest_version.serialization_version) {
+			// Serialization should not be skipped, this test relies on the serialization to remove certain fields for
+			// compatibility with older versions. This might change behavior, not doing this might make this test fail.
+		} else {
+			// if alternate verification is enabled we run the original operator
+			return;
+		}
+	}
 #endif
 	if (!op || !ClientConfig::GetConfig(context).verify_serializer) {
 		return;
@@ -173,7 +186,16 @@ void Planner::VerifyPlan(ClientContext &context, unique_ptr<LogicalOperator> &op
 	// format (de)serialization of this operator
 	try {
 		MemoryStream stream;
-		BinarySerializer::Serialize(*op, stream, true);
+
+		SerializationOptions options;
+		if (config.options.serialization_compatibility.manually_set) {
+			// Override the default of 'latest' if this was manually set (for testing, mostly)
+			options.serialization_compatibility = config.options.serialization_compatibility;
+		} else {
+			options.serialization_compatibility = SerializationCompatibility::Latest();
+		}
+
+		BinarySerializer::Serialize(*op, stream, options);
 		stream.Rewind();
 		bound_parameter_map_t parameters;
 		auto new_plan = BinaryDeserializer::Deserialize<LogicalOperator>(stream, context, parameters);
@@ -182,10 +204,14 @@ void Planner::VerifyPlan(ClientContext &context, unique_ptr<LogicalOperator> &op
 			*map = std::move(parameters);
 		}
 		op = std::move(new_plan);
-	} catch (SerializationException &ex) {  // NOLINT: explicitly allowing these errors (for now)
-		                                    // pass
-	} catch (NotImplementedException &ex) { // NOLINT: explicitly allowing these errors (for now)
-		                                    // pass
+	} catch (std::exception &ex) {
+		ErrorData error(ex);
+		switch (error.Type()) {
+		case ExceptionType::NOT_IMPLEMENTED: // NOLINT: explicitly allowing these errors (for now)
+			break;                           // pass
+		default:
+			throw;
+		}
 	}
 }
 

package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp

@@ -119,6 +119,8 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
                                                                                  idx_t lateral_depth) {
 	// first check if the logical operator has correlated expressions
 	auto entry = has_correlated_expressions.find(*plan);
+	bool exit_projection = false;
+	unique_ptr<LogicalDelimGet> delim_scan;
 	D_ASSERT(entry != has_correlated_expressions.end());
 	if (!entry->second) {
 		// we reached a node without correlated expressions
@@ -136,13 +138,21 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 			}
 		}
 
-		auto left_columns = plan->GetColumnBindings().size();
+		// create cross product with Delim Join
 		auto delim_index = binder.GenerateTableIndex();
-		this->base_binding = ColumnBinding(delim_index, 0);
-		this->delim_offset = left_columns;
-		this->data_offset = 0;
-		auto delim_scan = make_uniq<LogicalDelimGet>(delim_index, delim_types);
-		return LogicalCrossProduct::Create(std::move(plan), std::move(delim_scan));
+		base_binding = ColumnBinding(delim_index, 0);
+
+		auto left_columns = plan->GetColumnBindings().size();
+		delim_offset = left_columns;
+		data_offset = 0;
+		delim_scan = make_uniq<LogicalDelimGet>(delim_index, delim_types);
+		if (plan->type == LogicalOperatorType::LOGICAL_PROJECTION) {
+			// we want to keep the logical projection for positionality.
+			exit_projection = true;
+		} else {
+			auto cross_product = LogicalCrossProduct::Create(std::move(plan), std::move(delim_scan));
+			return cross_product;
+		}
 	}
 	switch (plan->type) {
 	case LogicalOperatorType::LOGICAL_UNNEST:
@@ -166,8 +176,18 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		for (auto &expr : plan->expressions) {
 			parent_propagate_null_values &= expr->PropagatesNullValues();
 		}
-		plan->children[0] =
-		    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
+
+		// if the node has no correlated expressions,
+		// push the cross product with the delim get only below the projection.
+		// This will preserve positionality of the columns and prevent errors when reordering of
+		// delim gets is enabled.
+		if (exit_projection) {
+			auto cross_product = LogicalCrossProduct::Create(std::move(plan->children[0]), std::move(delim_scan));
+			plan->children[0] = std::move(cross_product);
+		} else {
+			plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]),
+			                                                  parent_propagate_null_values, lateral_depth);
+		}
 
 		// then we replace any correlated expressions with the corresponding entry in the correlated_map
 		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth);
@@ -248,7 +268,7 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 				}
 			}
 			auto left_index = binder.GenerateTableIndex();
-			auto delim_scan = make_uniq<LogicalDelimGet>(left_index, delim_types);
+			delim_scan = make_uniq<LogicalDelimGet>(left_index, delim_types);
 			join->children.push_back(std::move(delim_scan));
 			join->children.push_back(std::move(plan));
 			for (idx_t i = 0; i < new_group_count; i++) {
@@ -555,6 +575,11 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		// push into children
 		plan->children[0] =
 		    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
+
+		// we replace any correlated expressions with the corresponding entry in the correlated_map
+		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth);
+		rewriter.VisitOperator(*plan);
+
 		// add the correlated columns to the PARTITION BY clauses in the Window
 		for (auto &expr : window.expressions) {
 			D_ASSERT(expr->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);

package/src/duckdb/src/planner/table_binding.cpp

@@ -171,7 +171,7 @@ ColumnBinding TableBinding::GetColumnBinding(column_t column_index) {
 	auto it = std::find_if(column_ids.begin(), column_ids.end(),
 	                       [&](const column_t &id) -> bool { return id == column_index; });
 	// Get the index of it
-	binding.column_index = std::distance(column_ids.begin(), it);
+	binding.column_index = NumericCast<idx_t>(std::distance(column_ids.begin(), it));
 	// If it wasn't found, add it
 	if (it == column_ids.end()) {
 		column_ids.push_back(column_index);

package/src/duckdb/src/storage/arena_allocator.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/storage/arena_allocator.hpp"
 
 #include "duckdb/common/assert.hpp"
+#include "duckdb/common/numeric_utils.hpp"
 
 namespace duckdb {
 
@@ -88,10 +89,11 @@ data_ptr_t ArenaAllocator::Reallocate(data_ptr_t pointer, idx_t old_size, idx_t
 	}
 
 	auto head_ptr = head->data.get() + head->current_position;
-	int64_t diff = size - old_size;
-	if (pointer == head_ptr && (size < old_size || head->current_position + diff <= head->maximum_size)) {
+	int64_t diff = NumericCast<int64_t>(size) - NumericCast<int64_t>(old_size);
+	if (pointer == head_ptr && (size < old_size || NumericCast<int64_t>(head->current_position) + diff <=
+	                                                   NumericCast<int64_t>(head->maximum_size))) {
 		// passed pointer is the head pointer, and the diff fits on the current chunk
-		head->current_position += diff;
+		head->current_position += NumericCast<idx_t>(diff);
 		return pointer;
 	} else {
 		// allocate new memory

package/src/duckdb/src/storage/buffer/block_handle.cpp

@@ -9,9 +9,9 @@
 namespace duckdb {
 
 BlockHandle::BlockHandle(BlockManager &block_manager, block_id_t block_id_p, MemoryTag tag)
-    : block_manager(block_manager), readers(0), block_id(block_id_p), tag(tag), buffer(nullptr), eviction_timestamp(0),
+    : block_manager(block_manager), readers(0), block_id(block_id_p), tag(tag), buffer(nullptr), eviction_seq_num(0),
       can_destroy(false), memory_charge(tag, block_manager.buffer_manager.GetBufferPool()), unswizzled(nullptr) {
-	eviction_timestamp = 0;
+	eviction_seq_num = 0;
 	state = BlockState::BLOCK_UNLOADED;
 	memory_usage = Storage::BLOCK_ALLOC_SIZE;
 }
@@ -19,7 +19,7 @@ BlockHandle::BlockHandle(BlockManager &block_manager, block_id_t block_id_p, Mem
 BlockHandle::BlockHandle(BlockManager &block_manager, block_id_t block_id_p, MemoryTag tag,
                          unique_ptr<FileBuffer> buffer_p, bool can_destroy_p, idx_t block_size,
                          BufferPoolReservation &&reservation)
-    : block_manager(block_manager), readers(0), block_id(block_id_p), tag(tag), eviction_timestamp(0),
+    : block_manager(block_manager), readers(0), block_id(block_id_p), tag(tag), eviction_seq_num(0),
       can_destroy(can_destroy_p), memory_charge(tag, block_manager.buffer_manager.GetBufferPool()),
       unswizzled(nullptr) {
 	buffer = std::move(buffer_p);

package/src/duckdb/src/storage/buffer/block_manager.cpp

@@ -23,7 +23,7 @@ shared_ptr<BlockHandle> BlockManager::RegisterBlock(block_id_t block_id) {
 		}
 	}
 	// create a new block pointer for this block
-	auto result = make_shared<BlockHandle>(*this, block_id, MemoryTag::BASE_TABLE);
+	auto result = make_shared_ptr<BlockHandle>(*this, block_id, MemoryTag::BASE_TABLE);
 	// register the block pointer in the set of blocks as a weak pointer
 	blocks[block_id] = weak_ptr<BlockHandle>(result);
 	return result;

package/src/duckdb/src/storage/buffer/buffer_pool.cpp

@@ -1,5 +1,6 @@
 #include "duckdb/storage/buffer/buffer_pool.hpp"
 
+#include "duckdb/common/chrono.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/parallel/concurrentqueue.hpp"
 #include "duckdb/storage/temporary_memory_manager.hpp"
@@ -12,8 +13,13 @@ struct EvictionQueue {
 	eviction_queue_t q;
 };
 
+BufferEvictionNode::BufferEvictionNode(weak_ptr<BlockHandle> handle_p, idx_t eviction_seq_num)
+    : handle(std::move(handle_p)), handle_sequence_number(eviction_seq_num) {
+	D_ASSERT(!handle.expired());
+}
+
 bool BufferEvictionNode::CanUnload(BlockHandle &handle_p) {
-	if (timestamp != handle_p.eviction_timestamp) {
+	if (handle_sequence_number != handle_p.eviction_seq_num) {
 		// handle was used in between
 		return false;
 	}
@@ -34,9 +40,10 @@ shared_ptr<BlockHandle> BufferEvictionNode::TryGetBlockHandle() {
 	return handle_p;
 }
 
-BufferPool::BufferPool(idx_t maximum_memory)
-    : current_memory(0), maximum_memory(maximum_memory), queue(make_uniq<EvictionQueue>()),
-      temporary_memory_manager(make_uniq<TemporaryMemoryManager>()), evict_queue_insertions(0), total_dead_nodes(0) {
+BufferPool::BufferPool(idx_t maximum_memory, bool track_eviction_timestamps)
+    : current_memory(0), maximum_memory(maximum_memory), track_eviction_timestamps(track_eviction_timestamps),
+      queue(make_uniq<EvictionQueue>()), temporary_memory_manager(make_uniq<TemporaryMemoryManager>()),
+      evict_queue_insertions(0), total_dead_nodes(0) {
 	for (idx_t i = 0; i < MEMORY_TAG_COUNT; i++) {
 		memory_usage_per_tag[i] = 0;
 	}
@@ -50,7 +57,13 @@ bool BufferPool::AddToEvictionQueue(shared_ptr<BlockHandle> &handle) {
 	// or the block handle is still a local variable (ConvertToPersistent)
 
 	D_ASSERT(handle->readers == 0);
-	auto ts = ++handle->eviction_timestamp;
+	auto ts = ++handle->eviction_seq_num;
+	if (track_eviction_timestamps) {
+		handle->lru_timestamp_msec =
+		    std::chrono::time_point_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now())
+		        .time_since_epoch()
+		        .count();
+	}
 
 	BufferEvictionNode evict_node(weak_ptr<BlockHandle>(handle), ts);
 	queue->q.enqueue(evict_node);
@@ -66,9 +79,14 @@ bool BufferPool::AddToEvictionQueue(shared_ptr<BlockHandle> &handle) {
 	return false;
 }
 
-void BufferPool::IncreaseUsedMemory(MemoryTag tag, idx_t size) {
-	current_memory += size;
-	memory_usage_per_tag[uint8_t(tag)] += size;
+void BufferPool::UpdateUsedMemory(MemoryTag tag, int64_t size) {
+	if (size < 0) {
+		current_memory -= UnsafeNumericCast<idx_t>(-size);
+		memory_usage_per_tag[uint8_t(tag)] -= UnsafeNumericCast<idx_t>(-size);
+	} else {
+		current_memory += UnsafeNumericCast<idx_t>(size);
+		memory_usage_per_tag[uint8_t(tag)] += UnsafeNumericCast<idx_t>(size);
+	}
 }
 
 idx_t BufferPool::GetUsedMemory() const {
@@ -89,18 +107,68 @@ TemporaryMemoryManager &BufferPool::GetTemporaryMemoryManager() {
 
 BufferPool::EvictionResult BufferPool::EvictBlocks(MemoryTag tag, idx_t extra_memory, idx_t memory_limit,
                                                    unique_ptr<FileBuffer> *buffer) {
-	BufferEvictionNode node;
 	TempBufferPoolReservation r(tag, *this, extra_memory);
+	bool found = false;
+
+	if (current_memory <= memory_limit) {
+		return {true, std::move(r)};
+	}
+
+	IterateUnloadableBlocks([&](BufferEvictionNode &, const shared_ptr<BlockHandle> &handle) {
+		// hooray, we can unload the block
+		if (buffer && handle->buffer->AllocSize() == extra_memory) {
+			// we can re-use the memory directly
+			*buffer = handle->UnloadAndTakeBlock();
+			found = true;
+			return false;
+		}
+
+		// release the memory and mark the block as unloaded
+		handle->Unload();
+
+		if (current_memory <= memory_limit) {
+			found = true;
+			return false;
+		}
 
-	while (current_memory > memory_limit) {
+		// Continue iteration
+		return true;
+	});
+
+	if (!found) {
+		r.Resize(0);
+	}
+
+	return {found, std::move(r)};
+}
+
+idx_t BufferPool::PurgeAgedBlocks(uint32_t max_age_sec) {
+	int64_t now = std::chrono::time_point_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now())
+	                  .time_since_epoch()
+	                  .count();
+	int64_t limit = now - (static_cast<int64_t>(max_age_sec) * 1000);
+	idx_t purged_bytes = 0;
+	IterateUnloadableBlocks([&](BufferEvictionNode &node, const shared_ptr<BlockHandle> &handle) {
+		// We will unload this block regardless. But stop the iteration immediately afterward if this
+		// block is younger than the age threshold.
+		bool is_fresh = handle->lru_timestamp_msec >= limit && handle->lru_timestamp_msec <= now;
+		purged_bytes += handle->GetMemoryUsage();
+		handle->Unload();
+		return is_fresh;
+	});
+	return purged_bytes;
+}
+
+template <typename FN>
+void BufferPool::IterateUnloadableBlocks(FN fn) {
+	for (;;) {
 		// get a block to unpin from the queue
+		BufferEvictionNode node;
 		if (!queue->q.try_dequeue(node)) {
 			// we could not dequeue any eviction node, so we try one more time,
 			// but more aggressively
 			if (!TryDequeueWithLock(node)) {
-				// still no success, we return
-				r.Resize(0);
-				return {false, std::move(r)};
+				return;
 			}
 		}
 
@@ -119,17 +187,10 @@ BufferPool::EvictionResult BufferPool::EvictBlocks(MemoryTag tag, idx_t extra_me
 			continue;
 		}
 
-		// hooray, we can unload the block
-		if (buffer && handle->buffer->AllocSize() == extra_memory) {
-			// we can re-use the memory directly
-			*buffer = handle->UnloadAndTakeBlock();
-			return {true, std::move(r)};
+		if (!fn(node, handle)) {
+			break;
 		}
-
-		// release the memory and mark the block as unloaded
-		handle->Unload();
 	}
-	return {true, std::move(r)};
 }
 
 bool BufferPool::TryDequeueWithLock(BufferEvictionNode &node) {

package/src/duckdb/src/storage/buffer/buffer_pool_reservation.cpp

@@ -23,8 +23,8 @@ BufferPoolReservation::~BufferPoolReservation() {
 }
 
 void BufferPoolReservation::Resize(idx_t new_size) {
-	int64_t delta = (int64_t)new_size - size;
-	pool.IncreaseUsedMemory(tag, delta);
+	auto delta = UnsafeNumericCast<int64_t>(new_size) - UnsafeNumericCast<int64_t>(size);
+	pool.UpdateUsedMemory(tag, delta);
 	size = new_size;
 }
 

package/src/duckdb/src/storage/buffer_manager.cpp

@@ -23,8 +23,12 @@ void BufferManager::FreeReservedMemory(idx_t size) {
 	throw NotImplementedException("This type of BufferManager can not free reserved memory");
 }
 
-void BufferManager::SetLimit(idx_t limit) {
-	throw NotImplementedException("This type of BufferManager can not set a limit");
+void BufferManager::SetMemoryLimit(idx_t limit) {
+	throw NotImplementedException("This type of BufferManager can not set a memory limit");
+}
+
+void BufferManager::SetSwapLimit(optional_idx limit) {
+	throw NotImplementedException("This type of BufferManager can not set a swap limit");
 }
 
 vector<TemporaryFileInformation> BufferManager::GetTemporaryFiles() {

package/src/duckdb/src/storage/checkpoint/row_group_writer.cpp

@@ -9,12 +9,21 @@ CompressionType RowGroupWriter::GetColumnCompressionType(idx_t i) {
 	return table.GetColumn(LogicalIndex(i)).CompressionType();
 }
 
+SingleFileRowGroupWriter::SingleFileRowGroupWriter(TableCatalogEntry &table, PartialBlockManager &partial_block_manager,
+                                                   TableDataWriter &writer, MetadataWriter &table_data_writer)
+    : RowGroupWriter(table, partial_block_manager), writer(writer), table_data_writer(table_data_writer) {
+}
+
 void SingleFileRowGroupWriter::WriteColumnDataPointers(ColumnCheckpointState &column_checkpoint_state,
                                                        Serializer &serializer) {
 	const auto &data_pointers = column_checkpoint_state.data_pointers;
 	serializer.WriteProperty(100, "data_pointers", data_pointers);
 }
 
+CheckpointType SingleFileRowGroupWriter::GetCheckpointType() const {
+	return writer.GetCheckpointType();
+}
+
 MetadataWriter &SingleFileRowGroupWriter::GetPayloadWriter() {
 	return table_data_writer;
 }

package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp

@@ -39,7 +39,12 @@ SingleFileTableDataWriter::SingleFileTableDataWriter(SingleFileCheckpointWriter
 }
 
 unique_ptr<RowGroupWriter> SingleFileTableDataWriter::GetRowGroupWriter(RowGroup &row_group) {
-	return make_uniq<SingleFileRowGroupWriter>(table, checkpoint_manager.partial_block_manager, table_data_writer);
+	return make_uniq<SingleFileRowGroupWriter>(table, checkpoint_manager.partial_block_manager, *this,
+	                                           table_data_writer);
+}
+
+CheckpointType SingleFileTableDataWriter::GetCheckpointType() const {
+	return checkpoint_manager.GetCheckpointType();
 }
 
 void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stats, DataTableInfo *info,
@@ -75,7 +80,7 @@ void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stat
 	serializer.WriteProperty(101, "table_pointer", pointer);
 	serializer.WriteProperty(102, "total_rows", total_rows);
 
-	auto index_storage_infos = info->indexes.GetStorageInfos();
+	auto index_storage_infos = info->GetIndexes().GetStorageInfos();
 	// write empty block pointers for forwards compatibility
 	vector<BlockPointer> compat_block_pointers;
 	serializer.WriteProperty(103, "index_pointers", compat_block_pointers);