duckdb 0.10.3-dev3.0 → 0.10.3-dev6.0

package/src/duckdb/src/storage/serialization/serialize_nodes.cpp

@@ -14,6 +14,8 @@
 #include "duckdb/parser/expression/case_expression.hpp"
 #include "duckdb/planner/expression/bound_case_expression.hpp"
 #include "duckdb/parser/parsed_data/sample_options.hpp"
+#include "duckdb/execution/reservoir_sample.hpp"
+#include "duckdb/common/queue.hpp"
 #include "duckdb/parser/tableref/pivotref.hpp"
 #include "duckdb/planner/tableref/bound_pivotref.hpp"
 #include "duckdb/parser/column_definition.hpp"
@@ -33,6 +35,52 @@
 
 namespace duckdb {
 
+void BlockingSample::Serialize(Serializer &serializer) const {
+	serializer.WritePropertyWithDefault<unique_ptr<BaseReservoirSampling>>(100, "base_reservoir_sample", base_reservoir_sample);
+	serializer.WriteProperty<SampleType>(101, "type", type);
+	serializer.WritePropertyWithDefault<bool>(102, "destroyed", destroyed);
+}
+
+unique_ptr<BlockingSample> BlockingSample::Deserialize(Deserializer &deserializer) {
+	auto base_reservoir_sample = deserializer.ReadPropertyWithDefault<unique_ptr<BaseReservoirSampling>>(100, "base_reservoir_sample");
+	auto type = deserializer.ReadProperty<SampleType>(101, "type");
+	auto destroyed = deserializer.ReadPropertyWithDefault<bool>(102, "destroyed");
+	unique_ptr<BlockingSample> result;
+	switch (type) {
+	case SampleType::RESERVOIR_PERCENTAGE_SAMPLE:
+		result = ReservoirSamplePercentage::Deserialize(deserializer);
+		break;
+	case SampleType::RESERVOIR_SAMPLE:
+		result = ReservoirSample::Deserialize(deserializer);
+		break;
+	default:
+		throw SerializationException("Unsupported type for deserialization of BlockingSample!");
+	}
+	result->base_reservoir_sample = std::move(base_reservoir_sample);
+	result->destroyed = destroyed;
+	return result;
+}
+
+void BaseReservoirSampling::Serialize(Serializer &serializer) const {
+	serializer.WritePropertyWithDefault<idx_t>(100, "next_index_to_sample", next_index_to_sample);
+	serializer.WriteProperty<double>(101, "min_weight_threshold", min_weight_threshold);
+	serializer.WritePropertyWithDefault<idx_t>(102, "min_weighted_entry_index", min_weighted_entry_index);
+	serializer.WritePropertyWithDefault<idx_t>(103, "num_entries_to_skip_b4_next_sample", num_entries_to_skip_b4_next_sample);
+	serializer.WritePropertyWithDefault<idx_t>(104, "num_entries_seen_total", num_entries_seen_total);
+	serializer.WritePropertyWithDefault<std::priority_queue<std::pair<double, idx_t>>>(105, "reservoir_weights", reservoir_weights);
+}
+
+unique_ptr<BaseReservoirSampling> BaseReservoirSampling::Deserialize(Deserializer &deserializer) {
+	auto result = duckdb::unique_ptr<BaseReservoirSampling>(new BaseReservoirSampling());
+	deserializer.ReadPropertyWithDefault<idx_t>(100, "next_index_to_sample", result->next_index_to_sample);
+	deserializer.ReadProperty<double>(101, "min_weight_threshold", result->min_weight_threshold);
+	deserializer.ReadPropertyWithDefault<idx_t>(102, "min_weighted_entry_index", result->min_weighted_entry_index);
+	deserializer.ReadPropertyWithDefault<idx_t>(103, "num_entries_to_skip_b4_next_sample", result->num_entries_to_skip_b4_next_sample);
+	deserializer.ReadPropertyWithDefault<idx_t>(104, "num_entries_seen_total", result->num_entries_seen_total);
+	deserializer.ReadPropertyWithDefault<std::priority_queue<std::pair<double, idx_t>>>(105, "reservoir_weights", result->reservoir_weights);
+	return result;
+}
+
 void BoundCaseCheck::Serialize(Serializer &serializer) const {
 	serializer.WritePropertyWithDefault<unique_ptr<Expression>>(100, "when_expr", when_expr);
 	serializer.WritePropertyWithDefault<unique_ptr<Expression>>(101, "then_expr", then_expr);
@@ -222,6 +270,7 @@ void ColumnDefinition::Serialize(Serializer &serializer) const {
 	serializer.WriteProperty<TableColumnType>(103, "category", category);
 	serializer.WriteProperty<duckdb::CompressionType>(104, "compression_type", compression_type);
 	serializer.WritePropertyWithDefault<Value>(105, "comment", comment, Value());
+	serializer.WritePropertyWithDefault<unordered_map<string, string>>(106, "tags", tags, unordered_map<string, string>());
 }
 
 ColumnDefinition ColumnDefinition::Deserialize(Deserializer &deserializer) {
@@ -232,6 +281,7 @@ ColumnDefinition ColumnDefinition::Deserialize(Deserializer &deserializer) {
 	ColumnDefinition result(std::move(name), std::move(type), std::move(expression), category);
 	deserializer.ReadProperty<duckdb::CompressionType>(104, "compression_type", result.compression_type);
 	deserializer.ReadPropertyWithDefault<Value>(105, "comment", result.comment, Value());
+	deserializer.ReadPropertyWithDefault<unordered_map<string, string>>(106, "tags", result.tags, unordered_map<string, string>());
 	return result;
 }
 
@@ -272,12 +322,12 @@ unique_ptr<CommonTableExpressionInfo> CommonTableExpressionInfo::Deserialize(Des
 }
 
 void CommonTableExpressionMap::Serialize(Serializer &serializer) const {
-	serializer.WritePropertyWithDefault<case_insensitive_map_t<unique_ptr<CommonTableExpressionInfo>>>(100, "map", map);
+	serializer.WritePropertyWithDefault<InsertionOrderPreservingMap<unique_ptr<CommonTableExpressionInfo>>>(100, "map", map);
 }
 
 CommonTableExpressionMap CommonTableExpressionMap::Deserialize(Deserializer &deserializer) {
 	CommonTableExpressionMap result;
-	deserializer.ReadPropertyWithDefault<case_insensitive_map_t<unique_ptr<CommonTableExpressionInfo>>>(100, "map", result.map);
+	deserializer.ReadPropertyWithDefault<InsertionOrderPreservingMap<unique_ptr<CommonTableExpressionInfo>>>(100, "map", result.map);
 	return result;
 }
 
@@ -421,6 +471,32 @@ unique_ptr<ReadCSVData> ReadCSVData::Deserialize(Deserializer &deserializer) {
 	return result;
 }
 
+void ReservoirSample::Serialize(Serializer &serializer) const {
+	BlockingSample::Serialize(serializer);
+	serializer.WritePropertyWithDefault<idx_t>(200, "sample_count", sample_count);
+	serializer.WritePropertyWithDefault<unique_ptr<ReservoirChunk>>(201, "reservoir_chunk", reservoir_chunk);
+}
+
+unique_ptr<BlockingSample> ReservoirSample::Deserialize(Deserializer &deserializer) {
+	auto sample_count = deserializer.ReadPropertyWithDefault<idx_t>(200, "sample_count");
+	auto result = duckdb::unique_ptr<ReservoirSample>(new ReservoirSample(sample_count));
+	deserializer.ReadPropertyWithDefault<unique_ptr<ReservoirChunk>>(201, "reservoir_chunk", result->reservoir_chunk);
+	return std::move(result);
+}
+
+void ReservoirSamplePercentage::Serialize(Serializer &serializer) const {
+	BlockingSample::Serialize(serializer);
+	serializer.WriteProperty<double>(200, "sample_percentage", sample_percentage);
+	serializer.WritePropertyWithDefault<idx_t>(201, "reservoir_sample_size", reservoir_sample_size);
+}
+
+unique_ptr<BlockingSample> ReservoirSamplePercentage::Deserialize(Deserializer &deserializer) {
+	auto sample_percentage = deserializer.ReadProperty<double>(200, "sample_percentage");
+	auto result = duckdb::unique_ptr<ReservoirSamplePercentage>(new ReservoirSamplePercentage(sample_percentage));
+	deserializer.ReadPropertyWithDefault<idx_t>(201, "reservoir_sample_size", result->reservoir_sample_size);
+	return std::move(result);
+}
+
 void SampleOptions::Serialize(Serializer &serializer) const {
 	serializer.WriteProperty<Value>(100, "sample_size", sample_size);
 	serializer.WritePropertyWithDefault<bool>(101, "is_percentage", is_percentage);

package/src/duckdb/src/storage/serialization/serialize_parse_info.cpp

@@ -15,6 +15,7 @@
 #include "duckdb/parser/parsed_data/detach_info.hpp"
 #include "duckdb/parser/parsed_data/drop_info.hpp"
 #include "duckdb/parser/parsed_data/load_info.hpp"
+#include "duckdb/parser/parsed_data/update_extensions_info.hpp"
 #include "duckdb/parser/parsed_data/pragma_info.hpp"
 #include "duckdb/parser/parsed_data/transaction_info.hpp"
 #include "duckdb/parser/parsed_data/vacuum_info.hpp"
@@ -56,6 +57,9 @@ unique_ptr<ParseInfo> ParseInfo::Deserialize(Deserializer &deserializer) {
 	case ParseInfoType::TRANSACTION_INFO:
 		result = TransactionInfo::Deserialize(deserializer);
 		break;
+	case ParseInfoType::UPDATE_EXTENSIONS_INFO:
+		result = UpdateExtensionsInfo::Deserialize(deserializer);
+		break;
 	case ParseInfoType::VACUUM_INFO:
 		result = VacuumInfo::Deserialize(deserializer);
 		break;
@@ -256,6 +260,7 @@ void CopyInfo::Serialize(Serializer &serializer) const {
 	serializer.WritePropertyWithDefault<string>(205, "format", format);
 	serializer.WritePropertyWithDefault<string>(206, "file_path", file_path);
 	serializer.WritePropertyWithDefault<case_insensitive_map_t<vector<Value>>>(207, "options", options);
+	serializer.WritePropertyWithDefault<unique_ptr<QueryNode>>(208, "select_statement", select_statement);
 }
 
 unique_ptr<ParseInfo> CopyInfo::Deserialize(Deserializer &deserializer) {
@@ -268,6 +273,7 @@ unique_ptr<ParseInfo> CopyInfo::Deserialize(Deserializer &deserializer) {
 	deserializer.ReadPropertyWithDefault<string>(205, "format", result->format);
 	deserializer.ReadPropertyWithDefault<string>(206, "file_path", result->file_path);
 	deserializer.ReadPropertyWithDefault<case_insensitive_map_t<vector<Value>>>(207, "options", result->options);
+	deserializer.ReadPropertyWithDefault<unique_ptr<QueryNode>>(208, "select_statement", result->select_statement);
 	return std::move(result);
 }
 
@@ -325,6 +331,8 @@ void LoadInfo::Serialize(Serializer &serializer) const {
 	serializer.WritePropertyWithDefault<string>(200, "filename", filename);
 	serializer.WriteProperty<LoadType>(201, "load_type", load_type);
 	serializer.WritePropertyWithDefault<string>(202, "repository", repository);
+	serializer.WritePropertyWithDefault<string>(203, "version", version);
+	serializer.WritePropertyWithDefault<bool>(204, "repo_is_alias", repo_is_alias);
 }
 
 unique_ptr<ParseInfo> LoadInfo::Deserialize(Deserializer &deserializer) {
@@ -332,6 +340,8 @@ unique_ptr<ParseInfo> LoadInfo::Deserialize(Deserializer &deserializer) {
 	deserializer.ReadPropertyWithDefault<string>(200, "filename", result->filename);
 	deserializer.ReadProperty<LoadType>(201, "load_type", result->load_type);
 	deserializer.ReadPropertyWithDefault<string>(202, "repository", result->repository);
+	deserializer.ReadPropertyWithDefault<string>(203, "version", result->version);
+	deserializer.ReadPropertyWithDefault<bool>(204, "repo_is_alias", result->repo_is_alias);
 	return std::move(result);
 }
 
@@ -463,6 +473,17 @@ unique_ptr<ParseInfo> TransactionInfo::Deserialize(Deserializer &deserializer) {
 	return std::move(result);
 }
 
+void UpdateExtensionsInfo::Serialize(Serializer &serializer) const {
+	ParseInfo::Serialize(serializer);
+	serializer.WritePropertyWithDefault<vector<string>>(200, "extensions_to_update", extensions_to_update);
+}
+
+unique_ptr<ParseInfo> UpdateExtensionsInfo::Deserialize(Deserializer &deserializer) {
+	auto result = duckdb::unique_ptr<UpdateExtensionsInfo>(new UpdateExtensionsInfo());
+	deserializer.ReadPropertyWithDefault<vector<string>>(200, "extensions_to_update", result->extensions_to_update);
+	return std::move(result);
+}
+
 void VacuumInfo::Serialize(Serializer &serializer) const {
 	ParseInfo::Serialize(serializer);
 	serializer.WriteProperty<VacuumOptions>(200, "options", options);

package/src/duckdb/src/storage/serialization/serialize_tableref.cpp

@@ -26,6 +26,9 @@ unique_ptr<TableRef> TableRef::Deserialize(Deserializer &deserializer) {
 	case TableReferenceType::BASE_TABLE:
 		result = BaseTableRef::Deserialize(deserializer);
 		break;
+	case TableReferenceType::COLUMN_DATA:
+		result = ColumnDataRef::Deserialize(deserializer);
+		break;
 	case TableReferenceType::EMPTY_FROM:
 		result = EmptyTableRef::Deserialize(deserializer);
 		break;
@@ -73,6 +76,19 @@ unique_ptr<TableRef> BaseTableRef::Deserialize(Deserializer &deserializer) {
 	return std::move(result);
 }
 
+void ColumnDataRef::Serialize(Serializer &serializer) const {
+	TableRef::Serialize(serializer);
+	serializer.WritePropertyWithDefault<vector<string>>(200, "expected_names", expected_names);
+	serializer.WritePropertyWithDefault<optionally_owned_ptr<ColumnDataCollection>>(202, "collection", collection);
+}
+
+unique_ptr<TableRef> ColumnDataRef::Deserialize(Deserializer &deserializer) {
+	auto expected_names = deserializer.ReadPropertyWithDefault<vector<string>>(200, "expected_names");
+	auto collection = deserializer.ReadPropertyWithDefault<optionally_owned_ptr<ColumnDataCollection>>(202, "collection");
+	auto result = duckdb::unique_ptr<ColumnDataRef>(new ColumnDataRef(std::move(expected_names), std::move(collection)));
+	return std::move(result);
+}
+
 void EmptyTableRef::Serialize(Serializer &serializer) const {
 	TableRef::Serialize(serializer);
 }

package/src/duckdb/src/storage/serialization/serialize_types.cpp

@@ -12,11 +12,13 @@ namespace duckdb {
 void ExtraTypeInfo::Serialize(Serializer &serializer) const {
 	serializer.WriteProperty<ExtraTypeInfoType>(100, "type", type);
 	serializer.WritePropertyWithDefault<string>(101, "alias", alias);
+	serializer.WritePropertyWithDefault<vector<Value>>(102, "modifiers", modifiers, vector<Value>());
 }
 
 shared_ptr<ExtraTypeInfo> ExtraTypeInfo::Deserialize(Deserializer &deserializer) {
 	auto type = deserializer.ReadProperty<ExtraTypeInfoType>(100, "type");
 	auto alias = deserializer.ReadPropertyWithDefault<string>(101, "alias");
+	auto modifiers = deserializer.ReadPropertyWithDefault<vector<Value>>(102, "modifiers", vector<Value>());
 	shared_ptr<ExtraTypeInfo> result;
 	switch (type) {
 	case ExtraTypeInfoType::AGGREGATE_STATE_TYPE_INFO:
@@ -35,7 +37,7 @@ shared_ptr<ExtraTypeInfo> ExtraTypeInfo::Deserialize(Deserializer &deserializer)
 		result = EnumTypeInfo::Deserialize(deserializer);
 		break;
 	case ExtraTypeInfoType::GENERIC_TYPE_INFO:
-		result = make_shared<ExtraTypeInfo>(type);
+		result = make_shared_ptr<ExtraTypeInfo>(type);
 		break;
 	case ExtraTypeInfoType::INTEGER_LITERAL_TYPE_INFO:
 		result = IntegerLiteralTypeInfo::Deserialize(deserializer);
@@ -58,6 +60,7 @@ shared_ptr<ExtraTypeInfo> ExtraTypeInfo::Deserialize(Deserializer &deserializer)
 		throw SerializationException("Unsupported type for deserialization of ExtraTypeInfo!");
 	}
 	result->alias = std::move(alias);
+	result->modifiers = std::move(modifiers);
 	return result;
 }
 
@@ -164,6 +167,7 @@ void UserTypeInfo::Serialize(Serializer &serializer) const {
 	serializer.WritePropertyWithDefault<string>(200, "user_type_name", user_type_name);
 	serializer.WritePropertyWithDefault<string>(201, "catalog", catalog, string());
 	serializer.WritePropertyWithDefault<string>(202, "schema", schema, string());
+	serializer.WritePropertyWithDefault<vector<Value>>(203, "user_type_modifiers", user_type_modifiers, vector<Value>());
 }
 
 shared_ptr<ExtraTypeInfo> UserTypeInfo::Deserialize(Deserializer &deserializer) {
@@ -171,6 +175,7 @@ shared_ptr<ExtraTypeInfo> UserTypeInfo::Deserialize(Deserializer &deserializer)
 	deserializer.ReadPropertyWithDefault<string>(200, "user_type_name", result->user_type_name);
 	deserializer.ReadPropertyWithDefault<string>(201, "catalog", result->catalog, string());
 	deserializer.ReadPropertyWithDefault<string>(202, "schema", result->schema, string());
+	deserializer.ReadPropertyWithDefault<vector<Value>>(203, "user_type_modifiers", result->user_type_modifiers, vector<Value>());
 	return std::move(result);
 }
 

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -62,7 +62,7 @@ MainHeader MainHeader::Read(ReadStream &source) {
 	if (header.version_number != VERSION_NUMBER) {
 		auto version = GetDuckDBVersion(header.version_number);
 		string version_text;
-		if (version) {
+		if (!version.empty()) {
 			// known version
 			version_text = "DuckDB version " + string(version);
 		} else {
@@ -195,8 +195,8 @@ void SingleFileBlockManager::CreateNewDatabase() {
 	DatabaseHeader h1;
 	// header 1
 	h1.iteration = 0;
-	h1.meta_block = INVALID_BLOCK;
-	h1.free_list = INVALID_BLOCK;
+	h1.meta_block = idx_t(INVALID_BLOCK);
+	h1.free_list = idx_t(INVALID_BLOCK);
 	h1.block_count = 0;
 	h1.block_size = Storage::BLOCK_ALLOC_SIZE;
 	h1.vector_size = STANDARD_VECTOR_SIZE;
@@ -205,8 +205,8 @@ void SingleFileBlockManager::CreateNewDatabase() {
 	// header 2
 	DatabaseHeader h2;
 	h2.iteration = 0;
-	h2.meta_block = INVALID_BLOCK;
-	h2.free_list = INVALID_BLOCK;
+	h2.meta_block = idx_t(INVALID_BLOCK);
+	h2.free_list = idx_t(INVALID_BLOCK);
 	h2.block_count = 0;
 	h2.block_size = Storage::BLOCK_ALLOC_SIZE;
 	h2.vector_size = STANDARD_VECTOR_SIZE;
@@ -228,7 +228,7 @@ void SingleFileBlockManager::LoadExistingDatabase() {
 	handle = fs.OpenFile(path, flags);
 	if (!handle) {
 		// this can only happen in read-only mode - as that is when we set FILE_FLAGS_NULL_IF_NOT_EXISTS
-		throw CatalogException("Cannot open database \"%s\" in read-only mode: database does not exist", path);
+		throw IOException("Cannot open database \"%s\" in read-only mode: database does not exist", path);
 	}
 
 	MainHeader::CheckMagicBytes(*handle);
@@ -266,8 +266,9 @@ void SingleFileBlockManager::ReadAndChecksum(FileBuffer &block, uint64_t locatio
 
 	// verify the checksum
 	if (stored_checksum != computed_checksum) {
-		throw IOException("Corrupt database file: computed checksum %llu does not match stored checksum %llu in block",
-		                  computed_checksum, stored_checksum);
+		throw IOException("Corrupt database file: computed checksum %llu does not match stored checksum %llu in block "
+		                  "at location %llu",
+		                  computed_checksum, stored_checksum, location);
 	}
 }
 
@@ -283,7 +284,7 @@ void SingleFileBlockManager::Initialize(DatabaseHeader &header) {
 	free_list_id = header.free_list;
 	meta_block = header.meta_block;
 	iteration_count = header.iteration;
-	max_block = header.block_count;
+	max_block = NumericCast<block_id_t>(header.block_count);
 }
 
 void SingleFileBlockManager::LoadFreeList() {
@@ -386,7 +387,7 @@ idx_t SingleFileBlockManager::GetMetaBlock() {
 
 idx_t SingleFileBlockManager::TotalBlocks() {
 	lock_guard<mutex> lock(block_lock);
-	return max_block;
+	return NumericCast<idx_t>(max_block);
 }
 
 idx_t SingleFileBlockManager::FreeBlocks() {
@@ -413,12 +414,12 @@ unique_ptr<Block> SingleFileBlockManager::CreateBlock(block_id_t block_id, FileB
 void SingleFileBlockManager::Read(Block &block) {
 	D_ASSERT(block.id >= 0);
 	D_ASSERT(std::find(free_list.begin(), free_list.end(), block.id) == free_list.end());
-	ReadAndChecksum(block, BLOCK_START + block.id * Storage::BLOCK_ALLOC_SIZE);
+	ReadAndChecksum(block, BLOCK_START + NumericCast<idx_t>(block.id) * Storage::BLOCK_ALLOC_SIZE);
 }
 
 void SingleFileBlockManager::Write(FileBuffer &buffer, block_id_t block_id) {
 	D_ASSERT(block_id >= 0);
-	ChecksumAndWrite(buffer, BLOCK_START + block_id * Storage::BLOCK_ALLOC_SIZE);
+	ChecksumAndWrite(buffer, BLOCK_START + NumericCast<idx_t>(block_id) * Storage::BLOCK_ALLOC_SIZE);
 }
 
 void SingleFileBlockManager::Truncate() {
@@ -440,7 +441,7 @@ void SingleFileBlockManager::Truncate() {
 	// truncate the file
 	free_list.erase(free_list.lower_bound(max_block), free_list.end());
 	newly_freed_list.erase(newly_freed_list.lower_bound(max_block), newly_freed_list.end());
-	handle->Truncate(BLOCK_START + max_block * Storage::BLOCK_ALLOC_SIZE);
+	handle->Truncate(NumericCast<int64_t>(BLOCK_START + NumericCast<idx_t>(max_block) * Storage::BLOCK_ALLOC_SIZE));
 }
 
 vector<MetadataHandle> SingleFileBlockManager::GetFreeListBlocks() {
@@ -488,15 +489,17 @@ protected:
 };
 
 void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
-	// set the iteration count
-	header.iteration = ++iteration_count;
-
 	auto free_list_blocks = GetFreeListBlocks();
 
 	// now handle the free list
 	auto &metadata_manager = GetMetadataManager();
 	// add all modified blocks to the free list: they can now be written to again
 	metadata_manager.MarkBlocksAsModified();
+
+	lock_guard<mutex> lock(block_lock);
+	// set the iteration count
+	header.iteration = ++iteration_count;
+
 	for (auto &block : modified_blocks) {
 		free_list.insert(block);
 		newly_freed_list.insert(block);
@@ -529,7 +532,7 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 		header.free_list = DConstants::INVALID_INDEX;
 	}
 	metadata_manager.Flush();
-	header.block_count = max_block;
+	header.block_count = NumericCast<idx_t>(max_block);
 
 	auto &config = DBConfig::Get(db);
 	if (config.options.checkpoint_abort == CheckpointAbort::DEBUG_ABORT_AFTER_FREE_LIST_WRITE) {
@@ -569,8 +572,8 @@ void SingleFileBlockManager::TrimFreeBlocks() {
 			// We are now one too far.
 			--itr;
 			// Trim the range.
-			handle->Trim(BLOCK_START + (first * Storage::BLOCK_ALLOC_SIZE),
-			             (last + 1 - first) * Storage::BLOCK_ALLOC_SIZE);
+			handle->Trim(BLOCK_START + (NumericCast<idx_t>(first) * Storage::BLOCK_ALLOC_SIZE),
+			             NumericCast<idx_t>(last + 1 - first) * Storage::BLOCK_ALLOC_SIZE);
 		}
 	}
 	newly_freed_list.clear();

package/src/duckdb/src/storage/standard_buffer_manager.cpp

@@ -42,16 +42,18 @@ unique_ptr<FileBuffer> StandardBufferManager::ConstructManagedBuffer(idx_t size,
 }
 
 void StandardBufferManager::SetTemporaryDirectory(const string &new_dir) {
-	if (temp_directory_handle) {
+	lock_guard<mutex> guard(temporary_directory.lock);
+	if (temporary_directory.handle) {
 		throw NotImplementedException("Cannot switch temporary directory after the current one has been used");
 	}
-	this->temp_directory = new_dir;
+	temporary_directory.path = new_dir;
 }
 
 StandardBufferManager::StandardBufferManager(DatabaseInstance &db, string tmp)
-    : BufferManager(), db(db), buffer_pool(db.GetBufferPool()), temp_directory(std::move(tmp)),
-      temporary_id(MAXIMUM_BLOCK), buffer_allocator(BufferAllocatorAllocate, BufferAllocatorFree,
-                                                    BufferAllocatorRealloc, make_uniq<BufferAllocatorData>(*this)) {
+    : BufferManager(), db(db), buffer_pool(db.GetBufferPool()), temporary_id(MAXIMUM_BLOCK),
+      buffer_allocator(BufferAllocatorAllocate, BufferAllocatorFree, BufferAllocatorRealloc,
+                       make_uniq<BufferAllocatorData>(*this)) {
+	temporary_directory.path = std::move(tmp);
 	temp_block_manager = make_uniq<InMemoryBlockManager>(*this);
 	for (idx_t i = 0; i < MEMORY_TAG_COUNT; i++) {
 		evicted_data_per_tag[i] = 0;
@@ -76,6 +78,22 @@ idx_t StandardBufferManager::GetMaxMemory() const {
 	return buffer_pool.GetMaxMemory();
 }
 
+idx_t StandardBufferManager::GetUsedSwap() {
+	lock_guard<mutex> guard(temporary_directory.lock);
+	if (!temporary_directory.handle) {
+		return 0;
+	}
+	return temporary_directory.handle->GetTempFile().GetTotalUsedSpaceInBytes();
+}
+
+optional_idx StandardBufferManager::GetMaxSwap() const {
+	lock_guard<mutex> guard(temporary_directory.lock);
+	if (!temporary_directory.handle) {
+		return optional_idx();
+	}
+	return temporary_directory.handle->GetTempFile().GetMaxSwapSpace();
+}
+
 template <typename... ARGS>
 TempBufferPoolReservation StandardBufferManager::EvictBlocksOrThrow(MemoryTag tag, idx_t memory_delta,
                                                                     unique_ptr<FileBuffer> *buffer, ARGS... args) {
@@ -98,8 +116,8 @@ shared_ptr<BlockHandle> StandardBufferManager::RegisterSmallMemory(idx_t block_s
 	auto buffer = ConstructManagedBuffer(block_size, nullptr, FileBufferType::TINY_BUFFER);
 
 	// create a new block pointer for this block
-	auto result = make_shared<BlockHandle>(*temp_block_manager, ++temporary_id, MemoryTag::BASE_TABLE,
-	                                       std::move(buffer), false, block_size, std::move(reservation));
+	auto result = make_shared_ptr<BlockHandle>(*temp_block_manager, ++temporary_id, MemoryTag::BASE_TABLE,
+	                                           std::move(buffer), false, block_size, std::move(reservation));
 #ifdef DUCKDB_DEBUG_DESTROY_BLOCKS
 	// Initialize the memory with garbage data
 	WriteGarbageIntoBuffer(*result->buffer);
@@ -118,8 +136,8 @@ shared_ptr<BlockHandle> StandardBufferManager::RegisterMemory(MemoryTag tag, idx
 	auto buffer = ConstructManagedBuffer(block_size, std::move(reusable_buffer));
 
 	// create a new block pointer for this block
-	return make_shared<BlockHandle>(*temp_block_manager, ++temporary_id, tag, std::move(buffer), can_destroy,
-	                                alloc_size, std::move(res));
+	return make_shared_ptr<BlockHandle>(*temp_block_manager, ++temporary_id, tag, std::move(buffer), can_destroy,
+	                                    alloc_size, std::move(res));
 }
 
 BufferHandle StandardBufferManager::Allocate(MemoryTag tag, idx_t block_size, bool can_destroy,
@@ -142,7 +160,7 @@ void StandardBufferManager::ReAllocate(shared_ptr<BlockHandle> &handle, idx_t bl
 	D_ASSERT(handle->memory_usage == handle->memory_charge.size);
 
 	auto req = handle->buffer->CalculateMemory(block_size);
-	int64_t memory_delta = NumericCast<int64_t>(req.alloc_size) - handle->memory_usage;
+	int64_t memory_delta = NumericCast<int64_t>(req.alloc_size) - NumericCast<int64_t>(handle->memory_usage);
 
 	if (memory_delta == 0) {
 		return;
@@ -150,10 +168,10 @@ void StandardBufferManager::ReAllocate(shared_ptr<BlockHandle> &handle, idx_t bl
 		// evict blocks until we have space to resize this block
 		// unlock the handle lock during the call to EvictBlocksOrThrow
 		lock.unlock();
-		auto reservation =
-		    EvictBlocksOrThrow(handle->tag, memory_delta, nullptr, "failed to resize block from %s to %s%s",
-		                       StringUtil::BytesToHumanReadableString(handle->memory_usage),
-		                       StringUtil::BytesToHumanReadableString(req.alloc_size));
+		auto reservation = EvictBlocksOrThrow(handle->tag, NumericCast<idx_t>(memory_delta), nullptr,
+		                                      "failed to resize block from %s to %s%s",
+		                                      StringUtil::BytesToHumanReadableString(handle->memory_usage),
+		                                      StringUtil::BytesToHumanReadableString(req.alloc_size));
 		lock.lock();
 
 		// EvictBlocks decrements 'current_memory' for us.
@@ -199,10 +217,10 @@ BufferHandle StandardBufferManager::Pin(shared_ptr<BlockHandle> &handle) {
 	auto buf = handle->Load(handle, std::move(reusable_buffer));
 	handle->memory_charge = std::move(reservation);
 	// In the case of a variable sized block, the buffer may be smaller than a full block.
-	int64_t delta = handle->buffer->AllocSize() - handle->memory_usage;
+	int64_t delta = NumericCast<int64_t>(handle->buffer->AllocSize()) - NumericCast<int64_t>(handle->memory_usage);
 	if (delta) {
 		D_ASSERT(delta < 0);
-		handle->memory_usage += delta;
+		handle->memory_usage += NumericCast<idx_t>(delta);
 		handle->memory_charge.Resize(handle->memory_usage);
 	}
 	D_ASSERT(handle->memory_usage == handle->buffer->AllocSize());
@@ -248,10 +266,19 @@ void StandardBufferManager::Unpin(shared_ptr<BlockHandle> &handle) {
 	}
 }
 
-void StandardBufferManager::SetLimit(idx_t limit) {
+void StandardBufferManager::SetMemoryLimit(idx_t limit) {
 	buffer_pool.SetLimit(limit, InMemoryWarning());
 }
 
+void StandardBufferManager::SetSwapLimit(optional_idx limit) {
+	lock_guard<mutex> guard(temporary_directory.lock);
+	if (temporary_directory.handle) {
+		temporary_directory.handle->GetTempFile().SetMaxSwapSpace(limit);
+	} else {
+		temporary_directory.maximum_swap_space = limit;
+	}
+}
+
 vector<MemoryInformation> StandardBufferManager::GetMemoryUsageInfo() const {
 	vector<MemoryInformation> result;
 	for (idx_t k = 0; k < MEMORY_TAG_COUNT; k++) {
@@ -275,19 +302,20 @@ unique_ptr<FileBuffer> StandardBufferManager::ReadTemporaryBufferInternal(Buffer
 
 string StandardBufferManager::GetTemporaryPath(block_id_t id) {
 	auto &fs = FileSystem::GetFileSystem(db);
-	return fs.JoinPath(temp_directory, "duckdb_temp_block-" + to_string(id) + ".block");
+	return fs.JoinPath(temporary_directory.path, "duckdb_temp_block-" + to_string(id) + ".block");
 }
 
 void StandardBufferManager::RequireTemporaryDirectory() {
-	if (temp_directory.empty()) {
+	if (temporary_directory.path.empty()) {
 		throw InvalidInputException(
 		    "Out-of-memory: cannot write buffer because no temporary directory is specified!\nTo enable "
 		    "temporary buffer eviction set a temporary directory using PRAGMA temp_directory='/path/to/tmp.tmp'");
 	}
-	lock_guard<mutex> temp_handle_guard(temp_handle_lock);
-	if (!temp_directory_handle) {
+	lock_guard<mutex> guard(temporary_directory.lock);
+	if (!temporary_directory.handle) {
 		// temp directory has not been created yet: initialize it
-		temp_directory_handle = make_uniq<TemporaryDirectoryHandle>(db, temp_directory);
+		temporary_directory.handle =
+		    make_uniq<TemporaryDirectoryHandle>(db, temporary_directory.path, temporary_directory.maximum_swap_space);
 	}
 }
 
@@ -295,7 +323,7 @@ void StandardBufferManager::WriteTemporaryBuffer(MemoryTag tag, block_id_t block
 	RequireTemporaryDirectory();
 	if (buffer.size == Storage::BLOCK_SIZE) {
 		evicted_data_per_tag[uint8_t(tag)] += Storage::BLOCK_SIZE;
-		temp_directory_handle->GetTempFile().WriteTemporaryBuffer(block_id, buffer);
+		temporary_directory.handle->GetTempFile().WriteTemporaryBuffer(block_id, buffer);
 		return;
 	}
 	evicted_data_per_tag[uint8_t(tag)] += buffer.size;
@@ -311,11 +339,11 @@ void StandardBufferManager::WriteTemporaryBuffer(MemoryTag tag, block_id_t block
 
 unique_ptr<FileBuffer> StandardBufferManager::ReadTemporaryBuffer(MemoryTag tag, block_id_t id,
                                                                   unique_ptr<FileBuffer> reusable_buffer) {
-	D_ASSERT(!temp_directory.empty());
-	D_ASSERT(temp_directory_handle.get());
-	if (temp_directory_handle->GetTempFile().HasTemporaryBuffer(id)) {
+	D_ASSERT(!temporary_directory.path.empty());
+	D_ASSERT(temporary_directory.handle.get());
+	if (temporary_directory.handle->GetTempFile().HasTemporaryBuffer(id)) {
 		evicted_data_per_tag[uint8_t(tag)] -= Storage::BLOCK_SIZE;
-		return temp_directory_handle->GetTempFile().ReadTemporaryBuffer(id, std::move(reusable_buffer));
+		return temporary_directory.handle->GetTempFile().ReadTemporaryBuffer(id, std::move(reusable_buffer));
 	}
 	idx_t block_size;
 	// open the temporary file and read the size
@@ -334,20 +362,20 @@ unique_ptr<FileBuffer> StandardBufferManager::ReadTemporaryBuffer(MemoryTag tag,
 }
 
 void StandardBufferManager::DeleteTemporaryFile(block_id_t id) {
-	if (temp_directory.empty()) {
+	if (temporary_directory.path.empty()) {
 		// no temporary directory specified: nothing to delete
 		return;
 	}
 	{
-		lock_guard<mutex> temp_handle_guard(temp_handle_lock);
-		if (!temp_directory_handle) {
+		lock_guard<mutex> guard(temporary_directory.lock);
+		if (!temporary_directory.handle) {
 			// temporary directory was not initialized yet: nothing to delete
 			return;
 		}
 	}
 	// check if we should delete the file from the shared pool of files, or from the general file system
-	if (temp_directory_handle->GetTempFile().HasTemporaryBuffer(id)) {
-		temp_directory_handle->GetTempFile().DeleteTemporaryBuffer(id);
+	if (temporary_directory.handle->GetTempFile().HasTemporaryBuffer(id)) {
+		temporary_directory.handle->GetTempFile().DeleteTemporaryBuffer(id);
 		return;
 	}
 	auto &fs = FileSystem::GetFileSystem(db);
@@ -358,22 +386,22 @@ void StandardBufferManager::DeleteTemporaryFile(block_id_t id) {
 }
 
 bool StandardBufferManager::HasTemporaryDirectory() const {
-	return !temp_directory.empty();
+	return !temporary_directory.path.empty();
 }
 
 vector<TemporaryFileInformation> StandardBufferManager::GetTemporaryFiles() {
 	vector<TemporaryFileInformation> result;
-	if (temp_directory.empty()) {
+	if (temporary_directory.path.empty()) {
 		return result;
 	}
 	{
-		lock_guard<mutex> temp_handle_guard(temp_handle_lock);
-		if (temp_directory_handle) {
-			result = temp_directory_handle->GetTempFile().GetTemporaryFiles();
+		lock_guard<mutex> temp_handle_guard(temporary_directory.lock);
+		if (temporary_directory.handle) {
+			result = temporary_directory.handle->GetTempFile().GetTemporaryFiles();
 		}
 	}
 	auto &fs = FileSystem::GetFileSystem(db);
-	fs.ListFiles(temp_directory, [&](const string &name, bool is_dir) {
+	fs.ListFiles(temporary_directory.path, [&](const string &name, bool is_dir) {
 		if (is_dir) {
 			return;
 		}
@@ -383,7 +411,7 @@ vector<TemporaryFileInformation> StandardBufferManager::GetTemporaryFiles() {
 		TemporaryFileInformation info;
 		info.path = name;
 		auto handle = fs.OpenFile(name, FileFlags::FILE_FLAGS_READ);
-		info.size = fs.GetFileSize(*handle);
+		info.size = NumericCast<idx_t>(fs.GetFileSize(*handle));
 		handle.reset();
 		result.push_back(info);
 	});
@@ -391,7 +419,7 @@ vector<TemporaryFileInformation> StandardBufferManager::GetTemporaryFiles() {
 }
 
 const char *StandardBufferManager::InMemoryWarning() {
-	if (!temp_directory.empty()) {
+	if (!temporary_directory.path.empty()) {
 		return "";
 	}
 	return "\nDatabase is launched in in-memory mode and no temporary directory is specified."