duckdb 0.10.2-dev0.0 → 0.10.2-dev3.0

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -229,7 +229,7 @@ MetadataManager &SingleFileCheckpointReader::GetMetadataManager() {
 	return storage.block_manager->GetMetadataManager();
 }
 
-void SingleFileCheckpointReader::LoadFromStorage() {
+void SingleFileCheckpointReader::LoadFromStorage(optional_ptr<ClientContext> context) {
 	auto &block_manager = *storage.block_manager;
 	auto &metadata_manager = GetMetadataManager();
 	MetaBlockPointer meta_block(block_manager.GetMetaBlock(), 0);
@@ -238,13 +238,20 @@ void SingleFileCheckpointReader::LoadFromStorage() {
 		return;
 	}
 
-	Connection con(storage.GetDatabase());
-	con.BeginTransaction();
-	// create the MetadataReader to read from the storage
-	MetadataReader reader(metadata_manager, meta_block);
-	//	reader.SetContext(*con.context);
-	LoadCheckpoint(*con.context, reader);
-	con.Commit();
+	if (context) {
+		// create the MetadataReader to read from the storage
+		MetadataReader reader(metadata_manager, meta_block);
+		//	reader.SetContext(*con.context);
+		LoadCheckpoint(*context, reader);
+	} else {
+		Connection con(storage.GetDatabase());
+		con.BeginTransaction();
+		// create the MetadataReader to read from the storage
+		MetadataReader reader(metadata_manager, meta_block);
+		//	reader.SetContext(*con.context);
+		LoadCheckpoint(*con.context, reader);
+		con.Commit();
+	}
 }
 
 void CheckpointWriter::WriteEntry(CatalogEntry &entry, Serializer &serializer) {

package/src/duckdb/src/storage/compression/bitpacking.cpp

@@ -514,7 +514,8 @@ public:
 		auto base_ptr = handle.Ptr();
 
 		// Compact the segment by moving the metadata next to the data.
-		idx_t metadata_offset = AlignValue(data_ptr - base_ptr);
+		idx_t unaligned_offset = data_ptr - base_ptr;
+		idx_t metadata_offset = AlignValue(unaligned_offset);
 		idx_t metadata_size = base_ptr + Storage::BLOCK_SIZE - metadata_ptr;
 		idx_t total_segment_size = metadata_offset + metadata_size;
 
@@ -523,6 +524,10 @@ public:
 			throw InternalException("Error in bitpacking size calculation");
 		}
 
+		if (unaligned_offset != metadata_offset) {
+			// zero initialize any padding bits
+			memset(base_ptr + unaligned_offset, 0, metadata_offset - unaligned_offset);
+		}
 		memmove(base_ptr + metadata_offset, metadata_ptr, metadata_size);
 
 		// Store the offset of the metadata of the first group (which is at the highest address).

package/src/duckdb/src/storage/compression/string_uncompressed.cpp

@@ -406,7 +406,7 @@ string_location_t UncompressedStringStorage::FetchStringLocation(StringDictionar
 	D_ASSERT(dict_offset >= -1 * int32_t(Storage::BLOCK_SIZE) && dict_offset <= int32_t(Storage::BLOCK_SIZE));
 	if (dict_offset < 0) {
 		string_location_t result;
-		ReadStringMarker(baseptr + dict.end - (-1 * dict_offset), result.block_id, result.offset);
+		ReadStringMarker(baseptr + dict.end - idx_t(-1 * dict_offset), result.block_id, result.offset);
 		return result;
 	} else {
 		return string_location_t(INVALID_BLOCK, dict_offset);

package/src/duckdb/src/storage/data_pointer.cpp

@@ -9,7 +9,7 @@ namespace duckdb {
 unique_ptr<ColumnSegmentState> ColumnSegmentState::Deserialize(Deserializer &deserializer) {
 	auto compression_type = deserializer.Get<CompressionType>();
 	auto &db = deserializer.Get<DatabaseInstance &>();
-	auto &type = deserializer.Get<LogicalType &>();
+	auto &type = deserializer.Get<const LogicalType &>();
 	auto compression_function = DBConfig::GetConfig(db).GetCompressionFunction(compression_type, type.InternalType());
 	if (!compression_function || !compression_function->deserialize_state) {
 		throw SerializationException("Deserializing a ColumnSegmentState but could not find deserialize method");

package/src/duckdb/src/storage/data_table.cpp

@@ -34,8 +34,8 @@ DataTableInfo::DataTableInfo(AttachedDatabase &db, shared_ptr<TableIOManager> ta
       table(std::move(table)) {
 }
 
-void DataTableInfo::InitializeIndexes(ClientContext &context) {
-	indexes.InitializeIndexes(context, *this);
+void DataTableInfo::InitializeIndexes(ClientContext &context, bool throw_on_failure) {
+	indexes.InitializeIndexes(context, *this, throw_on_failure);
 }
 
 bool DataTableInfo::IsTemporary() const {
@@ -89,8 +89,8 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, idx_t removed_co
 		column_definitions.emplace_back(column_def.Copy());
 	}
 
-	// try to initialize unknown indexes
 	info->InitializeIndexes(context);
+
 	// first check if there are any indexes that exist that point to the removed column
 	info->indexes.Scan([&](Index &index) {
 		for (auto &column_id : index.column_ids) {
@@ -137,6 +137,8 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, unique_ptr<Bound
 		column_definitions.emplace_back(column_def.Copy());
 	}
 
+	info->InitializeIndexes(context);
+
 	// Verify the new constraint against current persistent/local data
 	VerifyNewConstraint(context, parent, constraint.get());
 
@@ -155,7 +157,7 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, idx_t changed_id
 	for (auto &column_def : parent.column_definitions) {
 		column_definitions.emplace_back(column_def.Copy());
 	}
-	// try to initialize unknown indexes
+
 	info->InitializeIndexes(context);
 
 	// first check if there are any indexes that exist that point to the changed column
@@ -858,7 +860,9 @@ void DataTable::RevertAppend(idx_t start_row, idx_t count) {
 				row_data[i] = current_row_base + i;
 			}
 			info->indexes.Scan([&](Index &index) {
-				index.Delete(chunk, row_identifiers);
+				if (!index.IsUnknown()) {
+					index.Delete(chunk, row_identifiers);
+				}
 				return false;
 			});
 			current_row_base += chunk.size();
@@ -868,7 +872,9 @@ void DataTable::RevertAppend(idx_t start_row, idx_t count) {
 	// we need to vacuum the indexes to remove any buffers that are now empty
 	// due to reverting the appends
 	info->indexes.Scan([&](Index &index) {
-		index.Vacuum();
+		if (!index.IsUnknown()) {
+			index.Vacuum();
+		}
 		return false;
 	});
 
@@ -1000,6 +1006,8 @@ idx_t DataTable::Delete(TableCatalogEntry &table, ClientContext &context, Vector
 		return 0;
 	}
 
+	info->InitializeIndexes(context, true);
+
 	auto &transaction = DuckTransaction::Get(context, db);
 	auto &local_storage = LocalStorage::Get(transaction);
 	bool has_delete_constraints = TableHasDeleteConstraints(table);
@@ -1157,6 +1165,9 @@ void DataTable::Update(TableCatalogEntry &table, ClientContext &context, Vector
 		throw TransactionException("Transaction conflict: cannot update a table that has been altered!");
 	}
 
+	// check that there are no unknown indexes
+	info->InitializeIndexes(context, true);
+
 	// first verify that no constraints are violated
 	VerifyUpdateConstraints(context, table, updates, column_ids);
 
@@ -1244,7 +1255,7 @@ void DataTable::Checkpoint(TableDataWriter &writer, Serializer &serializer) {
 	//   row-group pointers
 	//   table pointer
 	//   index data
-	writer.FinalizeTable(std::move(global_stats), info.get(), serializer);
+	writer.FinalizeTable(global_stats, info.get(), serializer);
 }
 
 void DataTable::CommitDropColumn(idx_t index) {

package/src/duckdb/src/storage/local_storage.cpp

@@ -197,7 +197,11 @@ void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppen
 		// we need to vacuum the indexes to remove any buffers that are now empty
 		// due to reverting the appends
 		table.info->indexes.Scan([&](Index &index) {
-			index.Vacuum();
+			try {
+				index.Vacuum();
+			} catch (std::exception &ex) { // LCOV_EXCL_START
+				error = ErrorData(ex);
+			} // LCOV_EXCL_STOP
 			return false;
 		});
 		error.Throw();
@@ -354,6 +358,7 @@ bool LocalStorage::NextParallelScan(ClientContext &context, DataTable &table, Pa
 }
 
 void LocalStorage::InitializeAppend(LocalAppendState &state, DataTable &table) {
+	table.info->InitializeIndexes(context);
 	state.storage = &table_manager.GetOrCreateStorage(table);
 	state.storage->row_groups->InitializeAppend(TransactionData(transaction), state.append_state);
 }
@@ -445,6 +450,8 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
 	}
 	idx_t append_count = storage.row_groups->GetTotalRows() - storage.deleted_rows;
 
+	table.info->InitializeIndexes(context);
+
 	TableAppendState append_state;
 	table.AppendLock(append_state);
 	transaction.PushAppend(table, append_state.row_start, append_count);
@@ -470,9 +477,6 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
 		storage.AppendToIndexes(transaction, append_state, append_count, true);
 	}
 
-	// try to initialize any unknown indexes
-	table.info->InitializeIndexes(context);
-
 	// possibly vacuum any excess index data
 	table.info->indexes.Scan([&](Index &index) {
 		index.Vacuum();
@@ -575,7 +579,6 @@ TableIndexList &LocalStorage::GetIndexes(DataTable &table) {
 	if (!storage) {
 		throw InternalException("LocalStorage::GetIndexes - local storage not found");
 	}
-	table.info->InitializeIndexes(context);
 	return storage->indexes;
 }
 

package/src/duckdb/src/storage/magic_bytes.cpp

@@ -4,13 +4,14 @@
 
 namespace duckdb {
 
-DataFileType MagicBytes::CheckMagicBytes(FileSystem *fs_p, const string &path) {
-	LocalFileSystem lfs;
-	FileSystem &fs = fs_p ? *fs_p : lfs;
-	if (!fs.FileExists(path)) {
+DataFileType MagicBytes::CheckMagicBytes(FileSystem &fs, const string &path) {
+	if (path.empty() || path == IN_MEMORY_PATH) {
+		return DataFileType::DUCKDB_FILE;
+	}
+	auto handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_NULL_IF_NOT_EXISTS);
+	if (!handle) {
 		return DataFileType::FILE_DOES_NOT_EXIST;
 	}
-	auto handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ);
 
 	constexpr const idx_t MAGIC_BYTES_READ_SIZE = 16;
 	char buffer[MAGIC_BYTES_READ_SIZE];

package/src/duckdb/src/storage/partial_block_manager.cpp

@@ -95,7 +95,7 @@ bool PartialBlockManager::GetPartialBlock(idx_t segment_size, unique_ptr<Partial
 	return true;
 }
 
-void PartialBlockManager::RegisterPartialBlock(PartialBlockAllocation &&allocation) {
+void PartialBlockManager::RegisterPartialBlock(PartialBlockAllocation allocation) {
 	auto &state = allocation.partial_block->state;
 	D_ASSERT(checkpoint_type != CheckpointType::FULL_CHECKPOINT || state.block_id >= 0);
 	if (state.block_use_count < max_use_count) {

package/src/duckdb/src/storage/serialization/serialize_logical_operator.cpp

@@ -43,6 +43,9 @@ unique_ptr<LogicalOperator> LogicalOperator::Deserialize(Deserializer &deseriali
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
 		result = LogicalComparisonJoin::Deserialize(deserializer);
 		break;
+	case LogicalOperatorType::LOGICAL_COPY_DATABASE:
+		result = LogicalCopyDatabase::Deserialize(deserializer);
+		break;
 	case LogicalOperatorType::LOGICAL_COPY_TO_FILE:
 		result = LogicalCopyToFile::Deserialize(deserializer);
 		break;
@@ -169,9 +172,6 @@ unique_ptr<LogicalOperator> LogicalOperator::Deserialize(Deserializer &deseriali
 	case LogicalOperatorType::LOGICAL_UPDATE:
 		result = LogicalUpdate::Deserialize(deserializer);
 		break;
-	case LogicalOperatorType::LOGICAL_VACUUM:
-		result = LogicalSimple::Deserialize(deserializer);
-		break;
 	case LogicalOperatorType::LOGICAL_WINDOW:
 		result = LogicalWindow::Deserialize(deserializer);
 		break;
@@ -184,16 +184,16 @@ unique_ptr<LogicalOperator> LogicalOperator::Deserialize(Deserializer &deseriali
 }
 
 void FilenamePattern::Serialize(Serializer &serializer) const {
-	serializer.WritePropertyWithDefault<string>(200, "base", _base);
-	serializer.WritePropertyWithDefault<idx_t>(201, "pos", _pos);
-	serializer.WritePropertyWithDefault<bool>(202, "uuid", _uuid);
+	serializer.WritePropertyWithDefault<string>(200, "base", base);
+	serializer.WritePropertyWithDefault<idx_t>(201, "pos", pos);
+	serializer.WritePropertyWithDefault<bool>(202, "uuid", uuid);
 }
 
 FilenamePattern FilenamePattern::Deserialize(Deserializer &deserializer) {
 	FilenamePattern result;
-	deserializer.ReadPropertyWithDefault<string>(200, "base", result._base);
-	deserializer.ReadPropertyWithDefault<idx_t>(201, "pos", result._pos);
-	deserializer.ReadPropertyWithDefault<bool>(202, "uuid", result._uuid);
+	deserializer.ReadPropertyWithDefault<string>(200, "base", result.base);
+	deserializer.ReadPropertyWithDefault<idx_t>(201, "pos", result.pos);
+	deserializer.ReadPropertyWithDefault<bool>(202, "uuid", result.uuid);
 	return result;
 }
 
@@ -298,6 +298,17 @@ unique_ptr<LogicalOperator> LogicalComparisonJoin::Deserialize(Deserializer &des
 	return std::move(result);
 }
 
+void LogicalCopyDatabase::Serialize(Serializer &serializer) const {
+	LogicalOperator::Serialize(serializer);
+	serializer.WritePropertyWithDefault<unique_ptr<CopyDatabaseInfo>>(200, "info", info);
+}
+
+unique_ptr<LogicalOperator> LogicalCopyDatabase::Deserialize(Deserializer &deserializer) {
+	auto info = deserializer.ReadPropertyWithDefault<unique_ptr<ParseInfo>>(200, "info");
+	auto result = duckdb::unique_ptr<LogicalCopyDatabase>(new LogicalCopyDatabase(std::move(info)));
+	return std::move(result);
+}
+
 void LogicalCreate::Serialize(Serializer &serializer) const {
 	LogicalOperator::Serialize(serializer);
 	serializer.WritePropertyWithDefault<unique_ptr<CreateInfo>>(200, "info", info);

package/src/duckdb/src/storage/serialization/serialize_nodes.cpp

@@ -118,9 +118,9 @@ CSVOption<T> CSVOption<T>::Deserialize(Deserializer &deserializer) {
 }
 
 void CSVReaderOptions::Serialize(Serializer &serializer) const {
-	serializer.WritePropertyWithDefault<bool>(100, "ignore_errors", ignore_errors);
+	serializer.WritePropertyWithDefault<bool>(100, "ignore_errors", ignore_errors, false);
 	serializer.WritePropertyWithDefault<idx_t>(101, "buffer_sample_size", buffer_sample_size);
-	serializer.WritePropertyWithDefault<string>(102, "null_str", null_str);
+	serializer.WritePropertyWithDefault<vector<string>>(102, "null_str", null_str);
 	serializer.WriteProperty<FileCompressionType>(103, "compression", compression);
 	serializer.WritePropertyWithDefault<bool>(104, "allow_quoted_nulls", allow_quoted_nulls);
 	serializer.WritePropertyWithDefault<idx_t>(105, "maximum_line_size", maximum_line_size);
@@ -135,10 +135,10 @@ void CSVReaderOptions::Serialize(Serializer &serializer) const {
 	serializer.WritePropertyWithDefault<idx_t>(114, "buffer_size", buffer_size);
 	serializer.WriteProperty<MultiFileReaderOptions>(115, "file_options", file_options);
 	serializer.WritePropertyWithDefault<vector<bool>>(116, "force_quote", force_quote);
-	serializer.WritePropertyWithDefault<string>(117, "rejects_table_name", rejects_table_name);
+	serializer.WritePropertyWithDefault<string>(117, "rejects_table_name", rejects_table_name, "reject_errors");
 	serializer.WritePropertyWithDefault<idx_t>(118, "rejects_limit", rejects_limit);
-	serializer.WritePropertyWithDefault<vector<string>>(119, "rejects_recovery_columns", rejects_recovery_columns);
-	serializer.WritePropertyWithDefault<vector<idx_t>>(120, "rejects_recovery_column_ids", rejects_recovery_column_ids);
+	/* [Deleted] (vector<string>) "rejects_recovery_columns" */
+	/* [Deleted] (vector<idx_t>) "rejects_recovery_column_ids" */
 	serializer.WriteProperty<CSVOption<char>>(121, "dialect_options.state_machine_options.delimiter", dialect_options.state_machine_options.delimiter);
 	serializer.WriteProperty<CSVOption<char>>(122, "dialect_options.state_machine_options.quote", dialect_options.state_machine_options.quote);
 	serializer.WriteProperty<CSVOption<char>>(123, "dialect_options.state_machine_options.escape", dialect_options.state_machine_options.escape);
@@ -150,13 +150,14 @@ void CSVReaderOptions::Serialize(Serializer &serializer) const {
 	serializer.WritePropertyWithDefault<string>(129, "sniffer_user_mismatch_error", sniffer_user_mismatch_error);
 	serializer.WritePropertyWithDefault<bool>(130, "parallel", parallel);
 	serializer.WritePropertyWithDefault<vector<bool>>(131, "was_type_manually_set", was_type_manually_set);
+	serializer.WritePropertyWithDefault<CSVOption<string>>(132, "rejects_scan_name", rejects_scan_name, {"reject_scans"});
 }
 
 CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) {
 	CSVReaderOptions result;
-	deserializer.ReadPropertyWithDefault<bool>(100, "ignore_errors", result.ignore_errors);
+	deserializer.ReadPropertyWithDefault<bool>(100, "ignore_errors", result.ignore_errors, false);
 	deserializer.ReadPropertyWithDefault<idx_t>(101, "buffer_sample_size", result.buffer_sample_size);
-	deserializer.ReadPropertyWithDefault<string>(102, "null_str", result.null_str);
+	deserializer.ReadPropertyWithDefault<vector<string>>(102, "null_str", result.null_str);
 	deserializer.ReadProperty<FileCompressionType>(103, "compression", result.compression);
 	deserializer.ReadPropertyWithDefault<bool>(104, "allow_quoted_nulls", result.allow_quoted_nulls);
 	deserializer.ReadPropertyWithDefault<idx_t>(105, "maximum_line_size", result.maximum_line_size);
@@ -171,10 +172,10 @@ CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) {
 	deserializer.ReadPropertyWithDefault<idx_t>(114, "buffer_size", result.buffer_size);
 	deserializer.ReadProperty<MultiFileReaderOptions>(115, "file_options", result.file_options);
 	deserializer.ReadPropertyWithDefault<vector<bool>>(116, "force_quote", result.force_quote);
-	deserializer.ReadPropertyWithDefault<string>(117, "rejects_table_name", result.rejects_table_name);
+	deserializer.ReadPropertyWithDefault<string>(117, "rejects_table_name", result.rejects_table_name, "reject_errors");
 	deserializer.ReadPropertyWithDefault<idx_t>(118, "rejects_limit", result.rejects_limit);
-	deserializer.ReadPropertyWithDefault<vector<string>>(119, "rejects_recovery_columns", result.rejects_recovery_columns);
-	deserializer.ReadPropertyWithDefault<vector<idx_t>>(120, "rejects_recovery_column_ids", result.rejects_recovery_column_ids);
+	deserializer.ReadDeletedProperty<vector<string>>(119, "rejects_recovery_columns");
+	deserializer.ReadDeletedProperty<vector<idx_t>>(120, "rejects_recovery_column_ids");
 	deserializer.ReadProperty<CSVOption<char>>(121, "dialect_options.state_machine_options.delimiter", result.dialect_options.state_machine_options.delimiter);
 	deserializer.ReadProperty<CSVOption<char>>(122, "dialect_options.state_machine_options.quote", result.dialect_options.state_machine_options.quote);
 	deserializer.ReadProperty<CSVOption<char>>(123, "dialect_options.state_machine_options.escape", result.dialect_options.state_machine_options.escape);
@@ -186,6 +187,7 @@ CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) {
 	deserializer.ReadPropertyWithDefault<string>(129, "sniffer_user_mismatch_error", result.sniffer_user_mismatch_error);
 	deserializer.ReadPropertyWithDefault<bool>(130, "parallel", result.parallel);
 	deserializer.ReadPropertyWithDefault<vector<bool>>(131, "was_type_manually_set", result.was_type_manually_set);
+	deserializer.ReadPropertyWithDefault<CSVOption<string>>(132, "rejects_scan_name", result.rejects_scan_name, {"reject_scans"});
 	return result;
 }
 

package/src/duckdb/src/storage/serialization/serialize_parse_info.cpp

@@ -10,6 +10,7 @@
 #include "duckdb/parser/parsed_data/alter_table_info.hpp"
 #include "duckdb/parser/parsed_data/comment_on_column_info.hpp"
 #include "duckdb/parser/parsed_data/attach_info.hpp"
+#include "duckdb/parser/parsed_data/copy_database_info.hpp"
 #include "duckdb/parser/parsed_data/copy_info.hpp"
 #include "duckdb/parser/parsed_data/detach_info.hpp"
 #include "duckdb/parser/parsed_data/drop_info.hpp"
@@ -34,6 +35,9 @@ unique_ptr<ParseInfo> ParseInfo::Deserialize(Deserializer &deserializer) {
 	case ParseInfoType::ATTACH_INFO:
 		result = AttachInfo::Deserialize(deserializer);
 		break;
+	case ParseInfoType::COPY_DATABASE_INFO:
+		result = CopyDatabaseInfo::Deserialize(deserializer);
+		break;
 	case ParseInfoType::COPY_INFO:
 		result = CopyInfo::Deserialize(deserializer);
 		break;
@@ -229,6 +233,19 @@ unique_ptr<AlterTableInfo> ChangeColumnTypeInfo::Deserialize(Deserializer &deser
 	return std::move(result);
 }
 
+void CopyDatabaseInfo::Serialize(Serializer &serializer) const {
+	ParseInfo::Serialize(serializer);
+	serializer.WritePropertyWithDefault<string>(200, "target_database", target_database);
+	serializer.WritePropertyWithDefault<vector<unique_ptr<CreateInfo>>>(201, "entries", entries);
+}
+
+unique_ptr<ParseInfo> CopyDatabaseInfo::Deserialize(Deserializer &deserializer) {
+	auto result = duckdb::unique_ptr<CopyDatabaseInfo>(new CopyDatabaseInfo());
+	deserializer.ReadPropertyWithDefault<string>(200, "target_database", result->target_database);
+	deserializer.ReadPropertyWithDefault<vector<unique_ptr<CreateInfo>>>(201, "entries", result->entries);
+	return std::move(result);
+}
+
 void CopyInfo::Serialize(Serializer &serializer) const {
 	ParseInfo::Serialize(serializer);
 	serializer.WritePropertyWithDefault<string>(200, "catalog", catalog);
@@ -449,11 +466,17 @@ unique_ptr<ParseInfo> TransactionInfo::Deserialize(Deserializer &deserializer) {
 void VacuumInfo::Serialize(Serializer &serializer) const {
 	ParseInfo::Serialize(serializer);
 	serializer.WriteProperty<VacuumOptions>(200, "options", options);
+	serializer.WritePropertyWithDefault<bool>(201, "has_table", has_table);
+	serializer.WritePropertyWithDefault<unique_ptr<TableRef>>(202, "ref", ref);
+	serializer.WritePropertyWithDefault<vector<string>>(203, "columns", columns);
 }
 
 unique_ptr<ParseInfo> VacuumInfo::Deserialize(Deserializer &deserializer) {
 	auto options = deserializer.ReadProperty<VacuumOptions>(200, "options");
 	auto result = duckdb::unique_ptr<VacuumInfo>(new VacuumInfo(options));
+	deserializer.ReadPropertyWithDefault<bool>(201, "has_table", result->has_table);
+	deserializer.ReadPropertyWithDefault<unique_ptr<TableRef>>(202, "ref", result->ref);
+	deserializer.ReadPropertyWithDefault<vector<string>>(203, "columns", result->columns);
 	return std::move(result);
 }
 

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -150,31 +150,31 @@ SingleFileBlockManager::SingleFileBlockManager(AttachedDatabase &db, string path
       iteration_count(0), options(options) {
 }
 
-void SingleFileBlockManager::GetFileFlags(uint8_t &flags, FileLockType &lock, bool create_new) {
+FileOpenFlags SingleFileBlockManager::GetFileFlags(bool create_new) const {
+	FileOpenFlags result;
 	if (options.read_only) {
 		D_ASSERT(!create_new);
-		flags = FileFlags::FILE_FLAGS_READ;
-		lock = FileLockType::READ_LOCK;
+		result = FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_NULL_IF_NOT_EXISTS | FileLockType::READ_LOCK;
 	} else {
-		flags = FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_READ;
-		lock = FileLockType::WRITE_LOCK;
+		result = FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_READ | FileLockType::WRITE_LOCK;
 		if (create_new) {
-			flags |= FileFlags::FILE_FLAGS_FILE_CREATE;
+			result |= FileFlags::FILE_FLAGS_FILE_CREATE;
 		}
 	}
 	if (options.use_direct_io) {
-		flags |= FileFlags::FILE_FLAGS_DIRECT_IO;
+		result |= FileFlags::FILE_FLAGS_DIRECT_IO;
 	}
+	// database files can be read from in parallel
+	result |= FileFlags::FILE_FLAGS_PARALLEL_ACCESS;
+	return result;
 }
 
 void SingleFileBlockManager::CreateNewDatabase() {
-	uint8_t flags;
-	FileLockType lock;
-	GetFileFlags(flags, lock, true);
+	auto flags = GetFileFlags(true);
 
 	// open the RDBMS handle
 	auto &fs = FileSystem::Get(db);
-	handle = fs.OpenFile(path, flags, lock);
+	handle = fs.OpenFile(path, flags);
 
 	// if we create a new file, we fill the metadata of the file
 	// first fill in the new header
@@ -221,13 +221,15 @@ void SingleFileBlockManager::CreateNewDatabase() {
 }
 
 void SingleFileBlockManager::LoadExistingDatabase() {
-	uint8_t flags;
-	FileLockType lock;
-	GetFileFlags(flags, lock, false);
+	auto flags = GetFileFlags(false);
 
 	// open the RDBMS handle
 	auto &fs = FileSystem::Get(db);
-	handle = fs.OpenFile(path, flags, lock);
+	handle = fs.OpenFile(path, flags);
+	if (!handle) {
+		// this can only happen in read-only mode - as that is when we set FILE_FLAGS_NULL_IF_NOT_EXISTS
+		throw CatalogException("Cannot open database \"%s\" in read-only mode: database does not exist", path);
+	}
 
 	MainHeader::CheckMagicBytes(*handle);
 	// otherwise, we check the metadata of the file
@@ -294,7 +296,9 @@ void SingleFileBlockManager::LoadFreeList() {
 	auto free_list_count = reader.Read<uint64_t>();
 	free_list.clear();
 	for (idx_t i = 0; i < free_list_count; i++) {
-		free_list.insert(reader.Read<block_id_t>());
+		auto block = reader.Read<block_id_t>();
+		free_list.insert(block);
+		newly_freed_list.insert(block);
 	}
 	auto multi_use_blocks_count = reader.Read<uint64_t>();
 	multi_use_blocks.clear();
@@ -320,6 +324,7 @@ block_id_t SingleFileBlockManager::GetFreeBlockId() {
 		block = *free_list.begin();
 		// erase the entry from the free list again
 		free_list.erase(free_list.begin());
+		newly_freed_list.erase(block);
 	} else {
 		block = max_block++;
 	}
@@ -335,6 +340,7 @@ void SingleFileBlockManager::MarkBlockAsFree(block_id_t block_id) {
 	}
 	multi_use_blocks.erase(block_id);
 	free_list.insert(block_id);
+	newly_freed_list.insert(block_id);
 }
 
 void SingleFileBlockManager::MarkBlockAsModified(block_id_t block_id) {
@@ -432,9 +438,8 @@ void SingleFileBlockManager::Truncate() {
 		return;
 	}
 	// truncate the file
-	for (idx_t i = 0; i < blocks_to_truncate; i++) {
-		free_list.erase(max_block + i);
-	}
+	free_list.erase(free_list.lower_bound(max_block), free_list.end());
+	newly_freed_list.erase(newly_freed_list.lower_bound(max_block), newly_freed_list.end());
 	handle->Truncate(BLOCK_START + max_block * Storage::BLOCK_ALLOC_SIZE);
 }
 
@@ -494,6 +499,7 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 	metadata_manager.MarkBlocksAsModified();
 	for (auto &block : modified_blocks) {
 		free_list.insert(block);
+		newly_freed_list.insert(block);
 	}
 	modified_blocks.clear();
 
@@ -547,6 +553,27 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 	active_header = 1 - active_header;
 	//! Ensure the header write ends up on disk
 	handle->Sync();
+	// Release the free blocks to the filesystem.
+	TrimFreeBlocks();
+}
+
+void SingleFileBlockManager::TrimFreeBlocks() {
+	if (DBConfig::Get(db).options.trim_free_blocks) {
+		for (auto itr = newly_freed_list.begin(); itr != newly_freed_list.end(); ++itr) {
+			block_id_t first = *itr;
+			block_id_t last = first;
+			// Find end of contiguous range.
+			for (++itr; itr != newly_freed_list.end() && (*itr == last + 1); ++itr) {
+				last = *itr;
+			}
+			// We are now one too far.
+			--itr;
+			// Trim the range.
+			handle->Trim(BLOCK_START + (first * Storage::BLOCK_ALLOC_SIZE),
+			             (last + 1 - first) * Storage::BLOCK_ALLOC_SIZE);
+		}
+	}
+	newly_freed_list.clear();
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/standard_buffer_manager.cpp

@@ -13,6 +13,12 @@
 
 namespace duckdb {
 
+#ifdef DUCKDB_DEBUG_DESTROY_BLOCKS
+static void WriteGarbageIntoBuffer(FileBuffer &buffer) {
+	memset(buffer.buffer, 0xa5, buffer.size); // 0xa5 is default memory in debug mode
+}
+#endif
+
 struct BufferAllocatorData : PrivateAllocatorData {
 	explicit BufferAllocatorData(StandardBufferManager &manager) : manager(manager) {
 	}
@@ -85,14 +91,20 @@ TempBufferPoolReservation StandardBufferManager::EvictBlocksOrThrow(MemoryTag ta
 
 shared_ptr<BlockHandle> StandardBufferManager::RegisterSmallMemory(idx_t block_size) {
 	D_ASSERT(block_size < Storage::BLOCK_SIZE);
-	auto res = EvictBlocksOrThrow(MemoryTag::BASE_TABLE, block_size, nullptr, "could not allocate block of size %s%s",
-	                              StringUtil::BytesToHumanReadableString(block_size));
+	auto reservation =
+	    EvictBlocksOrThrow(MemoryTag::BASE_TABLE, block_size, nullptr, "could not allocate block of size %s%s",
+	                       StringUtil::BytesToHumanReadableString(block_size));
 
 	auto buffer = ConstructManagedBuffer(block_size, nullptr, FileBufferType::TINY_BUFFER);
 
 	// create a new block pointer for this block
-	return make_shared<BlockHandle>(*temp_block_manager, ++temporary_id, MemoryTag::BASE_TABLE, std::move(buffer),
-	                                false, block_size, std::move(res));
+	auto result = make_shared<BlockHandle>(*temp_block_manager, ++temporary_id, MemoryTag::BASE_TABLE,
+	                                       std::move(buffer), false, block_size, std::move(reservation));
+#ifdef DUCKDB_DEBUG_DESTROY_BLOCKS
+	// Initialize the memory with garbage data
+	WriteGarbageIntoBuffer(*result->buffer);
+#endif
+	return result;
 }
 
 shared_ptr<BlockHandle> StandardBufferManager::RegisterMemory(MemoryTag tag, idx_t block_size, bool can_destroy) {
@@ -115,6 +127,10 @@ BufferHandle StandardBufferManager::Allocate(MemoryTag tag, idx_t block_size, bo
 	shared_ptr<BlockHandle> local_block;
 	auto block_ptr = block ? block : &local_block;
 	*block_ptr = RegisterMemory(tag, block_size, can_destroy);
+#ifdef DUCKDB_DEBUG_DESTROY_BLOCKS
+	// Initialize the memory with garbage data
+	WriteGarbageIntoBuffer(*(*block_ptr)->buffer);
+#endif
 	return Pin(*block_ptr);
 }
 
@@ -206,7 +222,7 @@ void StandardBufferManager::VerifyZeroReaders(shared_ptr<BlockHandle> &handle) {
 	auto replacement_buffer = make_uniq<FileBuffer>(Allocator::Get(db), handle->buffer->type,
 	                                                handle->memory_usage - Storage::BLOCK_HEADER_SIZE);
 	memcpy(replacement_buffer->buffer, handle->buffer->buffer, handle->buffer->size);
-	memset(handle->buffer->buffer, 0xa5, handle->buffer->size); // 0xa5 is default memory in debug mode
+	WriteGarbageIntoBuffer(*handle->buffer);
 	handle->buffer = std::move(replacement_buffer);
 #endif
 }

package/src/duckdb/src/storage/statistics/array_stats.cpp

@@ -78,7 +78,7 @@ void ArrayStats::Deserialize(Deserializer &deserializer, BaseStatistics &base) {
 	auto &child_type = ArrayType::GetChildType(type);
 
 	// Push the logical type of the child type to the deserialization context
-	deserializer.Set<LogicalType &>(const_cast<LogicalType &>(child_type));
+	deserializer.Set<const LogicalType &>(child_type);
 	base.child_stats[0].Copy(deserializer.ReadProperty<BaseStatistics>(200, "child_stats"));
 	deserializer.Unset<LogicalType>();
 }

package/src/duckdb/src/storage/statistics/base_statistics.cpp

@@ -341,11 +341,10 @@ BaseStatistics BaseStatistics::Deserialize(Deserializer &deserializer) {
 	auto distinct_count = deserializer.ReadProperty<idx_t>(102, "distinct_count");
 
 	// Get the logical type from the deserializer context.
-	auto type = deserializer.Get<LogicalType &>();
-
+	auto &type = deserializer.Get<const LogicalType &>();
 	auto stats_type = GetStatsType(type);
 
-	BaseStatistics stats(std::move(type));
+	BaseStatistics stats(type);
 
 	stats.has_null = has_null;
 	stats.has_no_null = has_no_null;

package/src/duckdb/src/storage/statistics/list_stats.cpp

@@ -79,7 +79,7 @@ void ListStats::Deserialize(Deserializer &deserializer, BaseStatistics &base) {
 	auto &child_type = ListType::GetChildType(type);
 
 	// Push the logical type of the child type to the deserialization context
-	deserializer.Set<LogicalType &>(const_cast<LogicalType &>(child_type));
+	deserializer.Set<const LogicalType &>(child_type);
 	base.child_stats[0].Copy(deserializer.ReadProperty<BaseStatistics>(200, "child_stats"));
 	deserializer.Unset<LogicalType>();
 }