duckdb 0.6.2-dev971.0 → 0.7.1-dev2.0

package/src/duckdb/src/storage/index.cpp

@@ -8,19 +8,25 @@
 
 namespace duckdb {
 
-Index::Index(IndexType type, TableIOManager &table_io_manager, const vector<column_t> &column_ids_p,
-             const vector<unique_ptr<Expression>> &unbound_expressions, IndexConstraintType constraint_type_p)
-    : type(type), table_io_manager(table_io_manager), column_ids(column_ids_p), constraint_type(constraint_type_p) {
+Index::Index(AttachedDatabase &db, IndexType type, TableIOManager &table_io_manager,
+             const vector<column_t> &column_ids_p, const vector<unique_ptr<Expression>> &unbound_expressions,
+             IndexConstraintType constraint_type_p, bool track_memory)
+
+    : type(type), table_io_manager(table_io_manager), column_ids(column_ids_p), constraint_type(constraint_type_p),
+      db(db), buffer_manager(BufferManager::GetBufferManager(db)), memory_size(0), track_memory(track_memory) {
+
 	for (auto &expr : unbound_expressions) {
 		types.push_back(expr->return_type.InternalType());
 		logical_types.push_back(expr->return_type);
 		auto unbound_expression = expr->Copy();
 		bound_expressions.push_back(BindExpression(unbound_expression->Copy()));
-		this->unbound_expressions.emplace_back(move(unbound_expression));
+		this->unbound_expressions.emplace_back(std::move(unbound_expression));
 	}
 	for (auto &bound_expr : bound_expressions) {
 		executor.AddExpression(*bound_expr);
 	}
+
+	// create the column id set
 	for (auto column_id : column_ids) {
 		column_id_set.insert(column_id);
 	}
@@ -66,8 +72,8 @@ unique_ptr<Expression> Index::BindExpression(unique_ptr<Expression> expr) {
 		auto &bound_colref = (BoundColumnRefExpression &)*expr;
 		return make_unique<BoundReferenceExpression>(expr->return_type, column_ids[bound_colref.binding.column_index]);
 	}
-	ExpressionIterator::EnumerateChildren(*expr,
-	                                      [&](unique_ptr<Expression> &expr) { expr = BindExpression(move(expr)); });
+	ExpressionIterator::EnumerateChildren(
+	    *expr, [this](unique_ptr<Expression> &expr) { expr = BindExpression(std::move(expr)); });
 	return expr;
 }
 

package/src/duckdb/src/storage/local_storage.cpp

@@ -4,7 +4,7 @@
 #include "duckdb/storage/write_ahead_log.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/storage/table/row_group.hpp"
-#include "duckdb/transaction/transaction.hpp"
+#include "duckdb/transaction/duck_transaction.hpp"
 #include "duckdb/planner/table_filter.hpp"
 #include "duckdb/storage/partial_block_manager.hpp"
 
@@ -21,7 +21,8 @@ OptimisticDataWriter::OptimisticDataWriter(DataTable *table) : table(table) {
 }
 
 OptimisticDataWriter::OptimisticDataWriter(DataTable *table, OptimisticDataWriter &parent)
-    : table(table), partial_manager(move(parent.partial_manager)), written_blocks(move(parent.written_blocks)) {
+    : table(table), partial_manager(std::move(parent.partial_manager)),
+      written_blocks(std::move(parent.written_blocks)) {
 	if (partial_manager) {
 		partial_manager->FlushPartialBlocks();
 	}
@@ -125,8 +126,8 @@ LocalTableStorage::LocalTableStorage(DataTable &table)
 			for (auto &expr : art.unbound_expressions) {
 				unbound_expressions.push_back(expr->Copy());
 			}
-			indexes.AddIndex(make_unique<ART>(art.column_ids, art.table_io_manager, move(unbound_expressions),
-			                                  art.constraint_type, art.db));
+			indexes.AddIndex(make_unique<ART>(art.column_ids, art.table_io_manager, std::move(unbound_expressions),
+			                                  art.constraint_type, art.db, false));
 		}
 		return false;
 	});
@@ -136,7 +137,7 @@ LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &new_dt,
                                      idx_t changed_idx, const LogicalType &target_type,
                                      const vector<column_t> &bound_columns, Expression &cast_expr)
     : table(&new_dt), allocator(Allocator::Get(table->db)), deleted_rows(parent.deleted_rows),
-      optimistic_writer(table, parent.optimistic_writer), optimistic_writers(move(parent.optimistic_writers)) {
+      optimistic_writer(table, parent.optimistic_writer), optimistic_writers(std::move(parent.optimistic_writers)) {
 	row_groups = parent.row_groups->AlterType(context, changed_idx, target_type, bound_columns, cast_expr);
 	parent.row_groups.reset();
 	indexes.Move(parent.indexes);
@@ -144,7 +145,7 @@ LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &new_dt,
 
 LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &parent, idx_t drop_idx)
     : table(&new_dt), allocator(Allocator::Get(table->db)), deleted_rows(parent.deleted_rows),
-      optimistic_writer(table, parent.optimistic_writer), optimistic_writers(move(parent.optimistic_writers)) {
+      optimistic_writer(table, parent.optimistic_writer), optimistic_writers(std::move(parent.optimistic_writers)) {
 	row_groups = parent.row_groups->RemoveColumn(drop_idx);
 	parent.row_groups.reset();
 	indexes.Move(parent.indexes);
@@ -153,7 +154,7 @@ LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &paren
 LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &new_dt, LocalTableStorage &parent,
                                      ColumnDefinition &new_column, Expression *default_value)
     : table(&new_dt), allocator(Allocator::Get(table->db)), deleted_rows(parent.deleted_rows),
-      optimistic_writer(table, parent.optimistic_writer), optimistic_writers(move(parent.optimistic_writers)) {
+      optimistic_writer(table, parent.optimistic_writer), optimistic_writers(std::move(parent.optimistic_writers)) {
 	row_groups = parent.row_groups->AddColumn(context, new_column, default_value);
 	parent.row_groups.reset();
 	indexes.Move(parent.indexes);
@@ -196,7 +197,7 @@ void LocalTableStorage::FlushToDisk() {
 	optimistic_writer.FinalFlush();
 }
 
-bool LocalTableStorage::AppendToIndexes(Transaction &transaction, RowGroupCollection &source,
+bool LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, RowGroupCollection &source,
                                         TableIndexList &index_list, const vector<LogicalType> &table_types,
                                         row_t &start_row) {
 	// only need to scan for index append
@@ -223,8 +224,8 @@ bool LocalTableStorage::AppendToIndexes(Transaction &transaction, RowGroupCollec
 	return success;
 }
 
-void LocalTableStorage::AppendToIndexes(Transaction &transaction, TableAppendState &append_state, idx_t append_count,
-                                        bool append_to_table) {
+void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppendState &append_state,
+                                        idx_t append_count, bool append_to_table) {
 	bool constraint_violated = false;
 	if (append_to_table) {
 		table->InitializeAppend(transaction, append_state, append_count);
@@ -281,7 +282,7 @@ void LocalTableStorage::AppendToIndexes(Transaction &transaction, TableAppendSta
 
 OptimisticDataWriter *LocalTableStorage::CreateOptimisticWriter() {
 	auto writer = make_unique<OptimisticDataWriter>(table);
-	optimistic_writers.push_back(move(writer));
+	optimistic_writers.push_back(std::move(writer));
 	return optimistic_writers.back().get();
 }
 
@@ -308,7 +309,7 @@ LocalTableStorage *LocalTableManager::GetOrCreateStorage(DataTable *table) {
 	if (entry == table_storage.end()) {
 		auto new_storage = make_shared<LocalTableStorage>(*table);
 		auto storage = new_storage.get();
-		table_storage.insert(make_pair(table, move(new_storage)));
+		table_storage.insert(make_pair(table, std::move(new_storage)));
 		return storage;
 	} else {
 		return entry->second.get();
@@ -326,14 +327,14 @@ shared_ptr<LocalTableStorage> LocalTableManager::MoveEntry(DataTable *table) {
 	if (entry == table_storage.end()) {
 		return nullptr;
 	}
-	auto storage_entry = move(entry->second);
+	auto storage_entry = std::move(entry->second);
 	table_storage.erase(table);
 	return storage_entry;
 }
 
 unordered_map<DataTable *, shared_ptr<LocalTableStorage>> LocalTableManager::MoveEntries() {
 	lock_guard<mutex> l(table_storage_lock);
-	return move(table_storage);
+	return std::move(table_storage);
 }
 
 idx_t LocalTableManager::EstimatedSize() {
@@ -348,22 +349,22 @@ idx_t LocalTableManager::EstimatedSize() {
 void LocalTableManager::InsertEntry(DataTable *table, shared_ptr<LocalTableStorage> entry) {
 	lock_guard<mutex> l(table_storage_lock);
 	D_ASSERT(table_storage.find(table) == table_storage.end());
-	table_storage[table] = move(entry);
+	table_storage[table] = std::move(entry);
 }
 
 //===--------------------------------------------------------------------===//
 // LocalStorage
 //===--------------------------------------------------------------------===//
-LocalStorage::LocalStorage(ClientContext &context, Transaction &transaction)
+LocalStorage::LocalStorage(ClientContext &context, DuckTransaction &transaction)
     : context(context), transaction(transaction) {
 }
 
-LocalStorage &LocalStorage::Get(Transaction &transaction) {
+LocalStorage &LocalStorage::Get(DuckTransaction &transaction) {
 	return transaction.GetLocalStorage();
 }
 
 LocalStorage &LocalStorage::Get(ClientContext &context, AttachedDatabase &db) {
-	return Transaction::Get(context, db).GetLocalStorage();
+	return DuckTransaction::Get(context, db).GetLocalStorage();
 }
 
 LocalStorage &LocalStorage::Get(ClientContext &context, Catalog &catalog) {
@@ -514,7 +515,7 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
 	transaction.PushAppend(&table, append_state.row_start, append_count);
 }
 
-void LocalStorage::Commit(LocalStorage::CommitState &commit_state, Transaction &transaction) {
+void LocalStorage::Commit(LocalStorage::CommitState &commit_state, DuckTransaction &transaction) {
 	// commit local storage
 	// iterate over all entries in the table storage map and commit them
 	// after this, the local storage is no longer required and can be cleared
@@ -559,7 +560,7 @@ void LocalStorage::MoveStorage(DataTable *old_dt, DataTable *new_dt) {
 	}
 	// take over the storage from the old entry
 	new_storage->table = new_dt;
-	table_manager.InsertEntry(new_dt, move(new_storage));
+	table_manager.InsertEntry(new_dt, std::move(new_storage));
 }
 
 void LocalStorage::AddColumn(DataTable *old_dt, DataTable *new_dt, ColumnDefinition &new_column,
@@ -570,7 +571,7 @@ void LocalStorage::AddColumn(DataTable *old_dt, DataTable *new_dt, ColumnDefinit
 		return;
 	}
 	auto new_storage = make_unique<LocalTableStorage>(context, *new_dt, *storage, new_column, default_value);
-	table_manager.InsertEntry(new_dt, move(new_storage));
+	table_manager.InsertEntry(new_dt, std::move(new_storage));
 }
 
 void LocalStorage::DropColumn(DataTable *old_dt, DataTable *new_dt, idx_t removed_column) {
@@ -580,7 +581,7 @@ void LocalStorage::DropColumn(DataTable *old_dt, DataTable *new_dt, idx_t remove
 		return;
 	}
 	auto new_storage = make_unique<LocalTableStorage>(*new_dt, *storage, removed_column);
-	table_manager.InsertEntry(new_dt, move(new_storage));
+	table_manager.InsertEntry(new_dt, std::move(new_storage));
 }
 
 void LocalStorage::ChangeType(DataTable *old_dt, DataTable *new_dt, idx_t changed_idx, const LogicalType &target_type,
@@ -592,23 +593,17 @@ void LocalStorage::ChangeType(DataTable *old_dt, DataTable *new_dt, idx_t change
 	}
 	auto new_storage =
 	    make_unique<LocalTableStorage>(context, *new_dt, *storage, changed_idx, target_type, bound_columns, cast_expr);
-	table_manager.InsertEntry(new_dt, move(new_storage));
+	table_manager.InsertEntry(new_dt, std::move(new_storage));
 }
 
-void LocalStorage::FetchChunk(DataTable *table, Vector &row_ids, idx_t count, DataChunk &verify_chunk) {
+void LocalStorage::FetchChunk(DataTable *table, Vector &row_ids, idx_t count, const vector<column_t> &col_ids,
+                              DataChunk &chunk, ColumnFetchState &fetch_state) {
 	auto storage = table_manager.GetStorage(table);
 	if (!storage) {
 		throw InternalException("LocalStorage::FetchChunk - local storage not found");
 	}
 
-	ColumnFetchState fetch_state;
-	vector<column_t> col_ids;
-	vector<LogicalType> types = storage->table->GetTypes();
-	for (idx_t i = 0; i < types.size(); i++) {
-		col_ids.push_back(i);
-	}
-	verify_chunk.Initialize(storage->allocator, types);
-	storage->row_groups->Fetch(transaction, verify_chunk, col_ids, row_ids, count, fetch_state);
+	storage->row_groups->Fetch(transaction, chunk, col_ids, row_ids, count, fetch_state);
 }
 
 TableIndexList &LocalStorage::GetIndexes(DataTable *table) {

package/src/duckdb/src/storage/magic_bytes.cpp

@@ -0,0 +1,31 @@
+#include "duckdb/storage/magic_bytes.hpp"
+#include "duckdb/common/local_file_system.hpp"
+#include "duckdb/storage/storage_info.hpp"
+
+namespace duckdb {
+
+DataFileType MagicBytes::CheckMagicBytes(FileSystem *fs_p, const string &path) {
+	LocalFileSystem lfs;
+	FileSystem &fs = fs_p ? *fs_p : lfs;
+	if (!fs.FileExists(path)) {
+		return DataFileType::FILE_DOES_NOT_EXIST;
+	}
+	auto handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ);
+
+	constexpr const idx_t MAGIC_BYTES_READ_SIZE = 16;
+	char buffer[MAGIC_BYTES_READ_SIZE];
+
+	handle->Read(buffer, MAGIC_BYTES_READ_SIZE);
+	if (memcmp(buffer, "SQLite format 3\0\0\0", 16) == 0) {
+		return DataFileType::SQLITE_FILE;
+	}
+	if (memcmp(buffer, "PAR1", 4) == 0) {
+		return DataFileType::PARQUET_FILE;
+	}
+	if (memcmp(buffer + MainHeader::MAGIC_BYTE_OFFSET, MainHeader::MAGIC_BYTES, MainHeader::MAGIC_BYTE_SIZE) == 0) {
+		return DataFileType::DUCKDB_FILE;
+	}
+	return DataFileType::FILE_DOES_NOT_EXIST;
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/meta_block_writer.cpp

@@ -7,7 +7,7 @@ namespace duckdb {
 MetaBlockWriter::MetaBlockWriter(BlockManager &block_manager, block_id_t initial_block_id)
     : block_manager(block_manager) {
 	if (initial_block_id == INVALID_BLOCK) {
-		initial_block_id = GetNextBlockId();
+		initial_block_id = MetaBlockWriter::GetNextBlockId();
 	}
 	block = block_manager.CreateBlock(initial_block_id, nullptr);
 	Store<block_id_t>(-1, block->buffer);

package/src/duckdb/src/storage/partial_block_manager.cpp

@@ -45,7 +45,7 @@ bool PartialBlockManager::GetPartialBlock(idx_t segment_size, unique_ptr<Partial
 		return false;
 	}
 	// found a partially filled block! fill in the info
-	partial_block = move(entry->second);
+	partial_block = std::move(entry->second);
 	partially_filled_blocks.erase(entry);
 
 	D_ASSERT(partial_block->state.offset_in_block > 0);
@@ -62,14 +62,14 @@ void PartialBlockManager::RegisterPartialBlock(PartialBlockAllocation &&allocati
 		// check if the block is STILL partially filled after adding the segment_size
 		if (new_space_left >= Storage::BLOCK_SIZE - max_partial_block_size) {
 			// the block is still partially filled: add it to the partially_filled_blocks list
-			partially_filled_blocks.insert(make_pair(new_space_left, move(allocation.partial_block)));
+			partially_filled_blocks.insert(make_pair(new_space_left, std::move(allocation.partial_block)));
 		}
 	}
-	auto block_to_free = move(allocation.partial_block);
+	auto block_to_free = std::move(allocation.partial_block);
 	if (!block_to_free && partially_filled_blocks.size() > MAX_BLOCK_MAP_SIZE) {
 		// Free the page with the least space free.
 		auto itr = partially_filled_blocks.begin();
-		block_to_free = move(itr->second);
+		block_to_free = std::move(itr->second);
 		partially_filled_blocks.erase(itr);
 	}
 	// Flush any block that we're not going to reuse.

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/storage/single_file_block_manager.hpp"
 
 #include "duckdb/common/allocator.hpp"
+#include "duckdb/common/checksum.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/serializer/buffered_deserializer.hpp"
 #include "duckdb/common/serializer/buffered_serializer.hpp"
@@ -30,11 +31,11 @@ void MainHeader::Serialize(Serializer &ser) {
 void MainHeader::CheckMagicBytes(FileHandle &handle) {
 	data_t magic_bytes[MAGIC_BYTE_SIZE];
 	if (handle.GetFileSize() < MainHeader::MAGIC_BYTE_SIZE + MainHeader::MAGIC_BYTE_OFFSET) {
-		throw IOException("The file is not a valid DuckDB database file!");
+		throw IOException("The file \"%s\" exists, but it is not a valid DuckDB database file!", handle.path);
 	}
 	handle.Read(magic_bytes, MainHeader::MAGIC_BYTE_SIZE, MainHeader::MAGIC_BYTE_OFFSET);
 	if (memcmp(magic_bytes, MainHeader::MAGIC_BYTES, MainHeader::MAGIC_BYTE_SIZE) != 0) {
-		throw IOException("The file is not a valid DuckDB database file!");
+		throw IOException("The file \"%s\" exists, but it is not a valid DuckDB database file!", handle.path);
 	}
 }
 
@@ -106,14 +107,14 @@ T DeserializeHeaderStructure(data_ptr_t ptr) {
 	return T::Deserialize(source);
 }
 
-SingleFileBlockManager::SingleFileBlockManager(AttachedDatabase &db, string path_p, bool read_only, bool create_new,
-                                               bool use_direct_io)
-    : BlockManager(BufferManager::GetBufferManager(db)), db(db), path(move(path_p)),
+SingleFileBlockManager::SingleFileBlockManager(AttachedDatabase &db, string path_p, bool read_only, bool use_direct_io)
+    : BlockManager(BufferManager::GetBufferManager(db)), db(db), path(std::move(path_p)),
       header_buffer(Allocator::Get(db), FileBufferType::MANAGED_BUFFER,
                     Storage::FILE_HEADER_SIZE - Storage::BLOCK_HEADER_SIZE),
       iteration_count(0), read_only(read_only), use_direct_io(use_direct_io) {
-	uint8_t flags;
-	FileLockType lock;
+}
+
+void SingleFileBlockManager::GetFileFlags(uint8_t &flags, FileLockType &lock, bool create_new) {
 	if (read_only) {
 		D_ASSERT(!create_new);
 		flags = FileFlags::FILE_FLAGS_READ;
@@ -128,71 +129,108 @@ SingleFileBlockManager::SingleFileBlockManager(AttachedDatabase &db, string path
 	if (use_direct_io) {
 		flags |= FileFlags::FILE_FLAGS_DIRECT_IO;
 	}
+}
+
+void SingleFileBlockManager::CreateNewDatabase() {
+	uint8_t flags;
+	FileLockType lock;
+	GetFileFlags(flags, lock, true);
+
 	// open the RDBMS handle
 	auto &fs = FileSystem::Get(db);
 	handle = fs.OpenFile(path, flags, lock);
-	if (create_new) {
-		// if we create a new file, we fill the metadata of the file
-		// first fill in the new header
-		header_buffer.Clear();
-
-		MainHeader main_header;
-		main_header.version_number = VERSION_NUMBER;
-		memset(main_header.flags, 0, sizeof(uint64_t) * 4);
-
-		SerializeHeaderStructure<MainHeader>(main_header, header_buffer.buffer);
-		// now write the header to the file
-		header_buffer.ChecksumAndWrite(*handle, 0);
-		header_buffer.Clear();
-
-		// write the database headers
-		// initialize meta_block and free_list to INVALID_BLOCK because the database file does not contain any actual
-		// content yet
-		DatabaseHeader h1, h2;
-		// header 1
-		h1.iteration = 0;
-		h1.meta_block = INVALID_BLOCK;
-		h1.free_list = INVALID_BLOCK;
-		h1.block_count = 0;
-		SerializeHeaderStructure<DatabaseHeader>(h1, header_buffer.buffer);
-		header_buffer.ChecksumAndWrite(*handle, Storage::FILE_HEADER_SIZE);
-		// header 2
-		h2.iteration = 0;
-		h2.meta_block = INVALID_BLOCK;
-		h2.free_list = INVALID_BLOCK;
-		h2.block_count = 0;
-		SerializeHeaderStructure<DatabaseHeader>(h2, header_buffer.buffer);
-		header_buffer.ChecksumAndWrite(*handle, Storage::FILE_HEADER_SIZE * 2);
-		// ensure that writing to disk is completed before returning
-		handle->Sync();
-		// we start with h2 as active_header, this way our initial write will be in h1
-		iteration_count = 0;
-		active_header = 1;
-		max_block = 0;
+
+	// if we create a new file, we fill the metadata of the file
+	// first fill in the new header
+	header_buffer.Clear();
+
+	MainHeader main_header;
+	main_header.version_number = VERSION_NUMBER;
+	memset(main_header.flags, 0, sizeof(uint64_t) * 4);
+
+	SerializeHeaderStructure<MainHeader>(main_header, header_buffer.buffer);
+	// now write the header to the file
+	ChecksumAndWrite(header_buffer, 0);
+	header_buffer.Clear();
+
+	// write the database headers
+	// initialize meta_block and free_list to INVALID_BLOCK because the database file does not contain any actual
+	// content yet
+	DatabaseHeader h1, h2;
+	// header 1
+	h1.iteration = 0;
+	h1.meta_block = INVALID_BLOCK;
+	h1.free_list = INVALID_BLOCK;
+	h1.block_count = 0;
+	SerializeHeaderStructure<DatabaseHeader>(h1, header_buffer.buffer);
+	ChecksumAndWrite(header_buffer, Storage::FILE_HEADER_SIZE);
+	// header 2
+	h2.iteration = 0;
+	h2.meta_block = INVALID_BLOCK;
+	h2.free_list = INVALID_BLOCK;
+	h2.block_count = 0;
+	SerializeHeaderStructure<DatabaseHeader>(h2, header_buffer.buffer);
+	ChecksumAndWrite(header_buffer, Storage::FILE_HEADER_SIZE * 2);
+	// ensure that writing to disk is completed before returning
+	handle->Sync();
+	// we start with h2 as active_header, this way our initial write will be in h1
+	iteration_count = 0;
+	active_header = 1;
+	max_block = 0;
+}
+
+void SingleFileBlockManager::LoadExistingDatabase() {
+	uint8_t flags;
+	FileLockType lock;
+	GetFileFlags(flags, lock, false);
+
+	// open the RDBMS handle
+	auto &fs = FileSystem::Get(db);
+	handle = fs.OpenFile(path, flags, lock);
+
+	MainHeader::CheckMagicBytes(*handle);
+	// otherwise, we check the metadata of the file
+	ReadAndChecksum(header_buffer, 0);
+	DeserializeHeaderStructure<MainHeader>(header_buffer.buffer);
+
+	// read the database headers from disk
+	DatabaseHeader h1, h2;
+	ReadAndChecksum(header_buffer, Storage::FILE_HEADER_SIZE);
+	h1 = DeserializeHeaderStructure<DatabaseHeader>(header_buffer.buffer);
+	ReadAndChecksum(header_buffer, Storage::FILE_HEADER_SIZE * 2);
+	h2 = DeserializeHeaderStructure<DatabaseHeader>(header_buffer.buffer);
+	// check the header with the highest iteration count
+	if (h1.iteration > h2.iteration) {
+		// h1 is active header
+		active_header = 0;
+		Initialize(h1);
 	} else {
-		MainHeader::CheckMagicBytes(*handle);
-		// otherwise, we check the metadata of the file
-		header_buffer.ReadAndChecksum(*handle, 0);
-		DeserializeHeaderStructure<MainHeader>(header_buffer.buffer);
-
-		// read the database headers from disk
-		DatabaseHeader h1, h2;
-		header_buffer.ReadAndChecksum(*handle, Storage::FILE_HEADER_SIZE);
-		h1 = DeserializeHeaderStructure<DatabaseHeader>(header_buffer.buffer);
-		header_buffer.ReadAndChecksum(*handle, Storage::FILE_HEADER_SIZE * 2);
-		h2 = DeserializeHeaderStructure<DatabaseHeader>(header_buffer.buffer);
-		// check the header with the highest iteration count
-		if (h1.iteration > h2.iteration) {
-			// h1 is active header
-			active_header = 0;
-			Initialize(h1);
-		} else {
-			// h2 is active header
-			active_header = 1;
-			Initialize(h2);
-		}
-		LoadFreeList();
+		// h2 is active header
+		active_header = 1;
+		Initialize(h2);
 	}
+	LoadFreeList();
+}
+
+void SingleFileBlockManager::ReadAndChecksum(FileBuffer &block, uint64_t location) const {
+	// read the buffer from disk
+	block.Read(*handle, location);
+	// compute the checksum
+	auto stored_checksum = Load<uint64_t>(block.InternalBuffer());
+	uint64_t computed_checksum = Checksum(block.buffer, block.size);
+	// verify the checksum
+	if (stored_checksum != computed_checksum) {
+		throw IOException("Corrupt database file: computed checksum %llu does not match stored checksum %llu in block",
+		                  computed_checksum, stored_checksum);
+	}
+}
+
+void SingleFileBlockManager::ChecksumAndWrite(FileBuffer &block, uint64_t location) const {
+	// compute the checksum and write it to the start of the buffer (if not temp buffer)
+	uint64_t checksum = Checksum(block.buffer, block.size);
+	Store<uint64_t>(checksum, block.InternalBuffer());
+	// now write the buffer
+	block.Write(*handle, location);
 }
 
 void SingleFileBlockManager::Initialize(DatabaseHeader &header) {
@@ -317,12 +355,12 @@ unique_ptr<Block> SingleFileBlockManager::CreateBlock(block_id_t block_id, FileB
 void SingleFileBlockManager::Read(Block &block) {
 	D_ASSERT(block.id >= 0);
 	D_ASSERT(std::find(free_list.begin(), free_list.end(), block.id) == free_list.end());
-	block.ReadAndChecksum(*handle, BLOCK_START + block.id * Storage::BLOCK_ALLOC_SIZE);
+	ReadAndChecksum(block, BLOCK_START + block.id * Storage::BLOCK_ALLOC_SIZE);
 }
 
 void SingleFileBlockManager::Write(FileBuffer &buffer, block_id_t block_id) {
 	D_ASSERT(block_id >= 0);
-	buffer.ChecksumAndWrite(*handle, BLOCK_START + block_id * Storage::BLOCK_ALLOC_SIZE);
+	ChecksumAndWrite(buffer, BLOCK_START + block_id * Storage::BLOCK_ALLOC_SIZE);
 }
 
 vector<block_id_t> SingleFileBlockManager::GetFreeListBlocks() {
@@ -431,8 +469,7 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 	Store<DatabaseHeader>(header, header_buffer.buffer);
 	// now write the header to the file, active_header determines whether we write to h1 or h2
 	// note that if active_header is h1 we write to h2, and vice versa
-	header_buffer.ChecksumAndWrite(*handle,
-	                               active_header == 1 ? Storage::FILE_HEADER_SIZE : Storage::FILE_HEADER_SIZE * 2);
+	ChecksumAndWrite(header_buffer, active_header == 1 ? Storage::FILE_HEADER_SIZE : Storage::FILE_HEADER_SIZE * 2);
 	// switch active header to the other header
 	active_header = 1 - active_header;
 	//! Ensure the header write ends up on disk

package/src/duckdb/src/storage/statistics/base_statistics.cpp

@@ -11,7 +11,8 @@
 
 namespace duckdb {
 
-BaseStatistics::BaseStatistics(LogicalType type, StatisticsType stats_type) : type(move(type)), stats_type(stats_type) {
+BaseStatistics::BaseStatistics(LogicalType type, StatisticsType stats_type)
+    : type(std::move(type)), stats_type(stats_type) {
 }
 
 BaseStatistics::~BaseStatistics() {
@@ -93,20 +94,20 @@ unique_ptr<BaseStatistics> BaseStatistics::CreateEmpty(LogicalType type, Statist
 	case PhysicalType::INT128:
 	case PhysicalType::FLOAT:
 	case PhysicalType::DOUBLE:
-		result = make_unique<NumericStatistics>(move(type), stats_type);
+		result = make_unique<NumericStatistics>(std::move(type), stats_type);
 		break;
 	case PhysicalType::VARCHAR:
-		result = make_unique<StringStatistics>(move(type), stats_type);
+		result = make_unique<StringStatistics>(std::move(type), stats_type);
 		break;
 	case PhysicalType::STRUCT:
-		result = make_unique<StructStatistics>(move(type));
+		result = make_unique<StructStatistics>(std::move(type));
 		break;
 	case PhysicalType::LIST:
-		result = make_unique<ListStatistics>(move(type));
+		result = make_unique<ListStatistics>(std::move(type));
 		break;
 	case PhysicalType::INTERVAL:
 	default:
-		result = make_unique<BaseStatistics>(move(type), stats_type);
+		result = make_unique<BaseStatistics>(std::move(type), stats_type);
 	}
 	result->InitializeBase();
 	return result;
@@ -163,26 +164,26 @@ unique_ptr<BaseStatistics> BaseStatistics::Deserialize(Deserializer &source, Log
 	case PhysicalType::INT128:
 	case PhysicalType::FLOAT:
 	case PhysicalType::DOUBLE:
-		result = NumericStatistics::Deserialize(reader, move(type));
+		result = NumericStatistics::Deserialize(reader, std::move(type));
 		break;
 	case PhysicalType::VARCHAR:
-		result = StringStatistics::Deserialize(reader, move(type));
+		result = StringStatistics::Deserialize(reader, std::move(type));
 		break;
 	case PhysicalType::STRUCT:
-		result = StructStatistics::Deserialize(reader, move(type));
+		result = StructStatistics::Deserialize(reader, std::move(type));
 		break;
 	case PhysicalType::LIST:
-		result = ListStatistics::Deserialize(reader, move(type));
+		result = ListStatistics::Deserialize(reader, std::move(type));
 		break;
 	case PhysicalType::INTERVAL:
-		result = make_unique<BaseStatistics>(move(type), StatisticsType::LOCAL_STATS);
+		result = make_unique<BaseStatistics>(std::move(type), StatisticsType::LOCAL_STATS);
 		break;
 	default:
 		throw InternalException("Unimplemented type for statistics deserialization");
 	}
 
 	if (ptype != PhysicalType::BIT) {
-		result->validity_stats = move(validity_stats);
+		result->validity_stats = std::move(validity_stats);
 		result->stats_type = reader.ReadField<StatisticsType>(StatisticsType::LOCAL_STATS);
 		result->distinct_stats = reader.ReadOptional<DistinctStatistics>(nullptr);
 	}

package/src/duckdb/src/storage/statistics/column_statistics.cpp

@@ -2,12 +2,12 @@
 
 namespace duckdb {
 
-ColumnStatistics::ColumnStatistics(unique_ptr<BaseStatistics> stats_p) : stats(move(stats_p)) {
+ColumnStatistics::ColumnStatistics(unique_ptr<BaseStatistics> stats_p) : stats(std::move(stats_p)) {
 }
 
 shared_ptr<ColumnStatistics> ColumnStatistics::CreateEmptyStats(const LogicalType &type) {
 	auto col_stats = BaseStatistics::CreateEmpty(type, StatisticsType::GLOBAL_STATS);
-	return make_shared<ColumnStatistics>(move(col_stats));
+	return make_shared<ColumnStatistics>(std::move(col_stats));
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/statistics/distinct_statistics.cpp

@@ -13,8 +13,8 @@ DistinctStatistics::DistinctStatistics()
 }
 
 DistinctStatistics::DistinctStatistics(unique_ptr<HyperLogLog> log, idx_t sample_count, idx_t total_count)
-    : BaseStatistics(LogicalType::INVALID, StatisticsType::LOCAL_STATS), log(move(log)), sample_count(sample_count),
-      total_count(total_count) {
+    : BaseStatistics(LogicalType::INVALID, StatisticsType::LOCAL_STATS), log(std::move(log)),
+      sample_count(sample_count), total_count(total_count) {
 }
 
 unique_ptr<BaseStatistics> DistinctStatistics::Copy() const {