duckdb 0.8.2-dev37.0 → 0.8.2-dev3989.0

package/src/duckdb/src/storage/table/persistent_table_data.cpp

@@ -3,8 +3,7 @@
 
 namespace duckdb {
 
-PersistentTableData::PersistentTableData(idx_t column_count)
-    : total_rows(0), row_group_count(0), block_id(INVALID_BLOCK), offset(0) {
+PersistentTableData::PersistentTableData(idx_t column_count) : total_rows(0), row_group_count(0) {
 }
 
 PersistentTableData::~PersistentTableData() {

package/src/duckdb/src/storage/table/row_group.cpp

@@ -1,6 +1,5 @@
 #include "duckdb/storage/table/row_group.hpp"
 #include "duckdb/common/types/vector.hpp"
-#include "duckdb/transaction/transaction.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/field_writer.hpp"
 #include "duckdb/storage/table/column_data.hpp"
@@ -11,13 +10,15 @@
 #include "duckdb/planner/table_filter.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/storage/checkpoint/table_data_writer.hpp"
-#include "duckdb/storage/meta_block_reader.hpp"
+#include "duckdb/storage/metadata/metadata_reader.hpp"
 #include "duckdb/transaction/duck_transaction_manager.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/main/attached_database.hpp"
 #include "duckdb/transaction/duck_transaction.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
 
 namespace duckdb {
 
@@ -67,6 +68,21 @@ void VersionNode::SetStart(idx_t start) {
 	}
 }
 
+idx_t VersionNode::GetCommittedDeletedCount(idx_t count) {
+	idx_t deleted_count = 0;
+	for (idx_t r = 0, i = 0; r < count; r += STANDARD_VECTOR_SIZE, i++) {
+		if (!info[i]) {
+			continue;
+		}
+		idx_t max_count = MinValue<idx_t>(STANDARD_VECTOR_SIZE, count - r);
+		if (max_count == 0) {
+			break;
+		}
+		deleted_count += info[i]->GetCommittedDeletedCount(max_count);
+	}
+	return deleted_count;
+}
+
 RowGroup::~RowGroup() {
 }
 
@@ -101,21 +117,16 @@ ColumnData &RowGroup::GetColumn(storage_t c) {
 	if (column_pointers.size() != columns.size()) {
 		throw InternalException("Lazy loading a column but the pointer was not set");
 	}
-	auto &block_manager = GetCollection().GetBlockManager();
+	auto &metadata_manager = GetCollection().GetMetadataManager();
 	auto &types = GetCollection().GetTypes();
 	auto &block_pointer = column_pointers[c];
-	MetaBlockReader column_data_reader(block_manager, block_pointer.block_id);
-	column_data_reader.offset = block_pointer.offset;
+	MetadataReader column_data_reader(metadata_manager, block_pointer);
 	this->columns[c] =
 	    ColumnData::Deserialize(GetBlockManager(), GetTableInfo(), c, start, column_data_reader, types[c], nullptr);
 	is_loaded[c] = true;
 	return *columns[c];
 }
 
-DatabaseInstance &RowGroup::GetDatabase() {
-	return GetCollection().GetDatabase();
-}
-
 BlockManager &RowGroup::GetBlockManager() {
 	return GetCollection().GetBlockManager();
 }
@@ -270,7 +281,7 @@ unique_ptr<RowGroup> RowGroup::AlterType(RowGroupCollection &new_collection, con
 }
 
 unique_ptr<RowGroup> RowGroup::AddColumn(RowGroupCollection &new_collection, ColumnDefinition &new_column,
-                                         ExpressionExecutor &executor, Expression *default_value, Vector &result) {
+                                         ExpressionExecutor &executor, Expression &default_value, Vector &result) {
 	Verify();
 
 	// construct a new column data for the new column
@@ -285,10 +296,8 @@ unique_ptr<RowGroup> RowGroup::AddColumn(RowGroupCollection &new_collection, Col
 		added_column->InitializeAppend(state);
 		for (idx_t i = 0; i < rows_to_write; i += STANDARD_VECTOR_SIZE) {
 			idx_t rows_in_this_vector = MinValue<idx_t>(rows_to_write - i, STANDARD_VECTOR_SIZE);
-			if (default_value) {
-				dummy_chunk.SetCardinality(rows_in_this_vector);
-				executor.ExecuteExpression(dummy_chunk, result);
-			}
+			dummy_chunk.SetCardinality(rows_in_this_vector);
+			executor.ExecuteExpression(dummy_chunk, result);
 			added_column->Append(state, result, rows_in_this_vector);
 		}
 	}
@@ -796,6 +805,13 @@ RowGroupWriteData RowGroup::WriteToDisk(PartialBlockManager &manager,
 	return result;
 }
 
+bool RowGroup::AllDeleted() {
+	if (!version_info) {
+		return false;
+	}
+	return version_info->GetCommittedDeletedCount(count) == count;
+}
+
 RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats) {
 	RowGroupPointer row_group_pointer;
 
@@ -816,7 +832,7 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, TableStatistics &gl
 	for (auto &state : result.states) {
 		// get the current position of the table data writer
 		auto &data_writer = writer.GetPayloadWriter();
-		auto pointer = data_writer.GetBlockPointer();
+		auto pointer = data_writer.GetMetaBlockPointer();
 
 		// store the stats and the data pointers in the row group pointers
 		row_group_pointer.data_pointers.push_back(pointer);
@@ -882,8 +898,8 @@ void RowGroup::Serialize(RowGroupPointer &pointer, Serializer &main_serializer)
 	writer.WriteField<uint64_t>(pointer.tuple_count);
 	auto &serializer = writer.GetSerializer();
 	for (auto &data_pointer : pointer.data_pointers) {
-		serializer.Write<block_id_t>(data_pointer.block_id);
-		serializer.Write<uint64_t>(data_pointer.offset);
+		serializer.Write<idx_t>(data_pointer.block_pointer);
+		serializer.Write<uint32_t>(data_pointer.offset);
 	}
 	CheckpointDeletes(pointer.versions.get(), serializer);
 	writer.Finalize();
@@ -901,9 +917,9 @@ RowGroupPointer RowGroup::Deserialize(Deserializer &main_source, const vector<Lo
 
 	auto &source = reader.GetSource();
 	for (idx_t i = 0; i < physical_columns; i++) {
-		BlockPointer pointer;
-		pointer.block_id = source.Read<block_id_t>();
-		pointer.offset = source.Read<uint64_t>();
+		MetaBlockPointer pointer;
+		pointer.block_pointer = source.Read<idx_t>();
+		pointer.offset = source.Read<uint32_t>();
 		result.data_pointers.push_back(pointer);
 	}
 	result.versions = DeserializeDeletes(source);
@@ -912,6 +928,72 @@ RowGroupPointer RowGroup::Deserialize(Deserializer &main_source, const vector<Lo
 	return result;
 }
 
+void RowGroup::FormatSerialize(RowGroupPointer &pointer, FormatSerializer &serializer) {
+	serializer.WriteProperty(100, "row_start", pointer.row_start);
+	serializer.WriteProperty(101, "tuple_count", pointer.tuple_count);
+	serializer.WriteProperty(102, "data_pointers", pointer.data_pointers);
+
+	// Checkpoint deletes
+	auto versions = pointer.versions.get();
+
+	if (!versions) {
+		// no version information: write nothing
+		serializer.WriteProperty(103, "versions_count", 0);
+		return;
+	}
+	// first count how many ChunkInfo's we need to deserialize
+	idx_t chunk_info_count = 0;
+	idx_t idx_map[ROW_GROUP_VECTOR_COUNT];
+	for (idx_t vector_idx = 0; vector_idx < RowGroup::ROW_GROUP_VECTOR_COUNT; vector_idx++) {
+		auto chunk_info = versions->info[vector_idx].get();
+		if (!chunk_info) {
+			continue;
+		}
+		idx_map[chunk_info_count++] = vector_idx;
+	}
+
+	// now serialize the actual version information
+	serializer.WriteProperty(103, "versions_count", chunk_info_count);
+	serializer.WriteList(104, "versions", chunk_info_count, [&](FormatSerializer::List &list, idx_t i) {
+		auto vector_idx = idx_map[i];
+		auto chunk_info = versions->info[vector_idx].get();
+		list.WriteObject([&](FormatSerializer &obj) {
+			obj.WriteProperty(100, "vector_index", vector_idx);
+			obj.WriteProperty(101, "chunk_info", const_cast<const ChunkInfo *>(chunk_info));
+		});
+	});
+}
+
+RowGroupPointer RowGroup::FormatDeserialize(FormatDeserializer &deserializer) {
+	RowGroupPointer result;
+	result.row_start = deserializer.ReadProperty<uint64_t>(100, "row_start");
+	result.tuple_count = deserializer.ReadProperty<uint64_t>(101, "tuple_count");
+	result.data_pointers = deserializer.ReadProperty<vector<MetaBlockPointer>>(102, "data_pointers");
+	result.versions = nullptr;
+	// Deserialize Deletes
+	auto chunk_count = deserializer.ReadProperty<idx_t>(103, "versions_count");
+	if (chunk_count == 0) {
+		// no deletes
+		return result;
+	}
+
+	auto version_info = make_shared<VersionNode>();
+	deserializer.ReadList(104, "versions", [&](FormatDeserializer::List &list, idx_t i) {
+		list.ReadObject([&](FormatDeserializer &obj) {
+			auto vector_index = obj.ReadProperty<idx_t>(100, "vector_index");
+			if (vector_index >= RowGroup::ROW_GROUP_VECTOR_COUNT) {
+				throw Exception("In DeserializeDeletes, vector_index is out of range for the row group. Corrupted "
+				                "file?");
+			}
+			version_info->info[vector_index] = obj.ReadProperty<unique_ptr<ChunkInfo>>(101, "chunk_info");
+		});
+	});
+
+	result.versions = version_info;
+
+	return result;
+}
+
 //===--------------------------------------------------------------------===//
 // GetColumnSegmentInfo
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -6,7 +6,7 @@
 #include "duckdb/planner/constraints/bound_not_null_constraint.hpp"
 #include "duckdb/storage/checkpoint/table_data_writer.hpp"
 #include "duckdb/storage/table/row_group_segment_tree.hpp"
-#include "duckdb/storage/meta_block_reader.hpp"
+#include "duckdb/storage/metadata/metadata_reader.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/table_storage_info.hpp"
@@ -27,8 +27,7 @@ void RowGroupSegmentTree::Initialize(PersistentTableData &data) {
 	current_row_group = 0;
 	max_row_group = data.row_group_count;
 	finished_loading = false;
-	reader = make_uniq<MetaBlockReader>(collection.GetBlockManager(), data.block_id);
-	reader->offset = data.offset;
+	reader = make_uniq<MetadataReader>(collection.GetMetadataManager(), data.block_pointer);
 }
 
 unique_ptr<RowGroup> RowGroupSegmentTree::LoadSegment() {
@@ -67,8 +66,8 @@ AttachedDatabase &RowGroupCollection::GetAttached() {
 	return GetTableInfo().db;
 }
 
-DatabaseInstance &RowGroupCollection::GetDatabase() {
-	return GetAttached().GetDatabase();
+MetadataManager &RowGroupCollection::GetMetadataManager() {
+	return GetBlockManager().GetMetadataManager();
 }
 
 //===--------------------------------------------------------------------===//
@@ -97,10 +96,6 @@ RowGroup *RowGroupCollection::GetRowGroup(int64_t index) {
 	return (RowGroup *)row_groups->GetSegmentByIndex(index);
 }
 
-idx_t RowGroupCollection::RowGroupCount() {
-	return row_groups->GetSegmentCount();
-}
-
 void RowGroupCollection::Verify() {
 #ifdef DEBUG
 	idx_t current_total_rows = 0;
@@ -600,11 +595,24 @@ void RowGroupCollection::UpdateColumn(TransactionData transaction, Vector &row_i
 // Checkpoint
 //===--------------------------------------------------------------------===//
 void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &global_stats) {
-	for (auto &row_group : row_groups->Segments()) {
-		auto rowg_writer = writer.GetRowGroupWriter(row_group);
-		auto pointer = row_group.Checkpoint(*rowg_writer, global_stats);
-		writer.AddRowGroup(std::move(pointer), std::move(rowg_writer));
+	bool can_vacuum_deletes = info->indexes.Empty();
+	idx_t start = this->row_start;
+	auto segments = row_groups->MoveSegments();
+	auto l = row_groups->Lock();
+	for (auto &entry : segments) {
+		auto &row_group = *entry.node;
+		if (can_vacuum_deletes && row_group.AllDeleted()) {
+			row_group.CommitDrop();
+			continue;
+		}
+		row_group.MoveToCollection(*this, start);
+		auto row_group_writer = writer.GetRowGroupWriter(row_group);
+		auto pointer = row_group.Checkpoint(*row_group_writer, global_stats);
+		writer.AddRowGroup(std::move(pointer), std::move(row_group_writer));
+		row_groups->AppendSegment(l, std::move(entry.node));
+		start += row_group.count;
 	}
+	total_rows = start;
 }
 
 //===--------------------------------------------------------------------===//
@@ -637,7 +645,7 @@ vector<ColumnSegmentInfo> RowGroupCollection::GetColumnSegmentInfo() {
 // Alter
 //===--------------------------------------------------------------------===//
 shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ClientContext &context, ColumnDefinition &new_column,
-                                                             Expression *default_value) {
+                                                             Expression &default_value) {
 	idx_t new_column_idx = types.size();
 	auto new_types = types;
 	new_types.push_back(new_column.GetType());
@@ -647,11 +655,7 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ClientContext &cont
 	ExpressionExecutor executor(context);
 	DataChunk dummy_chunk;
 	Vector default_vector(new_column.GetType());
-	if (!default_value) {
-		FlatVector::Validity(default_vector).SetAllInvalid(STANDARD_VECTOR_SIZE);
-	} else {
-		executor.AddExpression(*default_value);
-	}
+	executor.AddExpression(default_value);
 
 	result->stats.InitializeAddColumn(stats, new_column.GetType());
 	auto &new_column_stats = result->stats.GetStats(new_column_idx);

package/src/duckdb/src/storage/table/table_statistics.cpp

@@ -1,5 +1,7 @@
 #include "duckdb/storage/table/table_statistics.hpp"
 #include "duckdb/storage/table/persistent_table_data.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
 
 namespace duckdb {
 
@@ -112,6 +114,25 @@ void TableStatistics::Deserialize(Deserializer &source, ColumnList &columns) {
 	}
 }
 
+void TableStatistics::FormatSerialize(FormatSerializer &serializer) {
+	auto column_count = column_stats.size();
+	serializer.WriteList(100, "column_stats", column_count,
+	                     [&](FormatSerializer::List &list, idx_t i) { list.WriteElement(column_stats[i]); });
+}
+
+void TableStatistics::FormatDeserialize(FormatDeserializer &deserializer, ColumnList &columns) {
+	auto physical_columns = columns.Physical();
+	auto iter = physical_columns.begin();
+	deserializer.ReadList(100, "column_stats", [&](FormatDeserializer::List &list, idx_t i) {
+		auto &col = *iter.operator++();
+		auto type = col.GetType();
+		deserializer.Set<LogicalType &>(type);
+		auto stats = ColumnStatistics::FormatDeserialize(deserializer);
+		deserializer.Unset<LogicalType>();
+		column_stats.push_back(std::move(stats));
+	});
+}
+
 unique_ptr<TableStatisticsLock> TableStatistics::GetLock() {
 	return make_uniq<TableStatisticsLock>(stats_lock);
 }

package/src/duckdb/src/storage/table/update_segment.cpp

@@ -427,7 +427,7 @@ void UpdateSegment::FetchRow(TransactionData transaction, idx_t row_id, Vector &
 	if (!root->info[vector_index]) {
 		return;
 	}
-	idx_t row_in_vector = row_id - vector_index * STANDARD_VECTOR_SIZE;
+	idx_t row_in_vector = (row_id - column_data.start) - vector_index * STANDARD_VECTOR_SIZE;
 	fetch_row_function(transaction.start_time, transaction.transaction_id, root->info[vector_index]->info.get(),
 	                   row_in_vector, result, result_idx);
 }

package/src/duckdb/src/storage/table_index_list.cpp

@@ -79,7 +79,7 @@ vector<column_t> TableIndexList::GetRequiredColumns() {
 	return result;
 }
 
-vector<BlockPointer> TableIndexList::SerializeIndexes(duckdb::MetaBlockWriter &writer) {
+vector<BlockPointer> TableIndexList::SerializeIndexes(duckdb::MetadataWriter &writer) {
 	vector<BlockPointer> blocks_info;
 	for (auto &index : indexes) {
 		blocks_info.emplace_back(index->Serialize(writer));

package/src/duckdb/src/storage/wal_replay.cpp

@@ -36,7 +36,6 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 	// first deserialize the WAL to look for a checkpoint flag
 	// if there is a checkpoint flag, we might have already flushed the contents of the WAL to disk
 	ReplayState checkpoint_state(database, *con.context, *initial_reader);
-	initial_reader->SetCatalog(checkpoint_state.catalog);
 	checkpoint_state.deserialize_only = true;
 	try {
 		while (true) {
@@ -54,14 +53,14 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 			}
 		}
 	} catch (std::exception &ex) { // LCOV_EXCL_START
-		Printer::Print(StringUtil::Format("Exception in WAL playback during initial read: %s\n", ex.what()));
+		Printer::PrintF("Exception in WAL playback during initial read: %s\n", ex.what());
 		return false;
 	} catch (...) {
 		Printer::Print("Unknown Exception in WAL playback during initial read");
 		return false;
 	} // LCOV_EXCL_STOP
 	initial_reader.reset();
-	if (checkpoint_state.checkpoint_id != INVALID_BLOCK) {
+	if (checkpoint_state.checkpoint_id.IsValid()) {
 		// there is a checkpoint flag: check if we need to deserialize the WAL
 		auto &manager = database.GetStorageManager();
 		if (manager.IsCheckpointClean(checkpoint_state.checkpoint_id)) {
@@ -73,7 +72,6 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 
 	// we need to recover from the WAL: actually set up the replay state
 	BufferedFileReader reader(FileSystem::Get(database), path.c_str(), con.context.get());
-	reader.SetCatalog(checkpoint_state.catalog);
 	ReplayState state(database, *con.context, reader);
 
 	// replay the WAL
@@ -101,7 +99,7 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 		}
 	} catch (std::exception &ex) { // LCOV_EXCL_START
 		// FIXME: this should report a proper warning in the connection
-		Printer::Print(StringUtil::Format("Exception in WAL playback: %s\n", ex.what()));
+		Printer::PrintF("Exception in WAL playback: %s\n", ex.what());
 		// exception thrown in WAL replay: rollback
 		con.Rollback();
 	} catch (...) {
@@ -195,7 +193,7 @@ void ReplayState::ReplayEntry(WALType entry_type) {
 // Replay Table
 //===--------------------------------------------------------------------===//
 void ReplayState::ReplayCreateTable() {
-	auto info = TableCatalogEntry::Deserialize(source, context);
+	auto info = TableCatalogEntry::Deserialize(source);
 	if (deserialize_only) {
 		return;
 	}
@@ -234,12 +232,12 @@ void ReplayState::ReplayAlter() {
 // Replay View
 //===--------------------------------------------------------------------===//
 void ReplayState::ReplayCreateView() {
-	auto entry = ViewCatalogEntry::Deserialize(source, context);
+	auto entry = CatalogEntry::Deserialize(source);
 	if (deserialize_only) {
 		return;
 	}
 
-	catalog.CreateView(context, *entry);
+	catalog.CreateView(context, entry->Cast<CreateViewInfo>());
 }
 
 void ReplayState::ReplayDropView() {
@@ -284,7 +282,7 @@ void ReplayState::ReplayDropSchema() {
 void ReplayState::ReplayCreateType() {
 	auto info = TypeCatalogEntry::Deserialize(source);
 	info->on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT;
-	catalog.CreateType(context, *info);
+	catalog.CreateType(context, info->Cast<CreateTypeInfo>());
 }
 
 void ReplayState::ReplayDropType() {
@@ -309,7 +307,7 @@ void ReplayState::ReplayCreateSequence() {
 		return;
 	}
 
-	catalog.CreateSequence(context, *entry);
+	catalog.CreateSequence(context, entry->Cast<CreateSequenceInfo>());
 }
 
 void ReplayState::ReplayDropSequence() {
@@ -345,12 +343,12 @@ void ReplayState::ReplaySequenceValue() {
 // Replay Macro
 //===--------------------------------------------------------------------===//
 void ReplayState::ReplayCreateMacro() {
-	auto entry = ScalarMacroCatalogEntry::Deserialize(source, context);
+	auto entry = ScalarMacroCatalogEntry::Deserialize(source);
 	if (deserialize_only) {
 		return;
 	}
 
-	catalog.CreateFunction(context, *entry);
+	catalog.CreateFunction(context, entry->Cast<CreateMacroInfo>());
 }
 
 void ReplayState::ReplayDropMacro() {
@@ -369,12 +367,12 @@ void ReplayState::ReplayDropMacro() {
 // Replay Table Macro
 //===--------------------------------------------------------------------===//
 void ReplayState::ReplayCreateTableMacro() {
-	auto entry = TableMacroCatalogEntry::Deserialize(source, context);
+	auto entry = TableMacroCatalogEntry::Deserialize(source);
 	if (deserialize_only) {
 		return;
 	}
 
-	catalog.CreateFunction(context, *entry);
+	catalog.CreateFunction(context, entry->Cast<CreateMacroInfo>());
 }
 
 void ReplayState::ReplayDropTableMacro() {
@@ -393,30 +391,31 @@ void ReplayState::ReplayDropTableMacro() {
 // Replay Index
 //===--------------------------------------------------------------------===//
 void ReplayState::ReplayCreateIndex() {
-	auto info = IndexCatalogEntry::Deserialize(source, context);
+	auto info = IndexCatalogEntry::Deserialize(source);
 	if (deserialize_only) {
 		return;
 	}
+	auto &index_info = info->Cast<CreateIndexInfo>();
 
 	// get the physical table to which we'll add the index
-	auto &table = catalog.GetEntry<TableCatalogEntry>(context, info->schema, info->table->table_name);
+	auto &table = catalog.GetEntry<TableCatalogEntry>(context, info->schema, index_info.table);
 	auto &data_table = table.GetStorage();
 
 	// bind the parsed expressions
-	if (info->expressions.empty()) {
-		for (auto &parsed_expr : info->parsed_expressions) {
-			info->expressions.push_back(parsed_expr->Copy());
+	if (index_info.expressions.empty()) {
+		for (auto &parsed_expr : index_info.parsed_expressions) {
+			index_info.expressions.push_back(parsed_expr->Copy());
 		}
 	}
 	auto binder = Binder::CreateBinder(context);
-	auto expressions = binder->BindCreateIndexExpressions(table, *info);
+	auto expressions = binder->BindCreateIndexExpressions(table, index_info);
 
 	// create the empty index
 	unique_ptr<Index> index;
-	switch (info->index_type) {
+	switch (index_info.index_type) {
 	case IndexType::ART: {
-		index = make_uniq<ART>(info->column_ids, TableIOManager::Get(data_table), expressions, info->constraint_type,
-		                       data_table.db);
+		index = make_uniq<ART>(index_info.column_ids, TableIOManager::Get(data_table), expressions,
+		                       index_info.constraint_type, data_table.db);
 		break;
 	}
 	default:
@@ -424,10 +423,10 @@ void ReplayState::ReplayCreateIndex() {
 	}
 
 	// add the index to the catalog
-	auto &index_entry = catalog.CreateIndex(context, *info)->Cast<DuckIndexEntry>();
+	auto &index_entry = catalog.CreateIndex(context, index_info)->Cast<DuckIndexEntry>();
 	index_entry.index = index.get();
 	index_entry.info = data_table.info;
-	for (auto &parsed_expr : info->parsed_expressions) {
+	for (auto &parsed_expr : index_info.parsed_expressions) {
 		index_entry.parsed_expressions.push_back(parsed_expr->Copy());
 	}
 
@@ -524,7 +523,8 @@ void ReplayState::ReplayUpdate() {
 }
 
 void ReplayState::ReplayCheckpoint() {
-	checkpoint_id = source.Read<block_id_t>();
+	checkpoint_id.block_pointer = source.Read<idx_t>();
+	checkpoint_id.offset = source.Read<uint32_t>();
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/write_ahead_log.cpp

@@ -48,9 +48,10 @@ void WriteAheadLog::Delete() {
 //===--------------------------------------------------------------------===//
 // Write Entries
 //===--------------------------------------------------------------------===//
-void WriteAheadLog::WriteCheckpoint(block_id_t meta_block) {
+void WriteAheadLog::WriteCheckpoint(MetaBlockPointer meta_block) {
 	writer->Write<WALType>(WALType::CHECKPOINT);
-	writer->Write<block_id_t>(meta_block);
+	writer->Write<idx_t>(meta_block.block_pointer);
+	writer->Write<uint32_t>(meta_block.offset);
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/verification/deserialized_statement_verifier_v2.cpp

@@ -11,10 +11,24 @@ DeserializedStatementVerifierV2::DeserializedStatementVerifierV2(unique_ptr<SQLS
 unique_ptr<StatementVerifier> DeserializedStatementVerifierV2::Create(const SQLStatement &statement) {
 	auto &select_stmt = statement.Cast<SelectStatement>();
 
-	auto blob = BinarySerializer::Serialize(select_stmt);
+	auto blob = BinarySerializer::Serialize(select_stmt, true);
 	auto result = BinaryDeserializer::Deserialize<SelectStatement>(blob.data(), blob.size());
 
 	return make_uniq<DeserializedStatementVerifierV2>(std::move(result));
 }
 
+DeserializedStatementVerifierNoDefaultV2::DeserializedStatementVerifierNoDefaultV2(unique_ptr<SQLStatement> statement_p)
+    : StatementVerifier(VerificationType::DESERIALIZED_V2_NO_DEFAULT, "Deserialized V2 without default values",
+                        std::move(statement_p)) {
+}
+
+unique_ptr<StatementVerifier> DeserializedStatementVerifierNoDefaultV2::Create(const SQLStatement &statement) {
+	auto &select_stmt = statement.Cast<SelectStatement>();
+
+	auto blob = BinarySerializer::Serialize(select_stmt, false);
+	auto result = BinaryDeserializer::Deserialize<SelectStatement>(blob.data(), blob.size());
+
+	return make_uniq<DeserializedStatementVerifierNoDefaultV2>(std::move(result));
+}
+
 } // namespace duckdb

package/src/duckdb/src/verification/prepared_statement_verifier.cpp

@@ -23,6 +23,9 @@ void PreparedStatementVerifier::Extract() {
 	ParsedExpressionIterator::EnumerateQueryNodeChildren(
 	    *select.node, [&](unique_ptr<ParsedExpression> &child) { ConvertConstants(child); });
 	statement->n_param = values.size();
+	for (auto &kv : values) {
+		statement->named_param_map[kv.first] = 0;
+	}
 	// create the PREPARE and EXECUTE statements
 	string name = "__duckdb_verification_prepared_statement";
 	auto prepare = make_uniq<PrepareStatement>();
@@ -31,7 +34,7 @@ void PreparedStatementVerifier::Extract() {
 
 	auto execute = make_uniq<ExecuteStatement>();
 	execute->name = name;
-	execute->values = std::move(values);
+	execute->named_values = std::move(values);
 
 	auto dealloc = make_uniq<DropStatement>();
 	dealloc->info->type = CatalogType::PREPARED_STATEMENT;
@@ -49,19 +52,21 @@ void PreparedStatementVerifier::ConvertConstants(unique_ptr<ParsedExpression> &c
 		child->alias = string();
 		// check if the value already exists
 		idx_t index = values.size();
-		for (idx_t v_idx = 0; v_idx < values.size(); v_idx++) {
-			if (values[v_idx]->Equals(*child)) {
-				// duplicate value! refer to the original value
-				index = v_idx;
-				break;
-			}
-		}
-		if (index == values.size()) {
-			values.push_back(std::move(child));
+		auto identifier = std::to_string(index + 1);
+		const auto predicate = [&](const std::pair<const string, unique_ptr<ParsedExpression>> &pair) {
+			return pair.second->Equals(*child.get());
+		};
+		auto result = std::find_if(values.begin(), values.end(), predicate);
+		if (result == values.end()) {
+			// If it doesn't exist yet, add it
+			values[identifier] = std::move(child);
+		} else {
+			identifier = result->first;
 		}
+
 		// replace it with an expression
 		auto parameter = make_uniq<ParameterExpression>();
-		parameter->parameter_nr = index + 1;
+		parameter->identifier = identifier;
 		parameter->alias = alias;
 		child = std::move(parameter);
 		return;

package/src/duckdb/src/verification/statement_verifier.cpp

@@ -35,6 +35,8 @@ unique_ptr<StatementVerifier> StatementVerifier::Create(VerificationType type, c
 		return DeserializedStatementVerifier::Create(statement_p);
 	case VerificationType::DESERIALIZED_V2:
 		return DeserializedStatementVerifierV2::Create(statement_p);
+	case VerificationType::DESERIALIZED_V2_NO_DEFAULT:
+		return DeserializedStatementVerifierNoDefaultV2::Create(statement_p);
 	case VerificationType::PARSED:
 		return ParsedStatementVerifier::Create(statement_p);
 	case VerificationType::UNOPTIMIZED:

package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h

@@ -80,7 +80,7 @@ namespace duckdb_moodycamel { namespace details {
 	static const thread_id_t invalid_thread_id2 = 0xFFFFFFFFU;	// Not technically guaranteed to be invalid, but is never used in practice. Note that all Win32 thread IDs are presently multiples of 4.
 	static inline thread_id_t thread_id() { return static_cast<thread_id_t>(::GetCurrentThreadId()); }
 } }
-#elif defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || (defined(__APPLE__) && TARGET_OS_IPHONE)
+#elif defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || (defined(__APPLE__) && TARGET_OS_IPHONE) || defined(__MVS__)
 namespace duckdb_moodycamel { namespace details {
 	static_assert(sizeof(std::thread::id) == 4 || sizeof(std::thread::id) == 8, "std::thread::id is expected to be either 4 or 8 bytes");
 	
@@ -194,7 +194,7 @@ namespace duckdb_moodycamel { namespace details {
 // VS2013 doesn't support `thread_local`, and MinGW-w64 w/ POSIX threading has a crippling bug: http://sourceforge.net/p/mingw-w64/bugs/445
 // g++ <=4.7 doesn't support thread_local either.
 // Finally, iOS/ARM doesn't have support for it either, and g++/ARM allows it to compile but it's unconfirmed to actually work
-#if (!defined(_MSC_VER) || _MSC_VER >= 1900) && (!defined(__MINGW32__) && !defined(__MINGW64__) || !defined(__WINPTHREADS_VERSION)) && (!defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__)
+#if (!defined(_MSC_VER) || _MSC_VER >= 1900) && (!defined(__MINGW32__) && !defined(__MINGW64__) || !defined(__WINPTHREADS_VERSION)) && (!defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__) && !defined(__MVS__)
 // Assume `thread_local` is fully supported in all other C++11 compilers/platforms
 //#define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED    // always disabled for now since several users report having problems with it on
 #endif