duckdb 0.8.2-dev2673.0 → 0.8.2-dev2809.0

package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp

@@ -9,47 +9,32 @@
 #pragma once
 
 #include "duckdb/common/common.hpp"
+#include "duckdb/common/optional_ptr.hpp"
+#include "duckdb/optimizer/join_order/join_relation.hpp"
+#include "duckdb/optimizer/join_order/join_node.hpp"
+#include "duckdb/optimizer/join_order/relation_manager.hpp"
 #include "duckdb/common/pair.hpp"
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/common/unordered_set.hpp"
-#include "duckdb/optimizer/join_order/join_relation.hpp"
 #include "duckdb/common/vector.hpp"
 #include "duckdb/planner/column_binding.hpp"
-#include "duckdb/common/optional_ptr.hpp"
 
 #include <functional>
 
 namespace duckdb {
-class Expression;
-class LogicalOperator;
-
-struct FilterInfo {
-	FilterInfo(JoinRelationSet &set, idx_t filter_index) : set(set), filter_index(filter_index) {
-	}
-
-	JoinRelationSet &set;
-	idx_t filter_index;
-	optional_ptr<JoinRelationSet> left_set;
-	optional_ptr<JoinRelationSet> right_set;
-	ColumnBinding left_binding;
-	ColumnBinding right_binding;
-};
 
-struct FilterNode {
-	vector<reference<FilterInfo>> filters;
-	unordered_map<idx_t, unique_ptr<FilterNode>> children;
-};
+struct FilterInfo;
 
 struct NeighborInfo {
-	NeighborInfo(JoinRelationSet &neighbor) : neighbor(neighbor) {
+	NeighborInfo(optional_ptr<JoinRelationSet> neighbor) : neighbor(neighbor) {
 	}
 
-	JoinRelationSet &neighbor;
-	vector<reference<FilterInfo>> filters;
+	optional_ptr<JoinRelationSet> neighbor;
+	vector<optional_ptr<FilterInfo>> filters;
 };
 
 //! The QueryGraph contains edges between relations and allows edges to be created/queried
-class QueryGraph {
+class QueryGraphEdges {
 public:
 	//! Contains a node with info about neighboring relations and child edge infos
 	struct QueryEdge {
@@ -61,22 +46,23 @@ public:
 	string ToString() const;
 	void Print();
 
-	//! Create an edge in the edge_set
-	void CreateEdge(JoinRelationSet &left, JoinRelationSet &right, optional_ptr<FilterInfo> info);
 	//! Returns a connection if there is an edge that connects these two sets, or nullptr otherwise
-	vector<reference<NeighborInfo>> GetConnections(JoinRelationSet &node, JoinRelationSet &other);
+	const vector<reference<NeighborInfo>> GetConnections(JoinRelationSet &node, JoinRelationSet &other) const;
 	//! Enumerate the neighbors of a specific node that do not belong to any of the exclusion_set. Note that if a
 	//! neighbor has multiple nodes, this function will return the lowest entry in that set.
-	vector<idx_t> GetNeighbors(JoinRelationSet &node, unordered_set<idx_t> &exclusion_set);
+	const vector<idx_t> GetNeighbors(JoinRelationSet &node, unordered_set<idx_t> &exclusion_set) const;
+
 	//! Enumerate all neighbors of a given JoinRelationSet node
-	void EnumerateNeighbors(JoinRelationSet &node, const std::function<bool(NeighborInfo &)> &callback);
+	void EnumerateNeighbors(JoinRelationSet &node, const std::function<bool(NeighborInfo &)> &callback) const;
+	//! Create an edge in the edge_set
+	void CreateEdge(JoinRelationSet &left, JoinRelationSet &right, optional_ptr<FilterInfo> info);
 
 private:
 	//! Get the QueryEdge of a specific node
-	QueryEdge &GetQueryEdge(JoinRelationSet &left);
+	optional_ptr<QueryEdge> GetQueryEdge(JoinRelationSet &left);
 
 	void EnumerateNeighborsDFS(JoinRelationSet &node, reference<QueryEdge> info, idx_t index,
-	                           const std::function<bool(NeighborInfo &)> &callback);
+	                           const std::function<bool(NeighborInfo &)> &callback) const;
 
 	QueryEdge root;
 };

package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph_manager.hpp

@@ -0,0 +1,113 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/optimizer/join_order/query_graph_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/optional_ptr.hpp"
+#include "duckdb/common/pair.hpp"
+#include "duckdb/common/unordered_map.hpp"
+#include "duckdb/common/unordered_set.hpp"
+#include "duckdb/common/vector.hpp"
+#include "duckdb/optimizer/join_order/join_node.hpp"
+#include "duckdb/optimizer/join_order/join_relation.hpp"
+#include "duckdb/optimizer/join_order/query_graph.hpp"
+#include "duckdb/optimizer/join_order/relation_manager.hpp"
+#include "duckdb/planner/column_binding.hpp"
+#include "duckdb/planner/logical_operator.hpp"
+
+#include <functional>
+
+namespace duckdb {
+
+struct GenerateJoinRelation {
+	GenerateJoinRelation(optional_ptr<JoinRelationSet> set, unique_ptr<LogicalOperator> op_p)
+	    : set(set), op(std::move(op_p)) {
+	}
+
+	optional_ptr<JoinRelationSet> set;
+	unique_ptr<LogicalOperator> op;
+};
+
+//! Filter info struct that is used by the cardinality estimator to set the initial cardinality
+//! but is also eventually transformed into a query edge.
+struct FilterInfo {
+	FilterInfo(unique_ptr<Expression> filter, JoinRelationSet &set, idx_t filter_index)
+	    : filter(std::move(filter)), set(set), filter_index(filter_index) {
+	}
+
+	unique_ptr<Expression> filter;
+	JoinRelationSet &set;
+	idx_t filter_index;
+	optional_ptr<JoinRelationSet> left_set;
+	optional_ptr<JoinRelationSet> right_set;
+	ColumnBinding left_binding;
+	ColumnBinding right_binding;
+};
+
+//! The QueryGraphManager manages the process of extracting the reorderable and nonreorderable operations
+//! from the logical plan and creating the intermediate structures needed by the plan enumerator.
+//! When the plan enumerator finishes, the Query Graph Manger can then recreate the logical plan.
+class QueryGraphManager {
+public:
+	QueryGraphManager(ClientContext &context) : relation_manager(context), context(context) {
+	}
+
+	//! manage relations and the logical operators they represent
+	RelationManager relation_manager;
+
+	//! A structure holding all the created JoinRelationSet objects
+	JoinRelationSetManager set_manager;
+
+	ClientContext &context;
+
+	//! Extract the join relations, optimizing non-reoderable relations when encountered
+	bool Build(LogicalOperator &op);
+
+	//! Reconstruct the logical plan using the plan found by the plan enumerator
+	unique_ptr<LogicalOperator> Reconstruct(unique_ptr<LogicalOperator> plan, JoinNode &node);
+
+	//! Get a reference to the QueryGraphEdges structure that stores edges between
+	//! nodes and hypernodes.
+	const QueryGraphEdges &GetQueryGraphEdges() const;
+
+	//! Get a list of the join filters in the join plan than eventually are
+	//! transformed into the query graph edges
+	const vector<unique_ptr<FilterInfo>> &GetFilterBindings() const;
+
+	//! Plan enumerator may not find a full plan and therefore will need to create cross
+	//! products to create edges.
+	void CreateQueryGraphCrossProduct(JoinRelationSet &left, JoinRelationSet &right);
+
+	//! after join order optimization, we perform build side probe side optimizations.
+	//! (Basically we put lower expected cardinality columns on the build side, and larger
+	//! tables on the probe side)
+	unique_ptr<LogicalOperator> LeftRightOptimizations(unique_ptr<LogicalOperator> op);
+
+private:
+	vector<reference<LogicalOperator>> filter_operators;
+
+	//! Filter information including the column_bindings that join filters
+	//! used by the cardinality estimator to estimate distinct counts
+	vector<unique_ptr<FilterInfo>> filters_and_bindings;
+
+	QueryGraphEdges query_graph;
+
+	void GetColumnBinding(Expression &expression, ColumnBinding &binding);
+
+	bool ExtractBindings(Expression &expression, unordered_set<idx_t> &bindings);
+	bool LeftCardLessThanRight(LogicalOperator &op);
+
+	void CreateHyperGraphEdges();
+
+	GenerateJoinRelation GenerateJoins(vector<unique_ptr<LogicalOperator>> &extracted_relations, JoinNode &node);
+
+	unique_ptr<LogicalOperator> RewritePlan(unique_ptr<LogicalOperator> plan, JoinNode &node);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/optimizer/join_order/relation_manager.hpp

@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/optimizer/join_order/relation_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/unordered_map.hpp"
+#include "duckdb/common/unordered_set.hpp"
+#include "duckdb/optimizer/join_order/join_relation.hpp"
+#include "duckdb/optimizer/join_order/cardinality_estimator.hpp"
+#include "duckdb/optimizer/join_order/relation_statistics_helper.hpp"
+#include "duckdb/optimizer/join_order/join_node.hpp"
+#include "duckdb/parser/expression_map.hpp"
+#include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/planner/logical_operator_visitor.hpp"
+
+namespace duckdb {
+
+struct FilterInfo;
+
+//! Represents a single relation and any metadata accompanying that relation
+struct SingleJoinRelation {
+	LogicalOperator &op;
+	optional_ptr<LogicalOperator> parent;
+	RelationStats stats;
+
+	SingleJoinRelation(LogicalOperator &op, optional_ptr<LogicalOperator> parent) : op(op), parent(parent) {
+	}
+	SingleJoinRelation(LogicalOperator &op, optional_ptr<LogicalOperator> parent, RelationStats stats)
+	    : op(op), parent(parent), stats(stats) {
+	}
+};
+
+class RelationManager {
+public:
+	explicit RelationManager(ClientContext &context) : context(context) {
+	}
+
+	idx_t NumRelations();
+
+	bool ExtractJoinRelations(LogicalOperator &input_op, vector<reference<LogicalOperator>> &filter_operators,
+	                          optional_ptr<LogicalOperator> parent = nullptr);
+
+	//! for each join filter in the logical plan op, extract the relations that are referred to on
+	//! both sides of the join filter, along with the tables & indexes.
+	vector<unique_ptr<FilterInfo>> ExtractEdges(LogicalOperator &op,
+	                                            vector<reference<LogicalOperator>> &filter_operators,
+	                                            JoinRelationSetManager &set_manager);
+
+	//! Extract the set of relations referred to inside an expression
+	bool ExtractBindings(Expression &expression, unordered_set<idx_t> &bindings);
+	void AddRelation(LogicalOperator &op, optional_ptr<LogicalOperator> parent, const RelationStats &stats);
+
+	void AddAggregateRelation(LogicalOperator &op, optional_ptr<LogicalOperator> parent, const RelationStats &stats);
+	vector<unique_ptr<SingleJoinRelation>> GetRelations();
+
+	const vector<RelationStats> GetRelationStats();
+	//! A mapping of base table index -> index into relations array (relation number)
+	unordered_map<idx_t, idx_t> relation_mapping;
+
+	void PrintRelationStats();
+
+private:
+	ClientContext &context;
+	//! Set of all relations considered in the join optimizer
+	vector<unique_ptr<SingleJoinRelation>> relations;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/optimizer/join_order/relation_statistics_helper.hpp

@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/optimizer/join_order/statistics_extractor.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+#pragma once
+
+#include "duckdb/planner/filter/conjunction_filter.hpp"
+#include "duckdb/planner/logical_operator.hpp"
+
+namespace duckdb {
+
+class CardinalityEstimator;
+
+struct DistinctCount {
+	idx_t distinct_count;
+	bool from_hll;
+};
+
+struct ExpressionBinding {
+	bool found_expression = false;
+	ColumnBinding child_binding;
+	bool expression_is_constant = false;
+};
+
+struct RelationStats {
+	// column_id -> estimated distinct count for column
+	vector<DistinctCount> column_distinct_count;
+	idx_t cardinality;
+	double filter_strength = 1;
+	bool stats_initialized = false;
+
+	// for debug, column names and tables
+	vector<string> column_names;
+	string table_name;
+
+	RelationStats() : cardinality(1), filter_strength(1), stats_initialized(false) {
+	}
+};
+
+class RelationStatisticsHelper {
+public:
+	static constexpr double DEFAULT_SELECTIVITY = 0.2;
+
+public:
+	static idx_t InspectConjunctionAND(idx_t cardinality, idx_t column_index, ConjunctionAndFilter &filter,
+	                                   BaseStatistics &base_stats);
+	//	static idx_t InspectConjunctionOR(idx_t cardinality, idx_t column_index, ConjunctionOrFilter &filter,
+	//	                                  BaseStatistics &base_stats);
+	//! Extract Statistics from a LogicalGet.
+	static RelationStats ExtractGetStats(LogicalGet &get, ClientContext &context);
+	static RelationStats ExtractDelimGetStats(LogicalDelimGet &delim_get, ClientContext &context);
+	//! Create the statistics for a projection using the statistics of the operator that sits underneath the
+	//! projection. Then also create statistics for any extra columns the projection creates.
+	static RelationStats ExtractDummyScanStats(LogicalDummyScan &dummy_scan, ClientContext &context);
+	static RelationStats ExtractExpressionGetStats(LogicalExpressionGet &expression_get, ClientContext &context);
+	//! All relation extractors for blocking relations
+	static RelationStats ExtractProjectionStats(LogicalProjection &proj, RelationStats &child_stats);
+	static RelationStats ExtractAggregationStats(LogicalAggregate &aggr, RelationStats &child_stats);
+	static RelationStats ExtractWindowStats(LogicalWindow &window, RelationStats &child_stats);
+	//! Called after reordering a query plan with potentially 2+ relations.
+	static RelationStats CombineStatsOfReorderableOperator(vector<ColumnBinding> &bindings,
+	                                                       vector<RelationStats> relation_stats);
+	//! Called after reordering a query plan with potentially 2+ relations.
+	static RelationStats CombineStatsOfNonReorderableOperator(LogicalOperator &op, vector<RelationStats> child_stats);
+	static void CopyRelationStats(RelationStats &to, const RelationStats &from);
+
+private:
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp

@@ -39,7 +39,7 @@ class TaskScheduler {
 	constexpr static int64_t TASK_TIMEOUT_USECS = 5000;
 
 public:
-	TaskScheduler(DatabaseInstance &db);
+	explicit TaskScheduler(DatabaseInstance &db);
 	~TaskScheduler();
 
 	DUCKDB_API static TaskScheduler &GetScheduler(ClientContext &context);
@@ -67,6 +67,9 @@ public:
 	//! Send signals to n threads, signalling for them to wake up and attempt to execute a task
 	void Signal(idx_t n);
 
+	//! Yield to other threads
+	void YieldThread();
+
 	//! Set the allocator flush threshold
 	void SetAllocatorFlushTreshold(idx_t threshold);
 

package/src/duckdb/src/include/duckdb/planner/logical_operator.hpp

@@ -49,8 +49,6 @@ public:
 	idx_t estimated_cardinality;
 	bool has_estimated_cardinality;
 
-	unique_ptr<EstimatedProperties> estimated_props;
-
 public:
 	virtual vector<ColumnBinding> GetColumnBindings();
 	static vector<ColumnBinding> GenerateColumnBindings(idx_t table_idx, idx_t column_count);

package/src/duckdb/src/include/duckdb/storage/block.hpp

@@ -24,10 +24,33 @@ public:
 };
 
 struct BlockPointer {
-	BlockPointer(block_id_t block_id_p, uint32_t offset_p) : block_id(block_id_p), offset(offset_p) {};
-	BlockPointer() {};
-	block_id_t block_id {0};
-	uint32_t offset {0};
+	BlockPointer(block_id_t block_id_p, uint32_t offset_p) : block_id(block_id_p), offset(offset_p) {
+	}
+	BlockPointer() : block_id(INVALID_BLOCK), offset(0) {
+	}
+
+	block_id_t block_id;
+	uint32_t offset;
+
+	bool IsValid() {
+		return block_id != INVALID_BLOCK;
+	}
+};
+
+struct MetaBlockPointer {
+	MetaBlockPointer(idx_t block_pointer, uint32_t offset_p) : block_pointer(block_pointer), offset(offset_p) {
+	}
+	MetaBlockPointer() : block_pointer(DConstants::INVALID_INDEX), offset(0) {
+	}
+
+	idx_t block_pointer;
+	uint32_t offset;
+
+	bool IsValid() {
+		return block_pointer != DConstants::INVALID_INDEX;
+	}
+	block_id_t GetBlockId();
+	uint32_t GetBlockIndex();
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/block_manager.hpp

@@ -19,13 +19,13 @@ class BlockHandle;
 class BufferManager;
 class ClientContext;
 class DatabaseInstance;
+class MetadataManager;
 
 //! BlockManager is an abstract representation to manage blocks on DuckDB. When writing or reading blocks, the
 //! BlockManager creates and accesses blocks. The concrete types implements how blocks are stored.
 class BlockManager {
 public:
-	explicit BlockManager(BufferManager &buffer_manager) : buffer_manager(buffer_manager) {
-	}
+	explicit BlockManager(BufferManager &buffer_manager);
 	virtual ~BlockManager() = default;
 
 	//! The buffer manager
@@ -38,7 +38,7 @@ public:
 	//! Return the next free block id
 	virtual block_id_t GetFreeBlockId() = 0;
 	//! Returns whether or not a specified block is the root block
-	virtual bool IsRootBlock(block_id_t root) = 0;
+	virtual bool IsRootBlock(MetaBlockPointer root) = 0;
 	//! Mark a block as "free"; free blocks are immediately added to the free list and can be immediately overwritten
 	virtual void MarkBlockAsFree(block_id_t block_id) = 0;
 	//! Mark a block as "modified"; modified blocks are added to the free list after a checkpoint (i.e. their data is
@@ -48,7 +48,7 @@ public:
 	//! called.
 	virtual void IncreaseBlockReferenceCount(block_id_t block_id) = 0;
 	//! Get the first meta block id
-	virtual block_id_t GetMetaBlock() = 0;
+	virtual idx_t GetMetaBlock() = 0;
 	//! Read the content of the block from disk
 	virtual void Read(Block &block) = 0;
 	//! Writes the block to disk
@@ -69,20 +69,20 @@ public:
 	virtual void Truncate();
 
 	//! Register a block with the given block id in the base file
-	shared_ptr<BlockHandle> RegisterBlock(block_id_t block_id, bool is_meta_block = false);
-	//! Clear cached handles for meta blocks
-	void ClearMetaBlockHandles();
+	shared_ptr<BlockHandle> RegisterBlock(block_id_t block_id);
 	//! Convert an existing in-memory buffer into a persistent disk-backed block
 	shared_ptr<BlockHandle> ConvertToPersistent(block_id_t block_id, shared_ptr<BlockHandle> old_block);
 
 	void UnregisterBlock(block_id_t block_id, bool can_destroy);
 
+	MetadataManager &GetMetadataManager();
+
 private:
 	//! The lock for the set of blocks
 	mutex blocks_lock;
 	//! A mapping of block id -> BlockHandle
 	unordered_map<block_id_t, weak_ptr<BlockHandle>> blocks;
-	//! A map to cache the BlockHandles of meta blocks
-	unordered_map<block_id_t, shared_ptr<BlockHandle>> meta_blocks;
+	//! The metadata manager
+	unique_ptr<MetadataManager> metadata_manager;
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/checkpoint/row_group_writer.hpp

@@ -32,7 +32,7 @@ public:
 
 	virtual void WriteColumnDataPointers(ColumnCheckpointState &column_checkpoint_state) = 0;
 
-	virtual MetaBlockWriter &GetPayloadWriter() = 0;
+	virtual MetadataWriter &GetPayloadWriter() = 0;
 
 	void RegisterPartialBlock(PartialBlockAllocation &&allocation);
 	PartialBlockAllocation GetBlockAllocation(uint32_t segment_size);
@@ -50,18 +50,18 @@ protected:
 class SingleFileRowGroupWriter : public RowGroupWriter {
 public:
 	SingleFileRowGroupWriter(TableCatalogEntry &table, PartialBlockManager &partial_block_manager,
-	                         MetaBlockWriter &table_data_writer)
+	                         MetadataWriter &table_data_writer)
 	    : RowGroupWriter(table, partial_block_manager), table_data_writer(table_data_writer) {
 	}
 
-	//! MetaBlockWriter is a cursor on a given BlockManager. This returns the
+	//! MetadataWriter is a cursor on a given BlockManager. This returns the
 	//! cursor against which we should write payload data for the specified RowGroup.
-	MetaBlockWriter &table_data_writer;
+	MetadataWriter &table_data_writer;
 
 public:
 	virtual void WriteColumnDataPointers(ColumnCheckpointState &column_checkpoint_state) override;
 
-	virtual MetaBlockWriter &GetPayloadWriter() override;
+	virtual MetadataWriter &GetPayloadWriter() override;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_reader.hpp

@@ -16,12 +16,12 @@ struct BoundCreateTableInfo;
 //! The table data reader is responsible for reading the data of a table from the block manager
 class TableDataReader {
 public:
-	TableDataReader(MetaBlockReader &reader, BoundCreateTableInfo &info);
+	TableDataReader(MetadataReader &reader, BoundCreateTableInfo &info);
 
 	void ReadTableData();
 
 private:
-	MetaBlockReader &reader;
+	MetadataReader &reader;
 	BoundCreateTableInfo &info;
 };
 

package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp

@@ -45,7 +45,7 @@ protected:
 class SingleFileTableDataWriter : public TableDataWriter {
 public:
 	SingleFileTableDataWriter(SingleFileCheckpointWriter &checkpoint_manager, TableCatalogEntry &table,
-	                          MetaBlockWriter &table_data_writer, MetaBlockWriter &meta_data_writer);
+	                          MetadataWriter &table_data_writer, MetadataWriter &meta_data_writer);
 
 public:
 	virtual void FinalizeTable(TableStatistics &&global_stats, DataTableInfo *info) override;
@@ -54,9 +54,9 @@ public:
 private:
 	SingleFileCheckpointWriter &checkpoint_manager;
 	// Writes the actual table data
-	MetaBlockWriter &table_data_writer;
+	MetadataWriter &table_data_writer;
 	// Writes the metadata of the table
-	MetaBlockWriter &meta_data_writer;
+	MetadataWriter &meta_data_writer;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp

@@ -16,7 +16,7 @@ namespace duckdb {
 class DatabaseInstance;
 class ClientContext;
 class ColumnSegment;
-class MetaBlockReader;
+class MetadataReader;
 class SchemaCatalogEntry;
 class SequenceCatalogEntry;
 class TableCatalogEntry;
@@ -33,7 +33,8 @@ public:
 	//! The database
 	AttachedDatabase &db;
 
-	virtual MetaBlockWriter &GetMetaBlockWriter() = 0;
+	virtual MetadataManager &GetMetadataManager() = 0;
+	virtual MetadataWriter &GetMetadataWriter() = 0;
 	virtual unique_ptr<TableDataWriter> GetTableDataWriter(TableCatalogEntry &table) = 0;
 
 protected:
@@ -58,17 +59,17 @@ protected:
 	Catalog &catalog;
 
 protected:
-	virtual void LoadCheckpoint(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadSchema(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadTable(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadView(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadSequence(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadMacro(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadTableMacro(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadIndex(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadType(ClientContext &context, MetaBlockReader &reader);
-
-	virtual void ReadTableData(ClientContext &context, MetaBlockReader &reader, BoundCreateTableInfo &bound_info);
+	virtual void LoadCheckpoint(ClientContext &context, MetadataReader &reader);
+	virtual void ReadSchema(ClientContext &context, MetadataReader &reader);
+	virtual void ReadTable(ClientContext &context, MetadataReader &reader);
+	virtual void ReadView(ClientContext &context, MetadataReader &reader);
+	virtual void ReadSequence(ClientContext &context, MetadataReader &reader);
+	virtual void ReadMacro(ClientContext &context, MetadataReader &reader);
+	virtual void ReadTableMacro(ClientContext &context, MetadataReader &reader);
+	virtual void ReadIndex(ClientContext &context, MetadataReader &reader);
+	virtual void ReadType(ClientContext &context, MetadataReader &reader);
+
+	virtual void ReadTableData(ClientContext &context, MetadataReader &reader, BoundCreateTableInfo &bound_info);
 };
 
 class SingleFileCheckpointReader final : public CheckpointReader {
@@ -78,6 +79,7 @@ public:
 	}
 
 	void LoadFromStorage();
+	MetadataManager &GetMetadataManager();
 
 	//! The database
 	SingleFileStorageManager &storage;
@@ -98,16 +100,17 @@ public:
 	//! connection is available because right now the checkpointing cannot be done online. (TODO)
 	void CreateCheckpoint();
 
-	virtual MetaBlockWriter &GetMetaBlockWriter() override;
+	virtual MetadataWriter &GetMetadataWriter() override;
+	virtual MetadataManager &GetMetadataManager() override;
 	virtual unique_ptr<TableDataWriter> GetTableDataWriter(TableCatalogEntry &table) override;
 
 	BlockManager &GetBlockManager();
 
 private:
 	//! The metadata writer is responsible for writing schema information
-	unique_ptr<MetaBlockWriter> metadata_writer;
+	unique_ptr<MetadataWriter> metadata_writer;
 	//! The table data writer is responsible for writing the DataPointers used by the table chunks
-	unique_ptr<MetaBlockWriter> table_metadata_writer;
+	unique_ptr<MetadataWriter> table_metadata_writer;
 	//! Because this is single-file storage, we can share partial blocks across
 	//! an entire checkpoint.
 	PartialBlockManager partial_block_manager;

package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp

@@ -33,7 +33,7 @@ struct RowGroupPointer {
 	uint64_t row_start;
 	uint64_t tuple_count;
 	//! The data pointers of the column segments stored in the row group
-	vector<BlockPointer> data_pointers;
+	vector<MetaBlockPointer> data_pointers;
 	//! The versions information of the row group (if any)
 	shared_ptr<VersionNode> versions;
 };

package/src/duckdb/src/include/duckdb/storage/in_memory_block_manager.hpp

@@ -29,7 +29,7 @@ public:
 	block_id_t GetFreeBlockId() override {
 		throw InternalException("Cannot perform IO in in-memory database - GetFreeBlockId!");
 	}
-	bool IsRootBlock(block_id_t root) override {
+	bool IsRootBlock(MetaBlockPointer root) override {
 		throw InternalException("Cannot perform IO in in-memory database - IsRootBlock!");
 	}
 	void MarkBlockAsFree(block_id_t block_id) override {
@@ -41,7 +41,7 @@ public:
 	void IncreaseBlockReferenceCount(block_id_t block_id) override {
 		throw InternalException("Cannot perform IO in in-memory database - IncreaseBlockReferenceCount!");
 	}
-	block_id_t GetMetaBlock() override {
+	idx_t GetMetaBlock() override {
 		throw InternalException("Cannot perform IO in in-memory database - GetMetaBlock!");
 	}
 	void Read(Block &block) override {