duckdb 0.8.2-dev37.0 → 0.8.2-dev3989.0

package/src/duckdb/src/include/duckdb/planner/operator/logical_simple.hpp

@@ -28,6 +28,9 @@ public:
 public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+
+	void FormatSerialize(FormatSerializer &serializer) const override;
+	static unique_ptr<LogicalOperator> FormatDeserialize(FormatDeserializer &deserializer);
 	idx_t EstimateCardinality(ClientContext &context) override;
 
 protected:

package/src/duckdb/src/include/duckdb/planner/operator/logical_top_n.hpp

@@ -35,6 +35,10 @@ public:
 	}
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+
+	void FormatSerialize(FormatSerializer &serializer) const override;
+	static unique_ptr<LogicalOperator> FormatDeserialize(FormatDeserializer &deserializer);
+
 	idx_t EstimateCardinality(ClientContext &context) override;
 
 protected:

package/src/duckdb/src/include/duckdb/planner/operator/logical_unnest.hpp

@@ -28,6 +28,8 @@ public:
 	vector<ColumnBinding> GetColumnBindings() override;
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	void FormatSerialize(FormatSerializer &serializer) const override;
+	static unique_ptr<LogicalOperator> FormatDeserialize(FormatDeserializer &deserializer);
 	vector<idx_t> GetTableIndex() const override;
 	string GetName() const override;
 

package/src/duckdb/src/include/duckdb/planner/operator/logical_update.hpp

@@ -33,11 +33,17 @@ public:
 public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	void FormatSerialize(FormatSerializer &serializer) const override;
+	static unique_ptr<LogicalOperator> FormatDeserialize(FormatDeserializer &deserializer);
+
 	idx_t EstimateCardinality(ClientContext &context) override;
 	string GetName() const override;
 
 protected:
 	vector<ColumnBinding> GetColumnBindings() override;
 	void ResolveTypes() override;
+
+private:
+	LogicalUpdate(ClientContext &context, const string &catalog, const string &schema, const string &table);
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/operator/logical_window.hpp

@@ -29,6 +29,9 @@ public:
 	vector<ColumnBinding> GetColumnBindings() override;
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+
+	void FormatSerialize(FormatSerializer &serializer) const override;
+	static unique_ptr<LogicalOperator> FormatDeserialize(FormatDeserializer &deserializer);
 	vector<idx_t> GetTableIndex() const override;
 	string GetName() const override;
 

package/src/duckdb/src/include/duckdb/planner/operator_extension.hpp

@@ -36,6 +36,7 @@ public:
 	virtual std::string GetName() = 0;
 	virtual unique_ptr<LogicalExtensionOperator> Deserialize(LogicalDeserializationState &state,
 	                                                         FieldReader &reader) = 0;
+	virtual unique_ptr<LogicalExtensionOperator> FormatDeserialize(FormatDeserializer &deserializer) = 0;
 
 	virtual ~OperatorExtension() {
 	}

package/src/duckdb/src/include/duckdb/planner/planner.hpp

@@ -25,21 +25,22 @@ class Planner {
 public:
 	explicit Planner(ClientContext &context);
 
+public:
 	unique_ptr<LogicalOperator> plan;
 	vector<string> names;
 	vector<LogicalType> types;
-	bound_parameter_map_t value_map;
-	vector<BoundParameterData> parameter_data;
+	case_insensitive_map_t<BoundParameterData> parameter_data;
 
 	shared_ptr<Binder> binder;
 	ClientContext &context;
 
 	StatementProperties properties;
+	bound_parameter_map_t value_map;
 
 public:
 	void CreatePlan(unique_ptr<SQLStatement> statement);
 	static void VerifyPlan(ClientContext &context, unique_ptr<LogicalOperator> &op,
-	                       bound_parameter_map_t *map = nullptr);
+	                       optional_ptr<bound_parameter_map_t> map = nullptr);
 
 private:
 	void CreatePlan(SQLStatement &statement);

package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp

@@ -0,0 +1,44 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/planner/query_node/bound_cte_node.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/planner/binder.hpp"
+#include "duckdb/planner/bound_query_node.hpp"
+
+namespace duckdb {
+
+class BoundCTENode : public BoundQueryNode {
+public:
+	static constexpr const QueryNodeType TYPE = QueryNodeType::CTE_NODE;
+
+public:
+	BoundCTENode() : BoundQueryNode(QueryNodeType::CTE_NODE) {
+	}
+
+	//! Keep track of the CTE name this node represents
+	string ctename;
+
+	//! The cte node
+	unique_ptr<BoundQueryNode> query;
+	//! The child node
+	unique_ptr<BoundQueryNode> child;
+	//! Index used by the set operation
+	idx_t setop_index;
+	//! The binder used by the query side of the CTE
+	shared_ptr<Binder> query_binder;
+	//! The binder used by the child side of the CTE
+	shared_ptr<Binder> child_binder;
+
+public:
+	idx_t GetRootIndex() override {
+		return child->GetRootIndex();
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp

@@ -1,3 +1,4 @@
 #include "duckdb/planner/query_node/bound_recursive_cte_node.hpp"
+#include "duckdb/planner/query_node/bound_cte_node.hpp"
 #include "duckdb/planner/query_node/bound_select_node.hpp"
 #include "duckdb/planner/query_node/bound_set_operation_node.hpp"

package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp

@@ -23,7 +23,7 @@ struct FlattenDependentJoins {
 
 	//! Detects which Logical Operators have correlated expressions that they are dependent upon, filling the
 	//! has_correlated_expressions map.
-	bool DetectCorrelatedExpressions(LogicalOperator *op, bool lateral = false);
+	bool DetectCorrelatedExpressions(LogicalOperator *op, bool lateral = false, idx_t lateral_depth = 0);
 
 	//! Push the dependent join down a LogicalOperator
 	unique_ptr<LogicalOperator> PushDownDependentJoin(unique_ptr<LogicalOperator> plan);
@@ -43,7 +43,7 @@ struct FlattenDependentJoins {
 
 private:
 	unique_ptr<LogicalOperator> PushDownDependentJoinInternal(unique_ptr<LogicalOperator> plan,
-	                                                          bool &parent_propagate_null_values);
+	                                                          bool &parent_propagate_null_values, idx_t lateral_depth);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp

@@ -16,7 +16,8 @@ namespace duckdb {
 //! Helper class to recursively detect correlated expressions inside a single LogicalOperator
 class HasCorrelatedExpressions : public LogicalOperatorVisitor {
 public:
-	explicit HasCorrelatedExpressions(const vector<CorrelatedColumnInfo> &correlated, bool lateral = false);
+	explicit HasCorrelatedExpressions(const vector<CorrelatedColumnInfo> &correlated, bool lateral = false,
+	                                  idx_t lateral_depth = 0);
 
 	void VisitOperator(LogicalOperator &op) override;
 
@@ -28,6 +29,8 @@ protected:
 	unique_ptr<Expression> VisitReplace(BoundSubqueryExpression &expr, unique_ptr<Expression> *expr_ptr) override;
 
 	const vector<CorrelatedColumnInfo> &correlated_columns;
+	// Tracks number of nested laterals
+	idx_t lateral_depth;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp

@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/planner/subquery/recursive_dependent_join_planner.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/planner/logical_operator_visitor.hpp"
+
+namespace duckdb {
+
+class Binder;
+
+/*
+ * Recursively plan subqueries and flatten dependent joins from outermost to innermost (like peeling an onion).
+ */
+class RecursiveDependentJoinPlanner : public LogicalOperatorVisitor {
+public:
+	explicit RecursiveDependentJoinPlanner(Binder &binder) : binder(binder) {
+	}
+	void VisitOperator(LogicalOperator &op) override;
+	unique_ptr<Expression> VisitReplace(BoundSubqueryExpression &expr, unique_ptr<Expression> *expr_ptr) override;
+
+private:
+	unique_ptr<LogicalOperator> root;
+	Binder &binder;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp

@@ -17,7 +17,8 @@ namespace duckdb {
 //! Helper class to rewrite correlated expressions within a single LogicalOperator
 class RewriteCorrelatedExpressions : public LogicalOperatorVisitor {
 public:
-	RewriteCorrelatedExpressions(ColumnBinding base_binding, column_binding_map_t<idx_t> &correlated_map);
+	RewriteCorrelatedExpressions(ColumnBinding base_binding, column_binding_map_t<idx_t> &correlated_map,
+	                             idx_t lateral_depth, bool recursive_rewrite = false);
 
 	void VisitOperator(LogicalOperator &op) override;
 
@@ -31,7 +32,7 @@ private:
 	public:
 		RewriteCorrelatedRecursive(BoundSubqueryExpression &parent, ColumnBinding base_binding,
 		                           column_binding_map_t<idx_t> &correlated_map);
-
+		void RewriteJoinRefRecursive(BoundTableRef &ref);
 		void RewriteCorrelatedSubquery(BoundSubqueryExpression &expr);
 		void RewriteCorrelatedExpressions(Expression &child);
 
@@ -43,6 +44,11 @@ private:
 private:
 	ColumnBinding base_binding;
 	column_binding_map_t<idx_t> &correlated_map;
+	// To keep track of the number of dependent joins encountered
+	idx_t lateral_depth;
+	// This flag is used to determine if the rewrite should recursively update the bindings for all
+	// bound columns ref in the plan, and update the depths to match the new source
+	bool recursive_rewrite;
 };
 
 //! Helper class that rewrites COUNT aggregates into a CASE expression turning NULL into 0 after a LEFT OUTER JOIN

package/src/duckdb/src/include/duckdb/planner/table_filter.hpp

@@ -29,7 +29,7 @@ enum class TableFilterType : uint8_t {
 //! TableFilter represents a filter pushed down into the table scan.
 class TableFilter {
 public:
-	TableFilter(TableFilterType filter_type_p) : filter_type(filter_type_p) {
+	explicit TableFilter(TableFilterType filter_type_p) : filter_type(filter_type_p) {
 	}
 	virtual ~TableFilter() {
 	}
@@ -48,6 +48,9 @@ public:
 	virtual void Serialize(FieldWriter &writer) const = 0;
 	static unique_ptr<TableFilter> Deserialize(Deserializer &source);
 
+	virtual void FormatSerialize(FormatSerializer &serializer) const;
+	static unique_ptr<TableFilter> FormatDeserialize(FormatDeserializer &deserializer);
+
 public:
 	template <class TARGET>
 	TARGET &Cast() {
@@ -100,6 +103,9 @@ public:
 
 	void Serialize(Serializer &serializer) const;
 	static unique_ptr<TableFilterSet> Deserialize(Deserializer &source);
+
+	void FormatSerialize(FormatSerializer &serializer) const;
+	static TableFilterSet FormatDeserialize(FormatDeserializer &deserializer);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp

@@ -17,8 +17,9 @@ public:
 	static constexpr const TableReferenceType TYPE = TableReferenceType::CTE;
 
 public:
-	BoundCTERef(idx_t bind_index, idx_t cte_index)
-	    : BoundTableRef(TableReferenceType::CTE), bind_index(bind_index), cte_index(cte_index) {
+	BoundCTERef(idx_t bind_index, idx_t cte_index, CTEMaterialize materialized_cte)
+	    : BoundTableRef(TableReferenceType::CTE), bind_index(bind_index), cte_index(cte_index),
+	      materialized_cte(materialized_cte) {
 	}
 
 	//! The set of columns bound to this base table reference
@@ -29,5 +30,7 @@ public:
 	idx_t bind_index;
 	//! The index of the cte
 	idx_t cte_index;
+	//! Is this a reference to a materialized CTE?
+	CTEMaterialize materialized_cte;
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/tableref/bound_pivotref.hpp

@@ -25,6 +25,9 @@ struct BoundPivotInfo {
 	vector<string> pivot_values;
 	//! The set of aggregate functions that is being executed
 	vector<unique_ptr<Expression>> aggregates;
+
+	void FormatSerialize(FormatSerializer &serializer) const;
+	static BoundPivotInfo FormatDeserialize(FormatDeserializer &deserializer);
 };
 
 class BoundPivotRef : public BoundTableRef {

package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp

@@ -45,7 +45,8 @@ public:
 	DUCKDB_API ArenaChunk *GetHead();
 	DUCKDB_API ArenaChunk *GetTail();
 
-	DUCKDB_API bool IsEmpty();
+	DUCKDB_API bool IsEmpty() const;
+	DUCKDB_API idx_t SizeInBytes() const;
 
 	//! Returns an "Allocator" wrapper for this arena allocator
 	Allocator &GetAllocator() {

package/src/duckdb/src/include/duckdb/storage/block.hpp

@@ -14,6 +14,9 @@
 
 namespace duckdb {
 
+class FormatSerializer;
+class FormatDeserializer;
+
 class Block : public FileBuffer {
 public:
 	Block(Allocator &allocator, block_id_t id);
@@ -24,10 +27,36 @@ public:
 };
 
 struct BlockPointer {
-	BlockPointer(block_id_t block_id_p, uint32_t offset_p) : block_id(block_id_p), offset(offset_p) {};
-	BlockPointer() {};
-	block_id_t block_id {0};
-	uint32_t offset {0};
+	BlockPointer(block_id_t block_id_p, uint32_t offset_p) : block_id(block_id_p), offset(offset_p) {
+	}
+	BlockPointer() : block_id(INVALID_BLOCK), offset(0) {
+	}
+
+	block_id_t block_id;
+	uint32_t offset;
+
+	bool IsValid() {
+		return block_id != INVALID_BLOCK;
+	}
+};
+
+struct MetaBlockPointer {
+	MetaBlockPointer(idx_t block_pointer, uint32_t offset_p) : block_pointer(block_pointer), offset(offset_p) {
+	}
+	MetaBlockPointer() : block_pointer(DConstants::INVALID_INDEX), offset(0) {
+	}
+
+	idx_t block_pointer;
+	uint32_t offset;
+
+	bool IsValid() {
+		return block_pointer != DConstants::INVALID_INDEX;
+	}
+	block_id_t GetBlockId();
+	uint32_t GetBlockIndex();
+
+	void FormatSerialize(FormatSerializer &serializer) const;
+	static MetaBlockPointer FormatDeserialize(FormatDeserializer &source);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/block_manager.hpp

@@ -19,13 +19,13 @@ class BlockHandle;
 class BufferManager;
 class ClientContext;
 class DatabaseInstance;
+class MetadataManager;
 
 //! BlockManager is an abstract representation to manage blocks on DuckDB. When writing or reading blocks, the
 //! BlockManager creates and accesses blocks. The concrete types implements how blocks are stored.
 class BlockManager {
 public:
-	explicit BlockManager(BufferManager &buffer_manager) : buffer_manager(buffer_manager) {
-	}
+	explicit BlockManager(BufferManager &buffer_manager);
 	virtual ~BlockManager() = default;
 
 	//! The buffer manager
@@ -38,7 +38,7 @@ public:
 	//! Return the next free block id
 	virtual block_id_t GetFreeBlockId() = 0;
 	//! Returns whether or not a specified block is the root block
-	virtual bool IsRootBlock(block_id_t root) = 0;
+	virtual bool IsRootBlock(MetaBlockPointer root) = 0;
 	//! Mark a block as "free"; free blocks are immediately added to the free list and can be immediately overwritten
 	virtual void MarkBlockAsFree(block_id_t block_id) = 0;
 	//! Mark a block as "modified"; modified blocks are added to the free list after a checkpoint (i.e. their data is
@@ -48,7 +48,7 @@ public:
 	//! called.
 	virtual void IncreaseBlockReferenceCount(block_id_t block_id) = 0;
 	//! Get the first meta block id
-	virtual block_id_t GetMetaBlock() = 0;
+	virtual idx_t GetMetaBlock() = 0;
 	//! Read the content of the block from disk
 	virtual void Read(Block &block) = 0;
 	//! Writes the block to disk
@@ -65,24 +65,24 @@ public:
 	//! Returns the number of free blocks
 	virtual idx_t FreeBlocks() = 0;
 
+	//! Truncate the underlying database file after a checkpoint
+	virtual void Truncate();
+
 	//! Register a block with the given block id in the base file
-	shared_ptr<BlockHandle> RegisterBlock(block_id_t block_id, bool is_meta_block = false);
-	//! Clear cached handles for meta blocks
-	void ClearMetaBlockHandles();
+	shared_ptr<BlockHandle> RegisterBlock(block_id_t block_id);
 	//! Convert an existing in-memory buffer into a persistent disk-backed block
 	shared_ptr<BlockHandle> ConvertToPersistent(block_id_t block_id, shared_ptr<BlockHandle> old_block);
 
 	void UnregisterBlock(block_id_t block_id, bool can_destroy);
 
-	static BlockManager &GetBlockManager(ClientContext &context);
-	static BlockManager &GetBlockManager(DatabaseInstance &db);
+	MetadataManager &GetMetadataManager();
 
 private:
 	//! The lock for the set of blocks
 	mutex blocks_lock;
 	//! A mapping of block id -> BlockHandle
 	unordered_map<block_id_t, weak_ptr<BlockHandle>> blocks;
-	//! A map to cache the BlockHandles of meta blocks
-	unordered_map<block_id_t, shared_ptr<BlockHandle>> meta_blocks;
+	//! The metadata manager
+	unique_ptr<MetadataManager> metadata_manager;
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/buffer/block_handle.hpp

@@ -97,6 +97,9 @@ public:
 	inline const idx_t &GetMemoryUsage() const {
 		return memory_usage;
 	}
+	bool IsUnloaded() {
+		return state == BlockState::BLOCK_UNLOADED;
+	}
 
 private:
 	static BufferHandle Load(shared_ptr<BlockHandle> &handle, unique_ptr<FileBuffer> buffer = nullptr);

package/src/duckdb/src/include/duckdb/storage/checkpoint/row_group_writer.hpp

@@ -32,7 +32,7 @@ public:
 
 	virtual void WriteColumnDataPointers(ColumnCheckpointState &column_checkpoint_state) = 0;
 
-	virtual MetaBlockWriter &GetPayloadWriter() = 0;
+	virtual MetadataWriter &GetPayloadWriter() = 0;
 
 	void RegisterPartialBlock(PartialBlockAllocation &&allocation);
 	PartialBlockAllocation GetBlockAllocation(uint32_t segment_size);
@@ -50,18 +50,18 @@ protected:
 class SingleFileRowGroupWriter : public RowGroupWriter {
 public:
 	SingleFileRowGroupWriter(TableCatalogEntry &table, PartialBlockManager &partial_block_manager,
-	                         MetaBlockWriter &table_data_writer)
+	                         MetadataWriter &table_data_writer)
 	    : RowGroupWriter(table, partial_block_manager), table_data_writer(table_data_writer) {
 	}
 
-	//! MetaBlockWriter is a cursor on a given BlockManager. This returns the
+	//! MetadataWriter is a cursor on a given BlockManager. This returns the
 	//! cursor against which we should write payload data for the specified RowGroup.
-	MetaBlockWriter &table_data_writer;
+	MetadataWriter &table_data_writer;
 
 public:
 	virtual void WriteColumnDataPointers(ColumnCheckpointState &column_checkpoint_state) override;
 
-	virtual MetaBlockWriter &GetPayloadWriter() override;
+	virtual MetadataWriter &GetPayloadWriter() override;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_reader.hpp

@@ -16,12 +16,12 @@ struct BoundCreateTableInfo;
 //! The table data reader is responsible for reading the data of a table from the block manager
 class TableDataReader {
 public:
-	TableDataReader(MetaBlockReader &reader, BoundCreateTableInfo &info);
+	TableDataReader(MetadataReader &reader, BoundCreateTableInfo &info);
 
 	void ReadTableData();
 
 private:
-	MetaBlockReader &reader;
+	MetadataReader &reader;
 	BoundCreateTableInfo &info;
 };
 

package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp

@@ -45,7 +45,7 @@ protected:
 class SingleFileTableDataWriter : public TableDataWriter {
 public:
 	SingleFileTableDataWriter(SingleFileCheckpointWriter &checkpoint_manager, TableCatalogEntry &table,
-	                          MetaBlockWriter &table_data_writer, MetaBlockWriter &meta_data_writer);
+	                          MetadataWriter &table_data_writer, MetadataWriter &meta_data_writer);
 
 public:
 	virtual void FinalizeTable(TableStatistics &&global_stats, DataTableInfo *info) override;
@@ -54,9 +54,9 @@ public:
 private:
 	SingleFileCheckpointWriter &checkpoint_manager;
 	// Writes the actual table data
-	MetaBlockWriter &table_data_writer;
+	MetadataWriter &table_data_writer;
 	// Writes the metadata of the table
-	MetaBlockWriter &meta_data_writer;
+	MetadataWriter &meta_data_writer;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp

@@ -16,7 +16,7 @@ namespace duckdb {
 class DatabaseInstance;
 class ClientContext;
 class ColumnSegment;
-class MetaBlockReader;
+class MetadataReader;
 class SchemaCatalogEntry;
 class SequenceCatalogEntry;
 class TableCatalogEntry;
@@ -33,7 +33,8 @@ public:
 	//! The database
 	AttachedDatabase &db;
 
-	virtual MetaBlockWriter &GetMetaBlockWriter() = 0;
+	virtual MetadataManager &GetMetadataManager() = 0;
+	virtual MetadataWriter &GetMetadataWriter() = 0;
 	virtual unique_ptr<TableDataWriter> GetTableDataWriter(TableCatalogEntry &table) = 0;
 
 protected:
@@ -58,17 +59,17 @@ protected:
 	Catalog &catalog;
 
 protected:
-	virtual void LoadCheckpoint(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadSchema(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadTable(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadView(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadSequence(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadMacro(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadTableMacro(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadIndex(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadType(ClientContext &context, MetaBlockReader &reader);
-
-	virtual void ReadTableData(ClientContext &context, MetaBlockReader &reader, BoundCreateTableInfo &bound_info);
+	virtual void LoadCheckpoint(ClientContext &context, MetadataReader &reader);
+	virtual void ReadSchema(ClientContext &context, MetadataReader &reader);
+	virtual void ReadTable(ClientContext &context, MetadataReader &reader);
+	virtual void ReadView(ClientContext &context, MetadataReader &reader);
+	virtual void ReadSequence(ClientContext &context, MetadataReader &reader);
+	virtual void ReadMacro(ClientContext &context, MetadataReader &reader);
+	virtual void ReadTableMacro(ClientContext &context, MetadataReader &reader);
+	virtual void ReadIndex(ClientContext &context, MetadataReader &reader);
+	virtual void ReadType(ClientContext &context, MetadataReader &reader);
+
+	virtual void ReadTableData(ClientContext &context, MetadataReader &reader, BoundCreateTableInfo &bound_info);
 };
 
 class SingleFileCheckpointReader final : public CheckpointReader {
@@ -78,6 +79,7 @@ public:
 	}
 
 	void LoadFromStorage();
+	MetadataManager &GetMetadataManager();
 
 	//! The database
 	SingleFileStorageManager &storage;
@@ -98,16 +100,17 @@ public:
 	//! connection is available because right now the checkpointing cannot be done online. (TODO)
 	void CreateCheckpoint();
 
-	virtual MetaBlockWriter &GetMetaBlockWriter() override;
+	virtual MetadataWriter &GetMetadataWriter() override;
+	virtual MetadataManager &GetMetadataManager() override;
 	virtual unique_ptr<TableDataWriter> GetTableDataWriter(TableCatalogEntry &table) override;
 
 	BlockManager &GetBlockManager();
 
 private:
 	//! The metadata writer is responsible for writing schema information
-	unique_ptr<MetaBlockWriter> metadata_writer;
+	unique_ptr<MetadataWriter> metadata_writer;
 	//! The table data writer is responsible for writing the DataPointers used by the table chunks
-	unique_ptr<MetaBlockWriter> table_metadata_writer;
+	unique_ptr<MetadataWriter> table_metadata_writer;
 	//! Because this is single-file storage, we can share partial blocks across
 	//! an entire checkpoint.
 	PartialBlockManager partial_block_manager;

package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp

@@ -33,7 +33,7 @@ struct RowGroupPointer {
 	uint64_t row_start;
 	uint64_t tuple_count;
 	//! The data pointers of the column segments stored in the row group
-	vector<BlockPointer> data_pointers;
+	vector<MetaBlockPointer> data_pointers;
 	//! The versions information of the row group (if any)
 	shared_ptr<VersionNode> versions;
 };

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -51,7 +51,7 @@ public:
 	          const string &table, vector<ColumnDefinition> column_definitions_p,
 	          unique_ptr<PersistentTableData> data = nullptr);
 	//! Constructs a DataTable as a delta on an existing data table with a newly added column
-	DataTable(ClientContext &context, DataTable &parent, ColumnDefinition &new_column, Expression *default_value);
+	DataTable(ClientContext &context, DataTable &parent, ColumnDefinition &new_column, Expression &default_value);
 	//! Constructs a DataTable as a delta on an existing data table but with one column removed
 	DataTable(ClientContext &context, DataTable &parent, idx_t removed_column);
 	//! Constructs a DataTable as a delta on an existing data table but with one column changed type
@@ -125,7 +125,7 @@ public:
 	void UpdateColumn(TableCatalogEntry &table, ClientContext &context, Vector &row_ids,
 	                  const vector<column_t> &column_path, DataChunk &updates);
 
-	//! Add an index to the DataTable. NOTE: for CREATE (UNIQUE) INDEX statements, we use the PhysicalCreateIndex
+	//! Add an index to the DataTable. NOTE: for CREATE (UNIQUE) INDEX statements, we use the PhysicalCreateARTIndex
 	//! operator. This function is only used during the WAL replay, and is a much less performant index creation
 	//! approach.
 	void WALAddIndex(ClientContext &context, unique_ptr<Index> index,

package/src/duckdb/src/include/duckdb/storage/in_memory_block_manager.hpp

@@ -29,7 +29,7 @@ public:
 	block_id_t GetFreeBlockId() override {
 		throw InternalException("Cannot perform IO in in-memory database - GetFreeBlockId!");
 	}
-	bool IsRootBlock(block_id_t root) override {
+	bool IsRootBlock(MetaBlockPointer root) override {
 		throw InternalException("Cannot perform IO in in-memory database - IsRootBlock!");
 	}
 	void MarkBlockAsFree(block_id_t block_id) override {
@@ -41,7 +41,7 @@ public:
 	void IncreaseBlockReferenceCount(block_id_t block_id) override {
 		throw InternalException("Cannot perform IO in in-memory database - IncreaseBlockReferenceCount!");
 	}
-	block_id_t GetMetaBlock() override {
+	idx_t GetMetaBlock() override {
 		throw InternalException("Cannot perform IO in in-memory database - GetMetaBlock!");
 	}
 	void Read(Block &block) override {