duckdb 0.7.2-dev12.0 → 0.7.2-dev1244.0

package/src/duckdb/src/include/duckdb/planner/expression_binder/select_binder.hpp

@@ -8,58 +8,29 @@
 
 #pragma once
 
-#include "duckdb/common/case_insensitive_map.hpp"
-#include "duckdb/parser/expression_map.hpp"
-#include "duckdb/planner/expression_binder.hpp"
+#include "duckdb/planner/expression_binder/base_select_binder.hpp"
 
 namespace duckdb {
-class BoundColumnRefExpression;
-class WindowExpression;
-
-class BoundSelectNode;
-
-struct BoundGroupInformation {
-	expression_map_t<idx_t> map;
-	case_insensitive_map_t<idx_t> alias_map;
-};
 
 //! The SELECT binder is responsible for binding an expression within the SELECT clause of a SQL statement
-class SelectBinder : public ExpressionBinder {
+class SelectBinder : public BaseSelectBinder {
 public:
 	SelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info,
 	             case_insensitive_map_t<idx_t> alias_map);
 	SelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info);
 
-	bool BoundAggregates() {
-		return bound_aggregate;
+	bool HasExpandedExpressions() {
+		return !expanded_expressions.empty();
 	}
-	void ResetBindings() {
-		this->bound_aggregate = false;
-		this->bound_columns.clear();
+	vector<unique_ptr<Expression>> &ExpandedExpressions() {
+		return expanded_expressions;
 	}
 
 protected:
-	BindResult BindExpression(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth,
-	                          bool root_expression = false) override;
-
-	BindResult BindAggregate(FunctionExpression &expr, AggregateFunctionCatalogEntry *function, idx_t depth) override;
-
-	BindResult BindUnnest(FunctionExpression &function, idx_t depth) override;
-
-	bool inside_window;
-	bool bound_aggregate = false;
-
-	BoundSelectNode &node;
-	BoundGroupInformation &info;
-	case_insensitive_map_t<idx_t> alias_map;
-
-protected:
-	BindResult BindColumnRef(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth);
-	BindResult BindGroupingFunction(OperatorExpression &op, idx_t depth) override;
-	BindResult BindWindow(WindowExpression &expr, idx_t depth);
+	BindResult BindUnnest(FunctionExpression &function, idx_t depth, bool root_expression) override;
 
-	idx_t TryBindGroup(ParsedExpression &expr, idx_t depth);
-	BindResult BindGroup(ParsedExpression &expr, idx_t depth, idx_t group_index);
+	idx_t unnest_level = 0;
+	vector<unique_ptr<Expression>> expanded_expressions;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/expression_binder.hpp

@@ -135,7 +135,7 @@ protected:
 	virtual BindResult BindFunction(FunctionExpression &expr, ScalarFunctionCatalogEntry *function, idx_t depth);
 	virtual BindResult BindLambdaFunction(FunctionExpression &expr, ScalarFunctionCatalogEntry *function, idx_t depth);
 	virtual BindResult BindAggregate(FunctionExpression &expr, AggregateFunctionCatalogEntry *function, idx_t depth);
-	virtual BindResult BindUnnest(FunctionExpression &expr, idx_t depth);
+	virtual BindResult BindUnnest(FunctionExpression &expr, idx_t depth, bool root_expression);
 	virtual BindResult BindMacro(FunctionExpression &expr, ScalarMacroCatalogEntry *macro, idx_t depth,
 	                             unique_ptr<ParsedExpression> *expr_ptr);
 

package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp

@@ -14,6 +14,7 @@ class LogicalOperator;
 
 class LogicalAggregate;
 class LogicalAnyJoin;
+class LogicalAsOfJoin;
 class LogicalColumnDataGet;
 class LogicalComparisonJoin;
 class LogicalCopyToFile;

package/src/duckdb/src/include/duckdb/planner/operator/list.hpp

@@ -1,5 +1,6 @@
 #include "duckdb/planner/operator/logical_aggregate.hpp"
 #include "duckdb/planner/operator/logical_any_join.hpp"
+#include "duckdb/planner/operator/logical_asof_join.hpp"
 #include "duckdb/planner/operator/logical_column_data_get.hpp"
 #include "duckdb/planner/operator/logical_comparison_join.hpp"
 #include "duckdb/planner/operator/logical_copy_to_file.hpp"

package/src/duckdb/src/include/duckdb/planner/operator/logical_asof_join.hpp

@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/planner/operator/logical_asof_join.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/planner/operator/logical_comparison_join.hpp"
+
+namespace duckdb {
+
+//! LogicalAsOfJoin represents a temporal-style join with one less-than inequality.
+//! This inequality matches the greatest value on the right that satisfies the condition.
+class LogicalAsOfJoin : public LogicalComparisonJoin {
+public:
+	explicit LogicalAsOfJoin(JoinType type);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/common/constants.hpp"
+#include "duckdb/common/enums/joinref_type.hpp"
 #include "duckdb/common/unordered_set.hpp"
 #include "duckdb/planner/joinside.hpp"
 #include "duckdb/planner/operator/logical_join.hpp"
@@ -34,10 +35,12 @@ public:
 	                        FieldReader &reader);
 
 public:
-	static unique_ptr<LogicalOperator> CreateJoin(JoinType type, unique_ptr<LogicalOperator> left_child,
+	static unique_ptr<LogicalOperator> CreateJoin(JoinType type, JoinRefType ref_type,
+	                                              unique_ptr<LogicalOperator> left_child,
 	                                              unique_ptr<LogicalOperator> right_child,
 	                                              unique_ptr<Expression> condition);
-	static unique_ptr<LogicalOperator> CreateJoin(JoinType type, unique_ptr<LogicalOperator> left_child,
+	static unique_ptr<LogicalOperator> CreateJoin(JoinType type, JoinRefType ref_type,
+	                                              unique_ptr<LogicalOperator> left_child,
 	                                              unique_ptr<LogicalOperator> right_child,
 	                                              vector<JoinCondition> conditions,
 	                                              vector<unique_ptr<Expression>> arbitrary_expressions);

package/src/duckdb/src/include/duckdb/planner/operator/logical_distinct.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/planner/bound_result_modifier.hpp"
 
 namespace duckdb {
 
@@ -22,6 +23,8 @@ public:
 	}
 	//! The set of distinct targets (optional).
 	vector<unique_ptr<Expression>> distinct_targets;
+	//! The order by modifier (optional, only for distinct on)
+	unique_ptr<BoundOrderModifier> order_by;
 
 public:
 	string ParamsToString() const override;

package/src/duckdb/src/include/duckdb/planner/query_node/bound_select_node.hpp

@@ -25,6 +25,13 @@ public:
 	vector<GroupingSet> grouping_sets;
 };
 
+struct BoundUnnestNode {
+	//! The index of the UNNEST node
+	idx_t index;
+	//! The set of expressions
+	vector<unique_ptr<Expression>> expressions;
+};
+
 //! Bound equivalent of SelectNode
 class BoundSelectNode : public BoundQueryNode {
 public:
@@ -78,9 +85,8 @@ public:
 	//! Window functions to compute (only used if HasWindow is true)
 	vector<unique_ptr<Expression>> windows;
 
-	idx_t unnest_index;
 	//! Unnest expression
-	vector<unique_ptr<Expression>> unnests;
+	unordered_map<idx_t, BoundUnnestNode> unnests;
 
 	//! Index of pruned node
 	idx_t prune_index;

package/src/duckdb/src/include/duckdb/storage/buffer/block_handle.hpp

@@ -17,6 +17,7 @@ namespace duckdb {
 class BlockManager;
 class BufferHandle;
 class BufferManager;
+class BufferPool;
 class DatabaseInstance;
 class FileBuffer;
 
@@ -55,6 +56,7 @@ class BlockHandle {
 	friend struct BufferEvictionNode;
 	friend class BufferHandle;
 	friend class BufferManager;
+	friend class BufferPool;
 
 public:
 	BlockHandle(BlockManager &block_manager, block_id_t block_id);

package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp

@@ -23,23 +23,75 @@ class DatabaseInstance;
 class TemporaryDirectoryHandle;
 struct EvictionQueue;
 
+//! The BufferPool is in charge of handling memory management for one or more databases. It defines memory limits
+//! and implements priority eviction among all users of the pool.
+class BufferPool {
+	friend class BlockHandle;
+	friend class BlockManager;
+	friend class BufferManager;
+
+public:
+	explicit BufferPool(idx_t maximum_memory);
+	virtual ~BufferPool();
+
+	//! Set a new memory limit to the buffer pool, throws an exception if the new limit is too low and not enough
+	//! blocks can be evicted
+	void SetLimit(idx_t limit, const char *exception_postscript);
+
+	idx_t GetUsedMemory() {
+		return current_memory;
+	}
+	idx_t GetMaxMemory() {
+		return maximum_memory;
+	}
+
+protected:
+	//! Evict blocks until the currently used memory + extra_memory fit, returns false if this was not possible
+	//! (i.e. not enough blocks could be evicted)
+	//! If the "buffer" argument is specified AND the system can find a buffer to re-use for the given allocation size
+	//! "buffer" will be made to point to the re-usable memory. Note that this is not guaranteed.
+	//! Returns a pair. result.first indicates if eviction was successful. result.second contains the
+	//! reservation handle, which can be moved to the BlockHandle that will own the reservation.
+	struct EvictionResult {
+		bool success;
+		TempBufferPoolReservation reservation;
+	};
+	virtual EvictionResult EvictBlocks(idx_t extra_memory, idx_t memory_limit,
+	                                   unique_ptr<FileBuffer> *buffer = nullptr);
+
+	//! Garbage collect eviction queue
+	void PurgeQueue();
+	void AddToEvictionQueue(shared_ptr<BlockHandle> &handle);
+
+private:
+	//! The lock for changing the memory limit
+	mutex limit_lock;
+	//! The current amount of memory that is occupied by the buffer manager (in bytes)
+	atomic<idx_t> current_memory;
+	//! The maximum amount of memory that the buffer manager can keep (in bytes)
+	atomic<idx_t> maximum_memory;
+	//! Eviction queue
+	unique_ptr<EvictionQueue> queue;
+	//! Total number of insertions into the eviction queue. This guides the schedule for calling PurgeQueue.
+	atomic<uint32_t> queue_insertions;
+};
+
 struct TemporaryFileInformation {
 	string path;
 	idx_t size;
 };
 
-//! The buffer manager is in charge of handling memory management for the database. It hands out memory buffers that can
-//! be used by the database internally.
-//
-//! BlockIds are NOT unique within the context of a BufferManager. A buffer manager
-//! can be shared by many BlockManagers.
+//! The BufferManager is in charge of handling memory management for a singke database. It cooperatively shares a
+//! BufferPool with other BufferManagers, belonging to different databases. It hands out memory buffers that can
+//! be used by the database internally, and offers configuration options specific to a database, which need not be
+//! shared by the BufferPool, including whether to support swapping temp buffers to disk, and where to swap them to.
 class BufferManager {
 	friend class BufferHandle;
 	friend class BlockHandle;
 	friend class BlockManager;
 
 public:
-	BufferManager(DatabaseInstance &db, string temp_directory, idx_t maximum_memory);
+	BufferManager(DatabaseInstance &db, string temp_directory);
 	virtual ~BufferManager();
 
 	//! Registers an in-memory buffer that cannot be unloaded until it is destroyed
@@ -58,21 +110,17 @@ public:
 	BufferHandle Pin(shared_ptr<BlockHandle> &handle);
 	void Unpin(shared_ptr<BlockHandle> &handle);
 
-	//! Set a new memory limit to the buffer manager, throws an exception if the new limit is too low and not enough
-	//! blocks can be evicted
-	void SetLimit(idx_t limit = (idx_t)-1);
-
 	DUCKDB_API static BufferManager &GetBufferManager(ClientContext &context);
 	DUCKDB_API static BufferManager &GetBufferManager(DatabaseInstance &db);
 	DUCKDB_API static BufferManager &GetBufferManager(AttachedDatabase &db);
 
 	//! Returns the currently allocated memory
 	idx_t GetUsedMemory() {
-		return current_memory;
+		return buffer_pool.current_memory;
 	}
 	//! Returns the maximum available memory
 	idx_t GetMaxMemory() {
-		return maximum_memory;
+		return buffer_pool.maximum_memory;
 	}
 
 	//! Increases the currently allocated memory, but the actual allocation does not go through the buffer manager
@@ -92,6 +140,10 @@ public:
 		return db;
 	}
 
+	BufferPool &GetBufferPool() {
+		return buffer_pool;
+	}
+
 	static idx_t GetAllocSize(idx_t block_size) {
 		return AlignValue<idx_t, Storage::SECTOR_SIZE>(block_size + Storage::BLOCK_HEADER_SIZE);
 	}
@@ -103,6 +155,11 @@ public:
 	DUCKDB_API void ReserveMemory(idx_t size);
 	DUCKDB_API void FreeReservedMemory(idx_t size);
 
+	//! Set a new memory limit to the buffer pool, throws an exception if the new limit is too low and not enough
+	//! blocks can be evicted. (Sugar for calling method directly on the BufferPool.)
+	void SetLimit(idx_t limit = (idx_t)-1) {
+		buffer_pool.SetLimit(limit, InMemoryWarning());
+	}
 	//! Returns a list of all temporary files
 	vector<TemporaryFileInformation> GetTemporaryFiles();
 
@@ -113,25 +170,6 @@ private:
 	//! This needs to be private to prevent creating blocks without ever pinning them:
 	//! blocks that are never pinned are never added to the eviction queue
 	shared_ptr<BlockHandle> RegisterMemory(idx_t block_size, bool can_destroy);
-	//! Evict blocks until the currently used memory + extra_memory fit, returns false if this was not possible
-	//! (i.e. not enough blocks could be evicted)
-	//! If the "buffer" argument is specified AND the system can find a buffer to re-use for the given allocation size
-	//! "buffer" will be made to point to the re-usable memory. Note that this is not guaranteed.
-	//! Returns a pair. result.first indicates if eviction was successful. result.second contains the
-	//! reservation handle, which can be moved to the BlockHandle that will own the reservation.
-	struct EvictionResult {
-		bool success;
-		TempBufferPoolReservation reservation;
-	};
-	EvictionResult EvictBlocks(idx_t extra_memory, idx_t memory_limit, unique_ptr<FileBuffer> *buffer = nullptr);
-
-	//! Helper
-	template <typename... ARGS>
-	TempBufferPoolReservation EvictBlocksOrThrow(idx_t extra_memory, idx_t limit, unique_ptr<FileBuffer> *buffer,
-	                                             ARGS...);
-
-	//! Garbage collect eviction queue
-	void PurgeQueue();
 
 	//! Write a temporary buffer to disk
 	void WriteTemporaryBuffer(block_id_t block_id, FileBuffer &buffer);
@@ -144,9 +182,7 @@ private:
 
 	void RequireTemporaryDirectory();
 
-	void AddToEvictionQueue(shared_ptr<BlockHandle> &handle);
-
-	string InMemoryWarning();
+	const char *InMemoryWarning();
 
 	static data_ptr_t BufferAllocatorAllocate(PrivateAllocatorData *private_data, idx_t size);
 	static void BufferAllocatorFree(PrivateAllocatorData *private_data, data_ptr_t pointer, idx_t size);
@@ -157,27 +193,23 @@ private:
 	//! overwrites the data within with garbage. Any readers that do not hold the pin will notice
 	void VerifyZeroReaders(shared_ptr<BlockHandle> &handle);
 
+	//! Helper
+	template <typename... ARGS>
+	TempBufferPoolReservation EvictBlocksOrThrow(idx_t extra_memory, unique_ptr<FileBuffer> *buffer, ARGS...);
+
 private:
 	//! The database instance
 	DatabaseInstance &db;
-	//! The lock for changing the memory limit
-	mutex limit_lock;
-	//! The current amount of memory that is occupied by the buffer manager (in bytes)
-	atomic<idx_t> current_memory;
-	//! The maximum amount of memory that the buffer manager can keep (in bytes)
-	atomic<idx_t> maximum_memory;
+	//! The buffer pool
+	BufferPool &buffer_pool;
 	//! The directory name where temporary files are stored
 	string temp_directory;
 	//! Lock for creating the temp handle
 	mutex temp_handle_lock;
 	//! Handle for the temporary directory
 	unique_ptr<TemporaryDirectoryHandle> temp_directory_handle;
-	//! Eviction queue
-	unique_ptr<EvictionQueue> queue;
 	//! The temporary id used for managed buffers
 	atomic<block_id_t> temporary_id;
-	//! Total number of insertions into the eviction queue. This guides the schedule for calling PurgeQueue.
-	atomic<uint32_t> queue_insertions;
 	//! Allocator associated with the buffer manager, that passes all allocations through this buffer manager
 	Allocator buffer_allocator;
 	//! Block manager for temp data

package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp

@@ -12,6 +12,7 @@
 
 namespace duckdb {
 class DuckTableEntry;
+class TableStatistics;
 
 //! The table data writer is responsible for writing the data of a table to
 //! storage.
@@ -30,7 +31,7 @@ public:
 
 	CompressionType GetColumnCompressionType(idx_t i);
 
-	virtual void FinalizeTable(vector<unique_ptr<BaseStatistics>> &&global_stats, DataTableInfo *info) = 0;
+	virtual void FinalizeTable(TableStatistics &&global_stats, DataTableInfo *info) = 0;
 	virtual unique_ptr<RowGroupWriter> GetRowGroupWriter(RowGroup &row_group) = 0;
 
 	virtual void AddRowGroup(RowGroupPointer &&row_group_pointer, unique_ptr<RowGroupWriter> &&writer);
@@ -47,7 +48,7 @@ public:
 	                          MetaBlockWriter &table_data_writer, MetaBlockWriter &meta_data_writer);
 
 public:
-	virtual void FinalizeTable(vector<unique_ptr<BaseStatistics>> &&global_stats, DataTableInfo *info) override;
+	virtual void FinalizeTable(TableStatistics &&global_stats, DataTableInfo *info) override;
 	virtual unique_ptr<RowGroupWriter> GetRowGroupWriter(RowGroup &row_group) override;
 
 private:

package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp

@@ -44,7 +44,7 @@ protected:
 	virtual void WriteMacro(ScalarMacroCatalogEntry &table);
 	virtual void WriteTableMacro(TableMacroCatalogEntry &table);
 	virtual void WriteIndex(IndexCatalogEntry &index_catalog);
-	virtual void WriteType(TypeCatalogEntry &table);
+	virtual void WriteType(TypeCatalogEntry &type);
 };
 
 class CheckpointReader {

package/src/duckdb/src/include/duckdb/storage/compression/chimp/chimp_compress.hpp

@@ -18,7 +18,7 @@
 #include "duckdb/function/compression/compression.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/operator/subtract.hpp"
@@ -150,7 +150,7 @@ public:
 
 		if (is_valid) {
 			T floating_point_value = Load<T>((const_data_ptr_t)&value);
-			NumericStatistics::Update<T>(current_segment->stats, floating_point_value);
+			NumericStats::Update<T>(current_segment->stats.statistics, floating_point_value);
 		} else {
 			//! FIXME: find a cheaper alternative to storing a NULL
 			// store this as "value_identical", only using 9 bits for a NULL

package/src/duckdb/src/include/duckdb/storage/compression/chimp/chimp_fetch.hpp

@@ -17,7 +17,7 @@
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/operator/subtract.hpp"

package/src/duckdb/src/include/duckdb/storage/compression/chimp/chimp_scan.hpp

@@ -17,13 +17,14 @@
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/operator/subtract.hpp"
 
 #include "duckdb/storage/compression/chimp/algorithm/flag_buffer.hpp"
 #include "duckdb/storage/compression/chimp/algorithm/leading_zero_buffer.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/storage/compression/patas/patas_compress.hpp

@@ -18,7 +18,7 @@
 #include "duckdb/function/compression/compression.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/operator/subtract.hpp"
@@ -49,7 +49,7 @@ public:
 			}
 
 			if (is_valid) {
-				NumericStatistics::Update<VALUE_TYPE>(state_wrapper->current_segment->stats, value);
+				NumericStats::Update<VALUE_TYPE>(state_wrapper->current_segment->stats.statistics, value);
 			}
 
 			state_wrapper->WriteValue(Load<EXACT_TYPE>((const_data_ptr_t)&value));

package/src/duckdb/src/include/duckdb/storage/compression/patas/patas_fetch.hpp

@@ -17,7 +17,7 @@
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/operator/subtract.hpp"

package/src/duckdb/src/include/duckdb/storage/compression/patas/patas_scan.hpp

@@ -17,10 +17,11 @@
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/operator/subtract.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp

@@ -18,12 +18,15 @@
 namespace duckdb {
 
 struct DataPointer {
+	DataPointer(BaseStatistics stats) : statistics(std::move(stats)) {
+	}
+
 	uint64_t row_start;
 	uint64_t tuple_count;
 	BlockPointer block_pointer;
 	CompressionType compression_type;
 	//! Type-specific statistics of the segment
-	unique_ptr<BaseStatistics> statistics;
+	BaseStatistics statistics;
 };
 
 struct RowGroupPointer {
@@ -31,8 +34,6 @@ struct RowGroupPointer {
 	uint64_t tuple_count;
 	//! The data pointers of the column segments stored in the row group
 	vector<BlockPointer> data_pointers;
-	//! The per-column statistics of the row group
-	vector<unique_ptr<BaseStatistics>> statistics;
 	//! The versions information of the row group (if any)
 	shared_ptr<VersionNode> versions;
 };

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -39,6 +39,7 @@ class Transaction;
 class WriteAheadLog;
 class TableDataWriter;
 class ConflictManager;
+class TableScanState;
 enum class VerifyExistenceType : uint8_t;
 
 //! DataTable represents a physical table on disk
@@ -150,8 +151,8 @@ public:
 
 	//! Append a chunk with the row ids [row_start, ..., row_start + chunk.size()] to all indexes of the table, returns
 	//! whether or not the append succeeded
-	bool AppendToIndexes(DataChunk &chunk, row_t row_start);
-	static bool AppendToIndexes(TableIndexList &indexes, DataChunk &chunk, row_t row_start);
+	PreservedError AppendToIndexes(DataChunk &chunk, row_t row_start);
+	static PreservedError AppendToIndexes(TableIndexList &indexes, DataChunk &chunk, row_t row_start);
 	//! Remove a chunk with the row ids [row_start, ..., row_start + chunk.size()] from all indexes of the table
 	void RemoveFromIndexes(TableAppendState &state, DataChunk &chunk, row_t row_start);
 	//! Remove the chunk with the specified set of row identifiers from all indexes of the table
@@ -169,7 +170,7 @@ public:
 	//! Get statistics of a physical column within the table
 	unique_ptr<BaseStatistics> GetStatistics(ClientContext &context, column_t column_id);
 	//! Sets statistics of a physical column within the table
-	void SetStatistics(column_t column_id, const std::function<void(BaseStatistics &)> &set_fun);
+	void SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats);
 
 	//! Checkpoint the table to the specified table data writer
 	void Checkpoint(TableDataWriter &writer);

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -14,7 +14,6 @@
 #include "duckdb/common/sort/sort.hpp"
 #include "duckdb/parser/parsed_expression.hpp"
 #include "duckdb/planner/expression.hpp"
-#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/meta_block_writer.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/common/types/constraint_conflict_info.hpp"
@@ -27,6 +26,7 @@ class Transaction;
 class ConflictManager;
 
 struct IndexLock;
+struct IndexScanState;
 
 //! The index is an abstract base class that serves as the basis for indexes
 class Index {
@@ -80,9 +80,9 @@ public:
 	//! Obtain a lock on the index
 	virtual void InitializeLock(IndexLock &state);
 	//! Called when data is appended to the index. The lock obtained from InitializeLock must be held
-	virtual bool Append(IndexLock &state, DataChunk &entries, Vector &row_identifiers) = 0;
+	virtual PreservedError Append(IndexLock &state, DataChunk &entries, Vector &row_identifiers) = 0;
 	//! Obtains a lock and calls Append while holding that lock
-	bool Append(DataChunk &entries, Vector &row_identifiers);
+	PreservedError Append(DataChunk &entries, Vector &row_identifiers);
 	//! Verify that data can be appended to the index without a constraint violation
 	virtual void VerifyAppend(DataChunk &chunk) = 0;
 	//! Verify that data can be appended to the index without a constraint violation using the conflict manager
@@ -96,7 +96,7 @@ public:
 	void Delete(DataChunk &entries, Vector &row_identifiers);
 
 	//! Insert a chunk of entries into the index
-	virtual bool Insert(IndexLock &lock, DataChunk &input, Vector &row_identifiers) = 0;
+	virtual PreservedError Insert(IndexLock &lock, DataChunk &input, Vector &row_identifiers) = 0;
 
 	//! Merge another index into this index. The lock obtained from InitializeLock must be held, and the other
 	//! index must also be locked during the merge
@@ -147,6 +147,7 @@ public:
 
 	//! Execute the index expressions on an input chunk
 	void ExecuteExpressions(DataChunk &input, DataChunk &result);
+	static string AppendRowError(DataChunk &input, idx_t index);
 
 protected:
 	//! Lock used for any changes to the index

package/src/duckdb/src/include/duckdb/storage/meta_block_reader.hpp

@@ -36,7 +36,14 @@ public:
 	//! Read content of size read_size into the buffer
 	void ReadData(data_ptr_t buffer, idx_t read_size) override;
 
+	ClientContext &GetContext() override;
+	Catalog *GetCatalog() override;
+	void SetCatalog(Catalog *catalog_p);
+	void SetContext(ClientContext *context_p);
+
 private:
 	void ReadNewBlock(block_id_t id);
+	Catalog *catalog = nullptr;
+	ClientContext *context = nullptr;
 };
 } // namespace duckdb