duckdb 0.10.3-dev3.0 → 0.10.3-dev6.0

package/src/duckdb/src/include/duckdb/planner/binder.hpp

@@ -43,6 +43,7 @@ class ColumnList;
 class ExternalDependency;
 class TableFunction;
 class TableStorageInfo;
+class BoundConstraint;
 
 struct CreateInfo;
 struct BoundCreateTableInfo;
@@ -55,6 +56,7 @@ struct PivotColumnEntry;
 struct UnpivotEntry;
 
 enum class BindingMode : uint8_t { STANDARD_BINDING, EXTRACT_NAMES };
+enum class BinderType : uint8_t { REGULAR_BINDER, VIEW_BINDER };
 
 struct CorrelatedColumnInfo {
 	ColumnBinding binding;
@@ -81,13 +83,13 @@ struct CorrelatedColumnInfo {
   tables and columns in the catalog. In the process, it also resolves types of
   all expressions.
 */
-class Binder : public std::enable_shared_from_this<Binder> {
+class Binder : public enable_shared_from_this<Binder> {
 	friend class ExpressionBinder;
 	friend class RecursiveDependentJoinPlanner;
 
 public:
 	DUCKDB_API static shared_ptr<Binder> CreateBinder(ClientContext &context, optional_ptr<Binder> parent = nullptr,
-	                                                  bool inherit_ctes = true);
+	                                                  BinderType binder_type = BinderType::REGULAR_BINDER);
 
 	//! The client context
 	ClientContext &context;
@@ -102,8 +104,6 @@ public:
 	vector<CorrelatedColumnInfo> correlated_columns;
 	//! The set of parameter expressions bound by this binder
 	optional_ptr<BoundParameterMap> parameters;
-	//! Statement properties
-	StatementProperties properties;
 	//! The alias for the currently processing subquery, if it exists
 	string alias;
 	//! Macro parameter bindings (if any)
@@ -119,7 +119,20 @@ public:
 
 	unique_ptr<BoundCreateTableInfo> BindCreateTableInfo(unique_ptr<CreateInfo> info);
 	unique_ptr<BoundCreateTableInfo> BindCreateTableInfo(unique_ptr<CreateInfo> info, SchemaCatalogEntry &schema);
-
+	unique_ptr<BoundCreateTableInfo> BindCreateTableInfo(unique_ptr<CreateInfo> info, SchemaCatalogEntry &schema,
+	                                                     vector<unique_ptr<Expression>> &bound_defaults);
+	static unique_ptr<BoundCreateTableInfo> BindCreateTableCheckpoint(unique_ptr<CreateInfo> info,
+	                                                                  SchemaCatalogEntry &schema);
+	static vector<unique_ptr<BoundConstraint>> BindConstraints(ClientContext &context,
+	                                                           const vector<unique_ptr<Constraint>> &constraints,
+	                                                           const string &table_name, const ColumnList &columns);
+	vector<unique_ptr<BoundConstraint>> BindConstraints(const vector<unique_ptr<Constraint>> &constraints,
+	                                                    const string &table_name, const ColumnList &columns);
+	vector<unique_ptr<BoundConstraint>> BindConstraints(const TableCatalogEntry &table);
+	vector<unique_ptr<BoundConstraint>> BindNewConstraints(vector<unique_ptr<Constraint>> &constraints,
+	                                                       const string &table_name, const ColumnList &columns);
+
+	void SetCatalogLookupCallback(catalog_entry_callback_t callback);
 	void BindCreateViewInfo(CreateViewInfo &base);
 	SchemaCatalogEntry &BindSchema(CreateInfo &info);
 	SchemaCatalogEntry &BindCreateFunctionInfo(CreateInfo &info);
@@ -135,6 +148,10 @@ public:
 	//! Generates an unused index for a table
 	idx_t GenerateTableIndex();
 
+	optional_ptr<CatalogEntry> GetCatalogEntry(CatalogType type, const string &catalog, const string &schema,
+	                                           const string &name, OnEntryNotFound on_entry_not_found,
+	                                           QueryErrorContext &error_context);
+
 	//! Add a common table expression to the binder
 	void AddCTE(const string &name, CommonTableExpressionInfo &cte);
 	//! Find all candidate common table expression by name; returns empty vector if none exists
@@ -167,8 +184,8 @@ public:
 	void BindVacuumTable(LogicalVacuum &vacuum, unique_ptr<LogicalOperator> &root);
 
 	static void BindSchemaOrCatalog(ClientContext &context, string &catalog, string &schema);
-	static void BindLogicalType(ClientContext &context, LogicalType &type, optional_ptr<Catalog> catalog = nullptr,
-	                            const string &schema = INVALID_SCHEMA);
+	void BindLogicalType(LogicalType &type, optional_ptr<Catalog> catalog = nullptr,
+	                     const string &schema = INVALID_SCHEMA);
 
 	bool HasMatchingBinding(const string &table_name, const string &column_name, ErrorData &error);
 	bool HasMatchingBinding(const string &schema_name, const string &table_name, const string &column_name,
@@ -187,6 +204,8 @@ public:
 	void SetCanContainNulls(bool can_contain_nulls);
 	void SetAlwaysRequireRebind();
 
+	StatementProperties &GetStatementProperties();
+
 private:
 	//! The parent binder (if any)
 	shared_ptr<Binder> parent;
@@ -198,8 +217,8 @@ private:
 	bool has_unplanned_dependent_joins = false;
 	//! Whether or not outside dependent joins have been planned and flattened
 	bool is_outside_flattened = true;
-	//! Whether CTEs should reference the parent binder (if it exists)
-	bool inherit_ctes = true;
+	//! What kind of node we are binding using this binder
+	BinderType binder_type = BinderType::REGULAR_BINDER;
 	//! Whether or not the binder can contain NULLs as the root of expressions
 	bool can_contain_nulls = false;
 	//! The root statement of the query that is currently being parsed
@@ -210,8 +229,12 @@ private:
 	unordered_set<string> table_names;
 	//! The set of bound views
 	reference_set_t<ViewCatalogEntry> bound_views;
+	//! Used to retrieve CatalogEntry's
+	CatalogEntryRetriever entry_retriever;
 	//! Unnamed subquery index
 	idx_t unnamed_subquery_index = 1;
+	//! Statement properties
+	StatementProperties prop;
 
 private:
 	//! Get the root binder (binder with no parent)
@@ -261,6 +284,7 @@ private:
 	BoundStatement Bind(AttachStatement &stmt);
 	BoundStatement Bind(DetachStatement &stmt);
 	BoundStatement Bind(CopyDatabaseStatement &stmt);
+	BoundStatement Bind(UpdateExtensionsStatement &stmt);
 
 	BoundStatement BindReturning(vector<unique_ptr<ParsedExpression>> returning_list, TableCatalogEntry &table,
 	                             const string &alias, idx_t update_table_index,
@@ -291,6 +315,7 @@ private:
 	unique_ptr<BoundTableRef> Bind(TableFunctionRef &ref);
 	unique_ptr<BoundTableRef> Bind(EmptyTableRef &ref);
 	unique_ptr<BoundTableRef> Bind(ExpressionListRef &ref);
+	unique_ptr<BoundTableRef> Bind(ColumnDataRef &ref);
 	unique_ptr<BoundTableRef> Bind(PivotRef &expr);
 	unique_ptr<BoundTableRef> Bind(ShowRef &ref);
 
@@ -309,11 +334,11 @@ private:
 	bool BindTableInTableOutFunction(vector<unique_ptr<ParsedExpression>> &expressions,
 	                                 unique_ptr<BoundSubqueryRef> &subquery, ErrorData &error);
 	unique_ptr<LogicalOperator> BindTableFunction(TableFunction &function, vector<Value> parameters);
-	unique_ptr<LogicalOperator>
-	BindTableFunctionInternal(TableFunction &table_function, const string &function_name, vector<Value> parameters,
-	                          named_parameter_map_t named_parameters, vector<LogicalType> input_table_types,
-	                          vector<string> input_table_names, const vector<string> &column_name_alias,
-	                          unique_ptr<ExternalDependency> external_dependency);
+	unique_ptr<LogicalOperator> BindTableFunctionInternal(TableFunction &table_function, const TableFunctionRef &ref,
+	                                                      vector<Value> parameters,
+	                                                      named_parameter_map_t named_parameters,
+	                                                      vector<LogicalType> input_table_types,
+	                                                      vector<string> input_table_names);
 
 	unique_ptr<LogicalOperator> CreatePlan(BoundBaseTableRef &ref);
 	unique_ptr<LogicalOperator> CreatePlan(BoundJoinRef &ref);
@@ -321,6 +346,7 @@ private:
 	unique_ptr<LogicalOperator> CreatePlan(BoundTableFunction &ref);
 	unique_ptr<LogicalOperator> CreatePlan(BoundEmptyTableRef &ref);
 	unique_ptr<LogicalOperator> CreatePlan(BoundExpressionListRef &ref);
+	unique_ptr<LogicalOperator> CreatePlan(BoundColumnDataRef &ref);
 	unique_ptr<LogicalOperator> CreatePlan(BoundCTERef &ref);
 	unique_ptr<LogicalOperator> CreatePlan(BoundPivotRef &ref);
 
@@ -379,9 +405,9 @@ private:
 	unique_ptr<BoundTableRef> BindSummarize(ShowRef &ref);
 
 public:
-	// This should really be a private constructor, but make_shared does not allow it...
+	// This should really be a private constructor, but make_shared_ptr does not allow it...
 	// If you are thinking about calling this, you should probably call Binder::CreateBinder
-	Binder(bool i_know_what_i_am_doing, ClientContext &context, shared_ptr<Binder> parent, bool inherit_ctes);
+	Binder(bool i_know_what_i_am_doing, ClientContext &context, shared_ptr<Binder> parent, BinderType binder_type);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp

@@ -52,6 +52,7 @@ class BoundSubqueryRef;
 class BoundTableFunction;
 class BoundEmptyTableRef;
 class BoundExpressionListRef;
+class BoundColumnDataRef;
 class BoundCTERef;
 class BoundPivotRef;
 

package/src/duckdb/src/include/duckdb/planner/expression_binder/alter_binder.hpp

@@ -14,22 +14,23 @@
 namespace duckdb {
 class TableCatalogEntry;
 
-//! The ALTER binder is responsible for binding an expression within alter statements
+//! The AlterBinder binds expressions in ALTER statements.
 class AlterBinder : public ExpressionBinder {
 public:
 	AlterBinder(Binder &binder, ClientContext &context, TableCatalogEntry &table, vector<LogicalIndex> &bound_columns,
 	            LogicalType target_type);
 
-	TableCatalogEntry &table;
-	vector<LogicalIndex> &bound_columns;
-
 protected:
+	BindResult BindLambdaReference(LambdaRefExpression &expr, idx_t depth);
+	BindResult BindColumnReference(ColumnRefExpression &expr, idx_t depth);
 	BindResult BindExpression(unique_ptr<ParsedExpression> &expr_ptr, idx_t depth,
 	                          bool root_expression = false) override;
 
-	BindResult BindColumn(ColumnRefExpression &expr);
-
 	string UnsupportedAggregateMessage() override;
+
+private:
+	TableCatalogEntry &table;
+	vector<LogicalIndex> &bound_columns;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/expression_binder/having_binder.hpp

@@ -14,13 +14,14 @@
 
 namespace duckdb {
 
-//! The HAVING binder is responsible for binding an expression within the HAVING clause of a SQL statement
+//! The HAVING binder is responsible for binding an expression within the HAVING clause of a SQL statement.
 class HavingBinder : public BaseSelectBinder {
 public:
 	HavingBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info,
 	             AggregateHandling aggregate_handling);
 
 protected:
+	BindResult BindLambdaReference(LambdaRefExpression &expr, idx_t depth);
 	BindResult BindWindow(WindowExpression &expr, idx_t depth) override;
 	BindResult BindColumnRef(unique_ptr<ParsedExpression> &expr_ptr, idx_t depth, bool root_expression) override;
 

package/src/duckdb/src/include/duckdb/planner/expression_binder/index_binder.hpp

@@ -12,17 +12,21 @@
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/parser/parsed_data/create_index_info.hpp"
 #include "duckdb/planner/expression_binder.hpp"
+#include "duckdb/execution/index/bound_index.hpp"
+#include "duckdb/execution/index/unbound_index.hpp"
 
 namespace duckdb {
 
 class BoundColumnRefExpression;
 
-//! The IndexBinder is responsible for binding an expression within an index statement
+//! The IndexBinder is responsible for binding indexes, as well as expressions within an index statement
 class IndexBinder : public ExpressionBinder {
 public:
 	IndexBinder(Binder &binder, ClientContext &context, optional_ptr<TableCatalogEntry> table = nullptr,
 	            optional_ptr<CreateIndexInfo> info = nullptr);
 
+	unique_ptr<BoundIndex> BindIndex(const UnboundIndex &index);
+
 protected:
 	BindResult BindExpression(unique_ptr<ParsedExpression> &expr_ptr, idx_t depth,
 	                          bool root_expression = false) override;

package/src/duckdb/src/include/duckdb/planner/expression_binder.hpp

@@ -18,6 +18,7 @@
 #include "duckdb/parser/parsed_expression.hpp"
 #include "duckdb/parser/tokens.hpp"
 #include "duckdb/planner/expression.hpp"
+#include "duckdb/catalog/catalog_entry_retriever.hpp"
 #include "duckdb/planner/expression/bound_lambda_expression.hpp"
 #include "duckdb/function/scalar_function.hpp"
 
@@ -90,6 +91,7 @@ public:
 		return bound_columns;
 	}
 
+	void SetCatalogLookupCallback(catalog_entry_callback_t callback);
 	ErrorData Bind(unique_ptr<ParsedExpression> &expr, idx_t depth, bool root_expression = false);
 
 	//! Returns the STRUCT_EXTRACT operator expression
@@ -194,6 +196,9 @@ protected:
 
 	virtual string UnsupportedAggregateMessage();
 	virtual string UnsupportedUnnestMessage();
+	optional_ptr<CatalogEntry> GetCatalogEntry(CatalogType type, const string &catalog, const string &schema,
+	                                           const string &name, OnEntryNotFound on_entry_not_found,
+	                                           QueryErrorContext &error_context);
 
 	Binder &binder;
 	ClientContext &context;

package/src/duckdb/src/include/duckdb/planner/logical_operator.hpp

@@ -11,7 +11,6 @@
 #include "duckdb/catalog/catalog.hpp"
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/enums/logical_operator_type.hpp"
-#include "duckdb/optimizer/join_order/estimated_properties.hpp"
 #include "duckdb/planner/column_binding.hpp"
 #include "duckdb/planner/expression.hpp"
 #include "duckdb/planner/logical_operator_visitor.hpp"

package/src/duckdb/src/include/duckdb/planner/operator/logical_column_data_get.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/types/column/column_data_collection.hpp"
 #include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/common/optionally_owned_ptr.hpp"
 
 namespace duckdb {
 
@@ -20,13 +21,16 @@ public:
 
 public:
 	LogicalColumnDataGet(idx_t table_index, vector<LogicalType> types, unique_ptr<ColumnDataCollection> collection);
+	LogicalColumnDataGet(idx_t table_index, vector<LogicalType> types, ColumnDataCollection &to_scan);
+	LogicalColumnDataGet(idx_t table_index, vector<LogicalType> types,
+	                     optionally_owned_ptr<ColumnDataCollection> to_scan);
 
 	//! The table index in the current bind context
 	idx_t table_index;
 	//! The types of the chunk
 	vector<LogicalType> chunk_types;
-	//! The chunk collection to scan
-	unique_ptr<ColumnDataCollection> collection;
+	//! (optionally owned) column data collection
+	optionally_owned_ptr<ColumnDataCollection> collection;
 
 public:
 	vector<ColumnBinding> GetColumnBindings() override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_delete.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/planner/bound_constraint.hpp"
 
 namespace duckdb {
 class TableCatalogEntry;
@@ -23,6 +24,7 @@ public:
 	TableCatalogEntry &table;
 	idx_t table_index;
 	bool return_chunk;
+	vector<unique_ptr<BoundConstraint>> bound_constraints;
 
 public:
 	void Serialize(Serializer &serializer) const override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_export.hpp

@@ -28,6 +28,13 @@ public:
 	unique_ptr<CopyInfo> copy_info;
 	BoundExportData exported_tables;
 
+	void Serialize(Serializer &serializer) const override {
+		throw NotImplementedException("FIXME: Serialize LogicalExport statement");
+	}
+	unique_ptr<LogicalOperator> Deserialize(Deserializer &deserializer) {
+		throw NotImplementedException("FIXME: Deserialize LogicalExport statement");
+	}
+
 public:
 protected:
 	void ResolveTypes() override {

package/src/duckdb/src/include/duckdb/planner/operator/logical_insert.hpp

@@ -37,6 +37,8 @@ public:
 	bool return_chunk;
 	//! The default statements used by the table
 	vector<unique_ptr<Expression>> bound_defaults;
+	//! The constraints used by the table
+	vector<unique_ptr<BoundConstraint>> bound_constraints;
 
 	//! Which action to take on conflict
 	OnConflictAction action_type;

package/src/duckdb/src/include/duckdb/planner/operator/logical_top_n.hpp

@@ -19,15 +19,15 @@ public:
 	static constexpr const LogicalOperatorType TYPE = LogicalOperatorType::LOGICAL_TOP_N;
 
 public:
-	LogicalTopN(vector<BoundOrderByNode> orders, int64_t limit, int64_t offset)
+	LogicalTopN(vector<BoundOrderByNode> orders, idx_t limit, idx_t offset)
 	    : LogicalOperator(LogicalOperatorType::LOGICAL_TOP_N), orders(std::move(orders)), limit(limit), offset(offset) {
 	}
 
 	vector<BoundOrderByNode> orders;
 	//! The maximum amount of elements to emit
-	int64_t limit;
+	idx_t limit;
 	//! The offset from the start to begin emitting elements
-	int64_t offset;
+	idx_t offset;
 
 public:
 	vector<ColumnBinding> GetColumnBindings() override {

package/src/duckdb/src/include/duckdb/planner/operator/logical_update.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/planner/bound_constraint.hpp"
 
 namespace duckdb {
 class TableCatalogEntry;
@@ -28,6 +29,7 @@ public:
 	bool return_chunk;
 	vector<PhysicalIndex> columns;
 	vector<unique_ptr<Expression>> bound_defaults;
+	vector<unique_ptr<BoundConstraint>> bound_constraints;
 	bool update_is_del_and_insert;
 
 public:

package/src/duckdb/src/include/duckdb/planner/parsed_data/bound_create_table_info.hpp

@@ -36,10 +36,6 @@ struct BoundCreateTableInfo {
 	ColumnDependencyManager column_dependency_manager;
 	//! List of constraints on the table
 	vector<unique_ptr<Constraint>> constraints;
-	//! List of bound constraints on the table
-	vector<unique_ptr<BoundConstraint>> bound_constraints;
-	//! Bound default values
-	vector<unique_ptr<Expression>> bound_defaults;
 	//! Dependents of the table (in e.g. default values)
 	LogicalDependencyList dependencies;
 	//! The existing table data on disk (if any)

package/src/duckdb/src/include/duckdb/planner/table_binding.hpp

@@ -133,12 +133,12 @@ public:
 
 public:
 	//! Binding macros
-	BindResult Bind(ColumnRefExpression &colref, idx_t depth) override;
+	BindResult Bind(ColumnRefExpression &col_ref, idx_t depth) override;
 	//! Binding lambdas
-	BindResult Bind(LambdaRefExpression &lambdaref, idx_t depth);
+	BindResult Bind(LambdaRefExpression &lambda_ref, idx_t depth);
 
-	//! Given the parameter colref, returns a copy of the argument that was supplied for this parameter
-	unique_ptr<ParsedExpression> ParamToArg(ColumnRefExpression &colref);
+	//! Returns a copy of the col_ref parameter as a parsed expression
+	unique_ptr<ParsedExpression> ParamToArg(ColumnRefExpression &col_ref);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/tableref/bound_column_data_ref.hpp

@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/planner/tableref/bound_column_data_ref.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/planner/bound_tableref.hpp"
+#include "duckdb/common/optionally_owned_ptr.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
+
+namespace duckdb {
+//! Represents a TableReference to a base table in the schema
+class BoundColumnDataRef : public BoundTableRef {
+public:
+	static constexpr const TableReferenceType TYPE = TableReferenceType::COLUMN_DATA;
+
+public:
+	explicit BoundColumnDataRef(optionally_owned_ptr<ColumnDataCollection> collection)
+	    : BoundTableRef(TableReferenceType::COLUMN_DATA), collection(std::move(collection)) {
+	}
+	//! The (optionally owned) materialized column data to scan
+	optionally_owned_ptr<ColumnDataCollection> collection;
+	//! The index in the bind context
+	idx_t bind_index;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/tableref/list.hpp

@@ -4,5 +4,6 @@
 #include "duckdb/planner/tableref/bound_expressionlistref.hpp"
 #include "duckdb/planner/tableref/bound_joinref.hpp"
 #include "duckdb/planner/tableref/bound_subqueryref.hpp"
+#include "duckdb/planner/tableref/bound_column_data_ref.hpp"
 #include "duckdb/planner/tableref/bound_table_function.hpp"
 #include "duckdb/planner/tableref/bound_pivotref.hpp"

package/src/duckdb/src/include/duckdb/storage/buffer/block_handle.hpp

@@ -11,6 +11,7 @@
 #include "duckdb/common/atomic.hpp"
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/mutex.hpp"
+#include "duckdb/common/numeric_utils.hpp"
 #include "duckdb/storage/storage_info.hpp"
 #include "duckdb/common/file_buffer.hpp"
 #include "duckdb/common/enums/memory_tag.hpp"
@@ -76,7 +77,7 @@ public:
 		D_ASSERT(buffer);
 		// resize and adjust current memory
 		buffer->Resize(block_size);
-		memory_usage += memory_delta;
+		memory_usage = NumericCast<idx_t>(NumericCast<int64_t>(memory_usage) + memory_delta);
 		D_ASSERT(memory_usage == buffer->AllocSize());
 	}
 
@@ -121,8 +122,10 @@ private:
 	MemoryTag tag;
 	//! Pointer to loaded data (if any)
 	unique_ptr<FileBuffer> buffer;
-	//! Internal eviction timestamp
-	atomic<idx_t> eviction_timestamp;
+	//! Internal eviction sequence number
+	atomic<idx_t> eviction_seq_num;
+	//! LRU timestamp (for age-based eviction)
+	atomic<int64_t> lru_timestamp_msec;
 	//! Whether or not the buffer can be destroyed (only used for temporary buffers)
 	bool can_destroy;
 	//! The memory usage of the block (when loaded). If we are pinning/loading

package/src/duckdb/src/include/duckdb/storage/buffer/buffer_pool.hpp

@@ -20,13 +20,10 @@ struct EvictionQueue;
 struct BufferEvictionNode {
 	BufferEvictionNode() {
 	}
-	BufferEvictionNode(weak_ptr<BlockHandle> handle_p, idx_t timestamp_p)
-	    : handle(std::move(handle_p)), timestamp(timestamp_p) {
-		D_ASSERT(!handle.expired());
-	}
+	BufferEvictionNode(weak_ptr<BlockHandle> handle_p, idx_t eviction_seq_num);
 
 	weak_ptr<BlockHandle> handle;
-	idx_t timestamp;
+	idx_t handle_sequence_number;
 
 	bool CanUnload(BlockHandle &handle_p);
 	shared_ptr<BlockHandle> TryGetBlockHandle();
@@ -41,14 +38,14 @@ class BufferPool {
 	friend class StandardBufferManager;
 
 public:
-	explicit BufferPool(idx_t maximum_memory);
+	explicit BufferPool(idx_t maximum_memory, bool track_eviction_timestamps);
 	virtual ~BufferPool();
 
 	//! Set a new memory limit to the buffer pool, throws an exception if the new limit is too low and not enough
 	//! blocks can be evicted
 	void SetLimit(idx_t limit, const char *exception_postscript);
 
-	void IncreaseUsedMemory(MemoryTag tag, idx_t size);
+	void UpdateUsedMemory(MemoryTag tag, int64_t size);
 
 	idx_t GetUsedMemory() const;
 
@@ -72,6 +69,16 @@ protected:
 	virtual EvictionResult EvictBlocks(MemoryTag tag, idx_t extra_memory, idx_t memory_limit,
 	                                   unique_ptr<FileBuffer> *buffer = nullptr);
 
+	//! Purge all blocks that haven't been pinned within the last N seconds
+	idx_t PurgeAgedBlocks(uint32_t max_age_sec);
+
+	//! Iterate over all purgable blocks and invoke the callback. If the callback returns true
+	//! iteration continues.
+	//! - Callback signature is: bool((BufferEvictionNode &, const std::shared_ptr<BlockHandle> &)
+	//! - Callback is invoked while holding the corresponding BlockHandle mutex.
+	template <typename FN>
+	void IterateUnloadableBlocks(FN fn);
+
 	//! Tries to dequeue an element from the eviction queue, but only after acquiring the purge queue lock.
 	bool TryDequeueWithLock(BufferEvictionNode &node);
 	//! Bulk purge dead nodes from the eviction queue. Then, enqueue those that are still alive.
@@ -98,6 +105,8 @@ protected:
 	atomic<idx_t> current_memory;
 	//! The maximum amount of memory that the buffer manager can keep (in bytes)
 	atomic<idx_t> maximum_memory;
+	//! Record timestamps of buffer manager unpin() events. Usable by custom eviction policies.
+	bool track_eviction_timestamps;
 	//! Eviction queue
 	unique_ptr<EvictionQueue> queue;
 	//! Memory manager for concurrently used temporary memory, e.g., for physical operators

package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp

@@ -40,10 +40,17 @@ public:
 	virtual void ReAllocate(shared_ptr<BlockHandle> &handle, idx_t block_size) = 0;
 	virtual BufferHandle Pin(shared_ptr<BlockHandle> &handle) = 0;
 	virtual void Unpin(shared_ptr<BlockHandle> &handle) = 0;
+
 	//! Returns the currently allocated memory
 	virtual idx_t GetUsedMemory() const = 0;
 	//! Returns the maximum available memory
 	virtual idx_t GetMaxMemory() const = 0;
+	//! Returns the currently used swap space
+	virtual idx_t GetUsedSwap() = 0;
+	//! Returns the maximum swap space that can be used
+	virtual optional_idx GetMaxSwap() const = 0;
+
+	//! Returns a new block of memory that is smaller than Storage::BLOCK_SIZE
 	virtual shared_ptr<BlockHandle> RegisterSmallMemory(idx_t block_size);
 	virtual DUCKDB_API Allocator &GetBufferAllocator();
 	virtual DUCKDB_API void ReserveMemory(idx_t size);
@@ -51,19 +58,21 @@ public:
 	virtual vector<MemoryInformation> GetMemoryUsageInfo() const = 0;
 	//! Set a new memory limit to the buffer manager, throws an exception if the new limit is too low and not enough
 	//! blocks can be evicted
-	virtual void SetLimit(idx_t limit = (idx_t)-1);
+	virtual void SetMemoryLimit(idx_t limit = (idx_t)-1);
+	virtual void SetSwapLimit(optional_idx limit = optional_idx());
+
 	virtual vector<TemporaryFileInformation> GetTemporaryFiles();
 	virtual const string &GetTemporaryDirectory() const;
 	virtual void SetTemporaryDirectory(const string &new_dir);
 	virtual bool HasTemporaryDirectory() const;
+
 	//! Construct a managed buffer.
 	virtual unique_ptr<FileBuffer> ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source,
 	                                                      FileBufferType type = FileBufferType::MANAGED_BUFFER);
 	//! Get the underlying buffer pool responsible for managing the buffers
 	virtual BufferPool &GetBufferPool() const;
-	//! Get the manager that assigns reservations for temporary memory, e.g., for query intermediates
-	virtual TemporaryMemoryManager &GetTemporaryMemoryManager();
 
+	virtual DatabaseInstance &GetDatabase() = 0;
 	// Static methods
 	DUCKDB_API static BufferManager &GetBufferManager(DatabaseInstance &db);
 	DUCKDB_API static const BufferManager &GetBufferManager(const DatabaseInstance &db);
@@ -77,6 +86,9 @@ public:
 	//! Returns the maximum available memory for a given query
 	idx_t GetQueryMaxMemory() const;
 
+	//! Get the manager that assigns reservations for temporary memory, e.g., for query intermediates
+	virtual TemporaryMemoryManager &GetTemporaryMemoryManager();
+
 protected:
 	virtual void PurgeQueue() = 0;
 	virtual void AddToEvictionQueue(shared_ptr<BlockHandle> &handle);

package/src/duckdb/src/include/duckdb/storage/checkpoint/row_group_writer.hpp

@@ -32,6 +32,7 @@ public:
 
 	virtual void WriteColumnDataPointers(ColumnCheckpointState &column_checkpoint_state, Serializer &serializer) = 0;
 
+	virtual CheckpointType GetCheckpointType() const = 0;
 	virtual MetadataWriter &GetPayloadWriter() = 0;
 
 	PartialBlockManager &GetPartialBlockManager() {
@@ -47,18 +48,20 @@ protected:
 class SingleFileRowGroupWriter : public RowGroupWriter {
 public:
 	SingleFileRowGroupWriter(TableCatalogEntry &table, PartialBlockManager &partial_block_manager,
-	                         MetadataWriter &table_data_writer)
-	    : RowGroupWriter(table, partial_block_manager), table_data_writer(table_data_writer) {
-	}
-
-	//! MetadataWriter is a cursor on a given BlockManager. This returns the
-	//! cursor against which we should write payload data for the specified RowGroup.
-	MetadataWriter &table_data_writer;
+	                         TableDataWriter &writer, MetadataWriter &table_data_writer);
 
 public:
 	void WriteColumnDataPointers(ColumnCheckpointState &column_checkpoint_state, Serializer &serializer) override;
 
+	CheckpointType GetCheckpointType() const override;
 	MetadataWriter &GetPayloadWriter() override;
+
+private:
+	//! Underlying writer object
+	TableDataWriter &writer;
+	//! MetadataWriter is a cursor on a given BlockManager. This returns the
+	//! cursor against which we should write payload data for the specified RowGroup.
+	MetadataWriter &table_data_writer;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp

@@ -35,6 +35,7 @@ public:
 	virtual unique_ptr<RowGroupWriter> GetRowGroupWriter(RowGroup &row_group) = 0;
 
 	virtual void AddRowGroup(RowGroupPointer &&row_group_pointer, unique_ptr<RowGroupWriter> writer);
+	virtual CheckpointType GetCheckpointType() const = 0;
 
 	TaskScheduler &GetScheduler();
 
@@ -52,6 +53,7 @@ public:
 public:
 	void FinalizeTable(const TableStatistics &global_stats, DataTableInfo *info, Serializer &serializer) override;
 	unique_ptr<RowGroupWriter> GetRowGroupWriter(RowGroup &row_group) override;
+	CheckpointType GetCheckpointType() const override;
 
 private:
 	SingleFileCheckpointWriter &checkpoint_manager;