duckdb 0.7.2-dev1734.0 → 0.7.2-dev1867.0

package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp

@@ -29,8 +29,8 @@ public:
 
 	/* The double {{}} in the intializer for excluded_entries is intentional, workaround for bug in gcc-4.9 */
 	template <class T, class MATCHER>
-	static bool MatchRecursive(vector<unique_ptr<MATCHER>> &matchers, vector<T *> &entries, vector<T *> &bindings,
-	                           unordered_set<idx_t> excluded_entries, idx_t m_idx = 0) {
+	static bool MatchRecursive(vector<unique_ptr<MATCHER>> &matchers, vector<reference<T>> &entries,
+	                           vector<reference<T>> &bindings, unordered_set<idx_t> excluded_entries, idx_t m_idx = 0) {
 		if (m_idx == matchers.size()) {
 			// matched all matchers!
 			return true;
@@ -65,8 +65,8 @@ public:
 	}
 
 	template <class T, class MATCHER>
-	static bool Match(vector<unique_ptr<MATCHER>> &matchers, vector<T *> &entries, vector<T *> &bindings,
-	                  Policy policy) {
+	static bool Match(vector<unique_ptr<MATCHER>> &matchers, vector<reference<T>> &entries,
+	                  vector<reference<T>> &bindings, Policy policy) {
 		if (policy == Policy::ORDERED) {
 			// ordered policy, count has to match
 			if (matchers.size() != entries.size()) {
@@ -100,12 +100,12 @@ public:
 	}
 
 	template <class T, class MATCHER>
-	static bool Match(vector<unique_ptr<MATCHER>> &matchers, vector<unique_ptr<T>> &entries, vector<T *> &bindings,
-	                  Policy policy) {
+	static bool Match(vector<unique_ptr<MATCHER>> &matchers, vector<unique_ptr<T>> &entries,
+	                  vector<reference<T>> &bindings, Policy policy) {
 		// convert vector of unique_ptr to vector of normal pointers
-		vector<T *> ptr_entries;
+		vector<reference<T>> ptr_entries;
 		for (auto &entry : entries) {
-			ptr_entries.push_back(entry.get());
+			ptr_entries.push_back(*entry);
 		}
 		// then just call the normal match function
 		return Match(matchers, ptr_entries, bindings, policy);

package/src/duckdb/src/include/duckdb/optimizer/rule/arithmetic_simplification.hpp

@@ -18,7 +18,7 @@ class ArithmeticSimplificationRule : public Rule {
 public:
 	explicit ArithmeticSimplificationRule(ExpressionRewriter &rewriter);
 
-	unique_ptr<Expression> Apply(LogicalOperator &op, vector<Expression *> &bindings, bool &changes_made,
+	unique_ptr<Expression> Apply(LogicalOperator &op, vector<reference<Expression>> &bindings, bool &changes_made,
 	                             bool is_root) override;
 };
 

package/src/duckdb/src/include/duckdb/optimizer/rule/case_simplification.hpp

@@ -17,7 +17,7 @@ class CaseSimplificationRule : public Rule {
 public:
 	explicit CaseSimplificationRule(ExpressionRewriter &rewriter);
 
-	unique_ptr<Expression> Apply(LogicalOperator &op, vector<Expression *> &bindings, bool &changes_made,
+	unique_ptr<Expression> Apply(LogicalOperator &op, vector<reference<Expression>> &bindings, bool &changes_made,
 	                             bool is_root) override;
 };
 

package/src/duckdb/src/include/duckdb/optimizer/rule/comparison_simplification.hpp

@@ -17,7 +17,7 @@ class ComparisonSimplificationRule : public Rule {
 public:
 	explicit ComparisonSimplificationRule(ExpressionRewriter &rewriter);
 
-	unique_ptr<Expression> Apply(LogicalOperator &op, vector<Expression *> &bindings, bool &changes_made,
+	unique_ptr<Expression> Apply(LogicalOperator &op, vector<reference<Expression>> &bindings, bool &changes_made,
 	                             bool is_root) override;
 };
 

package/src/duckdb/src/include/duckdb/optimizer/rule/conjunction_simplification.hpp

@@ -17,10 +17,10 @@ class ConjunctionSimplificationRule : public Rule {
 public:
 	explicit ConjunctionSimplificationRule(ExpressionRewriter &rewriter);
 
-	unique_ptr<Expression> Apply(LogicalOperator &op, vector<Expression *> &bindings, bool &changes_made,
+	unique_ptr<Expression> Apply(LogicalOperator &op, vector<reference<Expression>> &bindings, bool &changes_made,
 	                             bool is_root) override;
 
-	unique_ptr<Expression> RemoveExpression(BoundConjunctionExpression &conj, Expression *expr);
+	unique_ptr<Expression> RemoveExpression(BoundConjunctionExpression &conj, const Expression &expr);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/optimizer/rule/constant_folding.hpp

@@ -17,7 +17,7 @@ class ConstantFoldingRule : public Rule {
 public:
 	explicit ConstantFoldingRule(ExpressionRewriter &rewriter);
 
-	unique_ptr<Expression> Apply(LogicalOperator &op, vector<Expression *> &bindings, bool &changes_made,
+	unique_ptr<Expression> Apply(LogicalOperator &op, vector<reference<Expression>> &bindings, bool &changes_made,
 	                             bool is_root) override;
 };
 

package/src/duckdb/src/include/duckdb/optimizer/rule/date_part_simplification.hpp

@@ -18,7 +18,7 @@ class DatePartSimplificationRule : public Rule {
 public:
 	explicit DatePartSimplificationRule(ExpressionRewriter &rewriter);
 
-	unique_ptr<Expression> Apply(LogicalOperator &op, vector<Expression *> &bindings, bool &changes_made,
+	unique_ptr<Expression> Apply(LogicalOperator &op, vector<reference<Expression>> &bindings, bool &changes_made,
 	                             bool is_root) override;
 };
 

package/src/duckdb/src/include/duckdb/optimizer/rule/distributivity.hpp

@@ -18,7 +18,7 @@ class DistributivityRule : public Rule {
 public:
 	explicit DistributivityRule(ExpressionRewriter &rewriter);
 
-	unique_ptr<Expression> Apply(LogicalOperator &op, vector<Expression *> &bindings, bool &changes_made,
+	unique_ptr<Expression> Apply(LogicalOperator &op, vector<reference<Expression>> &bindings, bool &changes_made,
 	                             bool is_root) override;
 
 private:

package/src/duckdb/src/include/duckdb/optimizer/rule/empty_needle_removal.hpp

@@ -18,7 +18,7 @@ class EmptyNeedleRemovalRule : public Rule {
 public:
 	explicit EmptyNeedleRemovalRule(ExpressionRewriter &rewriter);
 
-	unique_ptr<Expression> Apply(LogicalOperator &op, vector<Expression *> &bindings, bool &changes_made,
+	unique_ptr<Expression> Apply(LogicalOperator &op, vector<reference<Expression>> &bindings, bool &changes_made,
 	                             bool is_root) override;
 };
 

package/src/duckdb/src/include/duckdb/optimizer/rule/enum_comparison.hpp

@@ -17,7 +17,7 @@ class EnumComparisonRule : public Rule {
 public:
 	explicit EnumComparisonRule(ExpressionRewriter &rewriter);
 
-	unique_ptr<Expression> Apply(LogicalOperator &op, vector<Expression *> &bindings, bool &changes_made,
+	unique_ptr<Expression> Apply(LogicalOperator &op, vector<reference<Expression>> &bindings, bool &changes_made,
 	                             bool is_root) override;
 };
 

package/src/duckdb/src/include/duckdb/optimizer/rule/equal_or_null_simplification.hpp

@@ -18,7 +18,7 @@ class EqualOrNullSimplification : public Rule {
 public:
 	explicit EqualOrNullSimplification(ExpressionRewriter &rewriter);
 
-	unique_ptr<Expression> Apply(LogicalOperator &op, vector<Expression *> &bindings, bool &changes_made,
+	unique_ptr<Expression> Apply(LogicalOperator &op, vector<reference<Expression>> &bindings, bool &changes_made,
 	                             bool is_root) override;
 };
 

package/src/duckdb/src/include/duckdb/optimizer/rule/in_clause_simplification.hpp

@@ -17,7 +17,7 @@ class InClauseSimplificationRule : public Rule {
 public:
 	explicit InClauseSimplificationRule(ExpressionRewriter &rewriter);
 
-	unique_ptr<Expression> Apply(LogicalOperator &op, vector<Expression *> &bindings, bool &changes_made,
+	unique_ptr<Expression> Apply(LogicalOperator &op, vector<reference<Expression>> &bindings, bool &changes_made,
 	                             bool is_root) override;
 };
 

package/src/duckdb/src/include/duckdb/optimizer/rule/like_optimizations.hpp

@@ -18,7 +18,7 @@ class LikeOptimizationRule : public Rule {
 public:
 	explicit LikeOptimizationRule(ExpressionRewriter &rewriter);
 
-	unique_ptr<Expression> Apply(LogicalOperator &op, vector<Expression *> &bindings, bool &changes_made,
+	unique_ptr<Expression> Apply(LogicalOperator &op, vector<reference<Expression>> &bindings, bool &changes_made,
 	                             bool is_root) override;
 
 	unique_ptr<Expression> ApplyRule(BoundFunctionExpression &expr, ScalarFunction function, string pattern,

package/src/duckdb/src/include/duckdb/optimizer/rule/move_constants.hpp

@@ -18,7 +18,7 @@ class MoveConstantsRule : public Rule {
 public:
 	explicit MoveConstantsRule(ExpressionRewriter &rewriter);
 
-	unique_ptr<Expression> Apply(LogicalOperator &op, vector<Expression *> &bindings, bool &changes_made,
+	unique_ptr<Expression> Apply(LogicalOperator &op, vector<reference<Expression>> &bindings, bool &changes_made,
 	                             bool is_root) override;
 };
 

package/src/duckdb/src/include/duckdb/optimizer/rule/ordered_aggregate_optimizer.hpp

@@ -17,7 +17,7 @@ class OrderedAggregateOptimizer : public Rule {
 public:
 	explicit OrderedAggregateOptimizer(ExpressionRewriter &rewriter);
 
-	unique_ptr<Expression> Apply(LogicalOperator &op, vector<Expression *> &bindings, bool &changes_made,
+	unique_ptr<Expression> Apply(LogicalOperator &op, vector<reference<Expression>> &bindings, bool &changes_made,
 	                             bool is_root) override;
 };
 

package/src/duckdb/src/include/duckdb/optimizer/rule/regex_optimizations.hpp

@@ -17,7 +17,7 @@ class RegexOptimizationRule : public Rule {
 public:
 	explicit RegexOptimizationRule(ExpressionRewriter &rewriter);
 
-	unique_ptr<Expression> Apply(LogicalOperator &op, vector<Expression *> &bindings, bool &changes_made,
+	unique_ptr<Expression> Apply(LogicalOperator &op, vector<reference<Expression>> &bindings, bool &changes_made,
 	                             bool is_root) override;
 
 	unique_ptr<Expression> ApplyRule(BoundFunctionExpression *expr, ScalarFunction function, string pattern,

package/src/duckdb/src/include/duckdb/optimizer/rule.hpp

@@ -29,8 +29,8 @@ public:
 	unique_ptr<ExpressionMatcher> root;
 
 	ClientContext &GetContext() const;
-	virtual unique_ptr<Expression> Apply(LogicalOperator &op, vector<Expression *> &bindings, bool &fixed_point,
-	                                     bool is_root) = 0;
+	virtual unique_ptr<Expression> Apply(LogicalOperator &op, vector<reference<Expression>> &bindings,
+	                                     bool &fixed_point, bool is_root) = 0;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/parser/base_expression.hpp

@@ -73,7 +73,7 @@ public:
 	//! Returns true if this expression is equal to another expression
 	virtual bool Equals(const BaseExpression *other) const;
 
-	static bool Equals(BaseExpression *left, BaseExpression *right) {
+	static bool Equals(const BaseExpression *left, const BaseExpression *right) {
 		if (left == right) {
 			return true;
 		}

package/src/duckdb/src/include/duckdb/parser/expression_map.hpp

@@ -11,25 +11,38 @@
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/common/unordered_set.hpp"
 #include "duckdb/parser/base_expression.hpp"
+#include "duckdb/parser/parsed_expression.hpp"
+#include "duckdb/planner/expression.hpp"
 
 namespace duckdb {
 class Expression;
 
+template <class T>
 struct ExpressionHashFunction {
-	uint64_t operator()(const BaseExpression *const &expr) const {
-		return (uint64_t)expr->Hash();
+	uint64_t operator()(const reference<T> &expr) const {
+		return (uint64_t)expr.get().Hash();
 	}
 };
 
+template <class T>
 struct ExpressionEquality {
-	bool operator()(const BaseExpression *const &a, const BaseExpression *const &b) const {
-		return a->Equals(b);
+	bool operator()(const reference<T> &a, const reference<T> &b) const {
+		return a.get().Equals(&b.get());
 	}
 };
 
 template <typename T>
-using expression_map_t = unordered_map<BaseExpression *, T, ExpressionHashFunction, ExpressionEquality>;
+using expression_map_t =
+    unordered_map<reference<Expression>, T, ExpressionHashFunction<Expression>, ExpressionEquality<Expression>>;
 
-using expression_set_t = unordered_set<BaseExpression *, ExpressionHashFunction, ExpressionEquality>;
+using expression_set_t =
+    unordered_set<reference<Expression>, ExpressionHashFunction<Expression>, ExpressionEquality<Expression>>;
+
+template <typename T>
+using parsed_expression_map_t = unordered_map<reference<ParsedExpression>, T, ExpressionHashFunction<ParsedExpression>,
+                                              ExpressionEquality<ParsedExpression>>;
+
+using parsed_expression_set_t = unordered_set<reference<ParsedExpression>, ExpressionHashFunction<ParsedExpression>,
+                                              ExpressionEquality<ParsedExpression>>;
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/parser/expression_util.hpp

@@ -28,7 +28,7 @@ public:
 private:
 	template <class T>
 	static bool ExpressionListEquals(const vector<unique_ptr<T>> &a, const vector<unique_ptr<T>> &b);
-	template <class T>
+	template <class T, class EXPRESSION_MAP>
 	static bool ExpressionSetEquals(const vector<unique_ptr<T>> &a, const vector<unique_ptr<T>> &b);
 };
 

package/src/duckdb/src/include/duckdb/planner/expression.hpp

@@ -48,8 +48,11 @@ public:
 		return return_type == ((Expression *)other)->return_type;
 	}
 
-	static bool Equals(Expression *left, Expression *right) {
-		return BaseExpression::Equals((BaseExpression *)left, (BaseExpression *)right);
+	static bool Equals(const Expression *left, const Expression *right) {
+		return BaseExpression::Equals((const BaseExpression *)left, (const BaseExpression *)right);
+	}
+	static bool Equals(const Expression &left, const Expression &right) {
+		return left.Equals(&right);
 	}
 	//! Create a copy of this expression
 	virtual unique_ptr<Expression> Copy() = 0;

package/src/duckdb/src/include/duckdb/planner/expression_binder/base_select_binder.hpp

@@ -19,7 +19,7 @@ class WindowExpression;
 class BoundSelectNode;
 
 struct BoundGroupInformation {
-	expression_map_t<idx_t> map;
+	parsed_expression_map_t<idx_t> map;
 	case_insensitive_map_t<idx_t> alias_map;
 };
 

package/src/duckdb/src/include/duckdb/planner/expression_binder/order_binder.hpp

@@ -22,9 +22,9 @@ class SelectNode;
 class OrderBinder {
 public:
 	OrderBinder(vector<Binder *> binders, idx_t projection_index, case_insensitive_map_t<idx_t> &alias_map,
-	            expression_map_t<idx_t> &projection_map, idx_t max_count);
+	            parsed_expression_map_t<idx_t> &projection_map, idx_t max_count);
 	OrderBinder(vector<Binder *> binders, idx_t projection_index, SelectNode &node,
-	            case_insensitive_map_t<idx_t> &alias_map, expression_map_t<idx_t> &projection_map);
+	            case_insensitive_map_t<idx_t> &alias_map, parsed_expression_map_t<idx_t> &projection_map);
 
 public:
 	unique_ptr<Expression> Bind(unique_ptr<ParsedExpression> expr);
@@ -51,7 +51,7 @@ private:
 	idx_t max_count;
 	vector<unique_ptr<ParsedExpression>> *extra_list;
 	case_insensitive_map_t<idx_t> &alias_map;
-	expression_map_t<idx_t> &projection_map;
+	parsed_expression_map_t<idx_t> &projection_map;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/buffer/block_handle.hpp

@@ -24,9 +24,9 @@ enum class BlockState : uint8_t { BLOCK_UNLOADED = 0, BLOCK_LOADED = 1 };
 
 struct BufferPoolReservation {
 	idx_t size {0};
+	BufferPool &pool;
 
-	BufferPoolReservation() {
-	}
+	BufferPoolReservation(BufferPool &pool);
 	BufferPoolReservation(const BufferPoolReservation &) = delete;
 	BufferPoolReservation &operator=(const BufferPoolReservation &) = delete;
 
@@ -35,18 +35,17 @@ struct BufferPoolReservation {
 
 	~BufferPoolReservation();
 
-	void Resize(atomic<idx_t> &counter, idx_t new_size);
+	void Resize(idx_t new_size);
 	void Merge(BufferPoolReservation &&src);
 };
 
 struct TempBufferPoolReservation : BufferPoolReservation {
-	atomic<idx_t> &counter;
-	TempBufferPoolReservation(atomic<idx_t> &counter, idx_t size) : counter(counter) {
-		Resize(counter, size);
+	TempBufferPoolReservation(BufferPool &pool, idx_t size) : BufferPoolReservation(pool) {
+		Resize(size);
 	}
 	TempBufferPoolReservation(TempBufferPoolReservation &&) = default;
 	~TempBufferPoolReservation() {
-		Resize(counter, 0);
+		Resize(0);
 	}
 };
 
@@ -55,6 +54,7 @@ class BlockHandle {
 	friend struct BufferEvictionNode;
 	friend class BufferHandle;
 	friend class BufferManager;
+	friend class StandardBufferManager;
 	friend class BufferPool;
 
 public:
@@ -70,6 +70,14 @@ public:
 		return block_id;
 	}
 
+	void ResizeBuffer(idx_t block_size, int64_t memory_delta) {
+		D_ASSERT(buffer);
+		// resize and adjust current memory
+		buffer->Resize(block_size);
+		memory_usage += memory_delta;
+		D_ASSERT(memory_usage == buffer->AllocSize());
+	}
+
 	int32_t Readers() const {
 		return readers;
 	}

package/src/duckdb/src/include/duckdb/storage/buffer/buffer_pool.hpp

@@ -30,6 +30,7 @@ class BufferPool {
 	friend class BlockHandle;
 	friend class BlockManager;
 	friend class BufferManager;
+	friend class StandardBufferManager;
 
 public:
 	explicit BufferPool(idx_t maximum_memory);
@@ -39,6 +40,8 @@ public:
 	//! blocks can be evicted
 	void SetLimit(idx_t limit, const char *exception_postscript);
 
+	void IncreaseUsedMemory(idx_t size);
+
 	idx_t GetUsedMemory();
 
 	idx_t GetMaxMemory();

package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp

@@ -8,156 +8,79 @@
 
 #pragma once
 
-#include "duckdb/common/allocator.hpp"
-#include "duckdb/common/atomic.hpp"
-#include "duckdb/common/file_system.hpp"
-#include "duckdb/common/mutex.hpp"
-#include "duckdb/common/unordered_map.hpp"
-#include "duckdb/storage/block_manager.hpp"
-#include "duckdb/storage/buffer/block_handle.hpp"
+#include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/storage/buffer/buffer_handle.hpp"
-#include "duckdb/storage/buffer/buffer_pool.hpp"
+#include "duckdb/storage/block_manager.hpp"
+#include "duckdb/common/file_system.hpp"
 #include "duckdb/storage/buffer/temporary_file_information.hpp"
+#include "duckdb/main/config.hpp"
 
 namespace duckdb {
-class BlockManager;
-class DatabaseInstance;
-class TemporaryDirectoryHandle;
-struct EvictionQueue;
 
-//! The BufferManager is in charge of handling memory management for a single database. It cooperatively shares a
-//! BufferPool with other BufferManagers, belonging to different databases. It hands out memory buffers that can
-//! be used by the database internally, and offers configuration options specific to a database, which need not be
-//! shared by the BufferPool, including whether to support swapping temp buffers to disk, and where to swap them to.
+class Allocator;
+class BufferPool;
+
 class BufferManager {
 	friend class BufferHandle;
 	friend class BlockHandle;
 	friend class BlockManager;
 
 public:
-	BufferManager(DatabaseInstance &db, string temp_directory);
-	virtual ~BufferManager();
-
-	//! Registers an in-memory buffer that cannot be unloaded until it is destroyed
-	//! This buffer can be small (smaller than BLOCK_SIZE)
-	//! Unpin and pin are nops on this block of memory
-	shared_ptr<BlockHandle> RegisterSmallMemory(idx_t block_size);
-
-	//! Allocate an in-memory buffer with a single pin.
-	//! The allocated memory is released when the buffer handle is destroyed.
-	DUCKDB_API BufferHandle Allocate(idx_t block_size, bool can_destroy = true,
-	                                 shared_ptr<BlockHandle> *block = nullptr);
+	BufferManager() {
+	}
+	virtual ~BufferManager() {
+	}
 
+public:
+	static unique_ptr<BufferManager> CreateStandardBufferManager(DatabaseInstance &db, DBConfig &config);
+	virtual BufferHandle Allocate(idx_t block_size, bool can_destroy = true,
+	                              shared_ptr<BlockHandle> *block = nullptr) = 0;
 	//! Reallocate an in-memory buffer that is pinned.
-	void ReAllocate(shared_ptr<BlockHandle> &handle, idx_t block_size);
-
-	BufferHandle Pin(shared_ptr<BlockHandle> &handle);
-	void Unpin(shared_ptr<BlockHandle> &handle);
-
-	DUCKDB_API static BufferManager &GetBufferManager(ClientContext &context);
-	DUCKDB_API static BufferManager &GetBufferManager(DatabaseInstance &db);
-	DUCKDB_API static BufferManager &GetBufferManager(AttachedDatabase &db);
-
+	virtual void ReAllocate(shared_ptr<BlockHandle> &handle, idx_t block_size) = 0;
+	virtual BufferHandle Pin(shared_ptr<BlockHandle> &handle) = 0;
+	virtual void Unpin(shared_ptr<BlockHandle> &handle) = 0;
 	//! Returns the currently allocated memory
-	idx_t GetUsedMemory() {
-		return buffer_pool.current_memory;
-	}
+	virtual idx_t GetUsedMemory() const = 0;
 	//! Returns the maximum available memory
-	idx_t GetMaxMemory() {
-		return buffer_pool.maximum_memory;
-	}
-
+	virtual idx_t GetMaxMemory() const = 0;
+	virtual shared_ptr<BlockHandle> RegisterSmallMemory(idx_t block_size);
+	virtual DUCKDB_API Allocator &GetBufferAllocator();
+	virtual DUCKDB_API void ReserveMemory(idx_t size);
+	virtual DUCKDB_API void FreeReservedMemory(idx_t size);
+	//! Set a new memory limit to the buffer manager, throws an exception if the new limit is too low and not enough
+	//! blocks can be evicted
+	virtual void SetLimit(idx_t limit = (idx_t)-1);
+	virtual vector<TemporaryFileInformation> GetTemporaryFiles();
+	virtual const string &GetTemporaryDirectory();
+	virtual void SetTemporaryDirectory(const string &new_dir);
+	virtual DatabaseInstance &GetDatabase();
+	virtual bool HasTemporaryDirectory() const;
+	//! Construct a managed buffer.
+	virtual unique_ptr<FileBuffer> ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source,
+	                                                      FileBufferType type = FileBufferType::MANAGED_BUFFER);
 	//! Increases the currently allocated memory, but the actual allocation does not go through the buffer manager
-	void IncreaseUsedMemory(idx_t size);
+	virtual void IncreaseUsedMemory(idx_t size, bool unsafe = false) = 0;
 	//! Decrease the currently allocated memory, but the actual deallocation does not go through the buffer manager
-	void DecreaseUsedMemory(idx_t size);
-
-	const string &GetTemporaryDirectory() {
-		return temp_directory;
-	}
-
-	void SetTemporaryDirectory(string new_dir);
-
-	DUCKDB_API Allocator &GetBufferAllocator();
+	virtual void DecreaseUsedMemory(idx_t size) = 0;
+	//! Get the underlying buffer pool responsible for managing the buffers
+	virtual BufferPool &GetBufferPool();
 
-	DatabaseInstance &GetDatabase() {
-		return db;
-	}
+	// Static methods
 
-	BufferPool &GetBufferPool() {
-		return buffer_pool;
-	}
+	DUCKDB_API static BufferManager &GetBufferManager(DatabaseInstance &db);
+	DUCKDB_API static BufferManager &GetBufferManager(ClientContext &context);
+	DUCKDB_API static BufferManager &GetBufferManager(AttachedDatabase &db);
 
 	static idx_t GetAllocSize(idx_t block_size) {
 		return AlignValue<idx_t, Storage::SECTOR_SIZE>(block_size + Storage::BLOCK_HEADER_SIZE);
 	}
 
-	//! Construct a managed buffer.
-	unique_ptr<FileBuffer> ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source,
-	                                              FileBufferType type = FileBufferType::MANAGED_BUFFER);
-
-	DUCKDB_API void ReserveMemory(idx_t size);
-	DUCKDB_API void FreeReservedMemory(idx_t size);
-
-	//! Set a new memory limit to the buffer pool, throws an exception if the new limit is too low and not enough
-	//! blocks can be evicted. (Sugar for calling method directly on the BufferPool.)
-	void SetLimit(idx_t limit = (idx_t)-1) {
-		buffer_pool.SetLimit(limit, InMemoryWarning());
-	}
-	//! Returns a list of all temporary files
-	vector<TemporaryFileInformation> GetTemporaryFiles();
-
-private:
-	//! Register an in-memory buffer of arbitrary size, as long as it is >= BLOCK_SIZE. can_destroy signifies whether or
-	//! not the buffer can be destroyed when unpinned, or whether or not it needs to be written to a temporary file so
-	//! it can be reloaded. The resulting buffer will already be allocated, but needs to be pinned in order to be used.
-	//! This needs to be private to prevent creating blocks without ever pinning them:
-	//! blocks that are never pinned are never added to the eviction queue
-	shared_ptr<BlockHandle> RegisterMemory(idx_t block_size, bool can_destroy);
-
-	//! Write a temporary buffer to disk
-	void WriteTemporaryBuffer(block_id_t block_id, FileBuffer &buffer);
-	//! Read a temporary buffer from disk
-	unique_ptr<FileBuffer> ReadTemporaryBuffer(block_id_t id, unique_ptr<FileBuffer> buffer = nullptr);
-	//! Get the path of the temporary buffer
-	string GetTemporaryPath(block_id_t id);
-
-	void DeleteTemporaryFile(block_id_t id);
-
-	void RequireTemporaryDirectory();
-
-	const char *InMemoryWarning();
-
-	static data_ptr_t BufferAllocatorAllocate(PrivateAllocatorData *private_data, idx_t size);
-	static void BufferAllocatorFree(PrivateAllocatorData *private_data, data_ptr_t pointer, idx_t size);
-	static data_ptr_t BufferAllocatorRealloc(PrivateAllocatorData *private_data, data_ptr_t pointer, idx_t old_size,
-	                                         idx_t size);
-
-	//! When the BlockHandle reaches 0 readers, this creates a new FileBuffer for this BlockHandle and
-	//! overwrites the data within with garbage. Any readers that do not hold the pin will notice
-	void VerifyZeroReaders(shared_ptr<BlockHandle> &handle);
-
-	//! Helper
-	template <typename... ARGS>
-	TempBufferPoolReservation EvictBlocksOrThrow(idx_t extra_memory, unique_ptr<FileBuffer> *buffer, ARGS...);
-
-private:
-	//! The database instance
-	DatabaseInstance &db;
-	//! The buffer pool
-	BufferPool &buffer_pool;
-	//! The directory name where temporary files are stored
-	string temp_directory;
-	//! Lock for creating the temp handle
-	mutex temp_handle_lock;
-	//! Handle for the temporary directory
-	unique_ptr<TemporaryDirectoryHandle> temp_directory_handle;
-	//! The temporary id used for managed buffers
-	atomic<block_id_t> temporary_id;
-	//! Allocator associated with the buffer manager, that passes all allocations through this buffer manager
-	Allocator buffer_allocator;
-	//! Block manager for temp data
-	unique_ptr<BlockManager> temp_block_manager;
+protected:
+	virtual void PurgeQueue() = 0;
+	virtual void AddToEvictionQueue(shared_ptr<BlockHandle> &handle);
+	virtual void WriteTemporaryBuffer(block_id_t block_id, FileBuffer &buffer);
+	virtual unique_ptr<FileBuffer> ReadTemporaryBuffer(block_id_t id, unique_ptr<FileBuffer> buffer);
+	virtual void DeleteTemporaryFile(block_id_t id);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/compression/chimp/algorithm/chimp128.hpp

@@ -17,6 +17,7 @@
 #include "duckdb/common/likely.hpp"
 #include "duckdb/storage/compression/chimp/algorithm/packed_data.hpp"
 #include "duckdb/common/limits.hpp"
+#include "duckdb/common/bit_utils.hpp"
 
 #include "duckdb/storage/compression/chimp/algorithm/bit_reader.hpp"
 #include "duckdb/storage/compression/chimp/algorithm/output_bit_stream.hpp"