duckdb 0.6.2-dev971.0 → 0.7.1-dev100.0

package/src/duckdb/src/include/duckdb/common/union_by_name.hpp

@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/union_by_name.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+#include <vector>
+#include <string>
+#include "duckdb/common/types.hpp"
+
+using std::string;
+using std::vector;
+
+namespace duckdb {
+
+template <class READER_TYPE, class OPTION_TYPE>
+class UnionByName {
+
+public:
+	//! Union all files(readers) by their col names
+	static vector<unique_ptr<READER_TYPE>>
+	UnionCols(ClientContext &context, const vector<string> &files, vector<LogicalType> &union_col_types,
+	          vector<string> &union_col_names, case_insensitive_map_t<idx_t> &union_names_map, OPTION_TYPE options) {
+		idx_t union_names_index = 0;
+		vector<unique_ptr<READER_TYPE>> union_readers;
+
+		for (idx_t file_idx = 0; file_idx < files.size(); ++file_idx) {
+			const auto file_name = files[file_idx];
+			auto reader = make_unique<READER_TYPE>(context, file_name, options);
+
+			auto &col_names = reader->names;
+			auto &sql_types = reader->return_types;
+			D_ASSERT(col_names.size() == sql_types.size());
+
+			for (idx_t col = 0; col < col_names.size(); ++col) {
+				auto union_find = union_names_map.find(col_names[col]);
+
+				if (union_find != union_names_map.end()) {
+					// given same name , union_col's type must compatible with col's type
+					LogicalType compatible_type;
+					compatible_type = LogicalType::MaxLogicalType(union_col_types[union_find->second], sql_types[col]);
+					union_col_types[union_find->second] = compatible_type;
+				} else {
+					union_names_map[col_names[col]] = union_names_index;
+					union_names_index++;
+
+					union_col_names.emplace_back(col_names[col]);
+					union_col_types.emplace_back(sql_types[col]);
+				}
+			}
+			union_readers.push_back(std::move(reader));
+		}
+		return union_readers;
+	}
+
+	//! Create information for reader's col mapping to union cols
+	static vector<unique_ptr<READER_TYPE>> CreateUnionMap(vector<unique_ptr<READER_TYPE>> union_readers,
+	                                                      vector<LogicalType> &union_col_types,
+	                                                      vector<string> &union_col_names,
+	                                                      case_insensitive_map_t<idx_t> &union_names_map) {
+		for (auto &reader : union_readers) {
+			auto &col_names = reader->names;
+			vector<bool> union_null_cols(union_col_names.size(), true);
+			vector<idx_t> union_idx_map(col_names.size(), 0);
+
+			for (idx_t col = 0; col < col_names.size(); ++col) {
+				idx_t union_idx = union_names_map[col_names[col]];
+				union_idx_map[col] = union_idx;
+				union_null_cols[union_idx] = false;
+			}
+
+			reader->union_col_types = union_col_types;
+			reader->union_idx_map = std::move(union_idx_map);
+			reader->union_null_cols = std::move(union_null_cols);
+		}
+		return union_readers;
+	}
+
+	//! Set nulls into the cols that mismtach union names
+	static void SetNullUnionCols(DataChunk &result, const vector<bool> &union_null_cols) {
+		for (idx_t col = 0; col < union_null_cols.size(); ++col) {
+			if (union_null_cols[col]) {
+				result.data[col].SetVectorType(VectorType::CONSTANT_VECTOR);
+				ConstantVector::SetNull(result.data[col], true);
+			}
+		}
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/vector_operations/generic_executor.hpp

@@ -164,6 +164,56 @@ struct StructTypeTernary {
 	}
 };
 
+template <class A_TYPE, class B_TYPE, class C_TYPE, class D_TYPE>
+struct StructTypeQuaternary {
+	A_TYPE a_val;
+	B_TYPE b_val;
+	C_TYPE c_val;
+	D_TYPE d_val;
+
+	using STRUCT_STATE = StructTypeState<4>;
+
+	static bool ConstructType(STRUCT_STATE &state, idx_t i,
+	                          StructTypeQuaternary<A_TYPE, B_TYPE, C_TYPE, D_TYPE> &result) {
+		auto &a_data = state.child_data[0];
+		auto &b_data = state.child_data[1];
+		auto &c_data = state.child_data[2];
+		auto &d_data = state.child_data[3];
+
+		auto a_idx = a_data.sel->get_index(i);
+		auto b_idx = b_data.sel->get_index(i);
+		auto c_idx = c_data.sel->get_index(i);
+		auto d_idx = d_data.sel->get_index(i);
+		if (!a_data.validity.RowIsValid(a_idx) || !b_data.validity.RowIsValid(b_idx) ||
+		    !c_data.validity.RowIsValid(c_idx) || !d_data.validity.RowIsValid(d_idx)) {
+			return false;
+		}
+		auto a_ptr = (A_TYPE *)a_data.data;
+		auto b_ptr = (B_TYPE *)b_data.data;
+		auto c_ptr = (C_TYPE *)c_data.data;
+		auto d_ptr = (D_TYPE *)d_data.data;
+		result.a_val = a_ptr[a_idx];
+		result.b_val = b_ptr[b_idx];
+		result.c_val = c_ptr[c_idx];
+		result.d_val = d_ptr[d_idx];
+		return true;
+	}
+
+	static void AssignResult(Vector &result, idx_t i, StructTypeQuaternary<A_TYPE, B_TYPE, C_TYPE, D_TYPE> value) {
+		auto &entries = StructVector::GetEntries(result);
+
+		auto a_data = FlatVector::GetData<A_TYPE>(*entries[0]);
+		auto b_data = FlatVector::GetData<B_TYPE>(*entries[1]);
+		auto c_data = FlatVector::GetData<C_TYPE>(*entries[2]);
+		auto d_data = FlatVector::GetData<D_TYPE>(*entries[3]);
+
+		a_data[i] = value.a_val;
+		b_data[i] = value.b_val;
+		c_data[i] = value.c_val;
+		d_data[i] = value.d_val;
+	}
+};
+
 //! The GenericExecutor can handle struct types in addition to primitive types
 struct GenericExecutor {
 private:
@@ -222,6 +272,87 @@ private:
 		}
 	}
 
+	template <class A_TYPE, class B_TYPE, class C_TYPE, class RESULT_TYPE, class FUNC>
+	static void ExecuteTernaryInternal(Vector &a, Vector &b, Vector &c, Vector &result, idx_t count, FUNC &fun) {
+		auto constant = a.GetVectorType() == VectorType::CONSTANT_VECTOR &&
+		                b.GetVectorType() == VectorType::CONSTANT_VECTOR &&
+		                c.GetVectorType() == VectorType::CONSTANT_VECTOR;
+
+		typename A_TYPE::STRUCT_STATE a_state;
+		typename B_TYPE::STRUCT_STATE b_state;
+		typename C_TYPE::STRUCT_STATE c_state;
+
+		a_state.PrepareVector(a, count);
+		b_state.PrepareVector(b, count);
+		c_state.PrepareVector(c, count);
+
+		for (idx_t i = 0; i < (constant ? 1 : count); i++) {
+			auto a_idx = a_state.main_data.sel->get_index(i);
+			auto b_idx = a_state.main_data.sel->get_index(i);
+			auto c_idx = a_state.main_data.sel->get_index(i);
+			if (!a_state.main_data.validity.RowIsValid(a_idx) || !b_state.main_data.validity.RowIsValid(b_idx) ||
+			    !c_state.main_data.validity.RowIsValid(c_idx)) {
+				FlatVector::SetNull(result, i, true);
+				continue;
+			}
+			A_TYPE a_val;
+			B_TYPE b_val;
+			C_TYPE c_val;
+			if (!A_TYPE::ConstructType(a_state, i, a_val) || !B_TYPE::ConstructType(b_state, i, b_val) ||
+			    !C_TYPE::ConstructType(c_state, i, c_val)) {
+				FlatVector::SetNull(result, i, true);
+				continue;
+			}
+			RESULT_TYPE::AssignResult(result, i, fun(a_val, b_val, c_val));
+		}
+		if (constant) {
+			result.SetVectorType(VectorType::CONSTANT_VECTOR);
+		}
+	}
+
+	template <class A_TYPE, class B_TYPE, class C_TYPE, class D_TYPE, class RESULT_TYPE, class FUNC>
+	static void ExecuteQuaternaryInternal(Vector &a, Vector &b, Vector &c, Vector &d, Vector &result, idx_t count,
+	                                      FUNC &fun) {
+		auto constant =
+		    a.GetVectorType() == VectorType::CONSTANT_VECTOR && b.GetVectorType() == VectorType::CONSTANT_VECTOR &&
+		    c.GetVectorType() == VectorType::CONSTANT_VECTOR && d.GetVectorType() == VectorType::CONSTANT_VECTOR;
+
+		typename A_TYPE::STRUCT_STATE a_state;
+		typename B_TYPE::STRUCT_STATE b_state;
+		typename C_TYPE::STRUCT_STATE c_state;
+		typename D_TYPE::STRUCT_STATE d_state;
+
+		a_state.PrepareVector(a, count);
+		b_state.PrepareVector(b, count);
+		c_state.PrepareVector(c, count);
+		d_state.PrepareVector(d, count);
+
+		for (idx_t i = 0; i < (constant ? 1 : count); i++) {
+			auto a_idx = a_state.main_data.sel->get_index(i);
+			auto b_idx = a_state.main_data.sel->get_index(i);
+			auto c_idx = a_state.main_data.sel->get_index(i);
+			auto d_idx = a_state.main_data.sel->get_index(i);
+			if (!a_state.main_data.validity.RowIsValid(a_idx) || !b_state.main_data.validity.RowIsValid(b_idx) ||
+			    !c_state.main_data.validity.RowIsValid(c_idx) || !d_state.main_data.validity.RowIsValid(d_idx)) {
+				FlatVector::SetNull(result, i, true);
+				continue;
+			}
+			A_TYPE a_val;
+			B_TYPE b_val;
+			C_TYPE c_val;
+			D_TYPE d_val;
+			if (!A_TYPE::ConstructType(a_state, i, a_val) || !B_TYPE::ConstructType(b_state, i, b_val) ||
+			    !C_TYPE::ConstructType(c_state, i, c_val) || !D_TYPE::ConstructType(d_state, i, d_val)) {
+				FlatVector::SetNull(result, i, true);
+				continue;
+			}
+			RESULT_TYPE::AssignResult(result, i, fun(a_val, b_val, c_val, d_val));
+		}
+		if (constant) {
+			result.SetVectorType(VectorType::CONSTANT_VECTOR);
+		}
+	}
+
 public:
 	template <class A_TYPE, class RESULT_TYPE, class FUNC = std::function<RESULT_TYPE(A_TYPE)>>
 	static void ExecuteUnary(Vector &input, Vector &result, idx_t count, FUNC fun) {
@@ -231,6 +362,16 @@ public:
 	static void ExecuteBinary(Vector &a, Vector &b, Vector &result, idx_t count, FUNC fun) {
 		ExecuteBinaryInternal<A_TYPE, B_TYPE, RESULT_TYPE, FUNC>(a, b, result, count, fun);
 	}
+	template <class A_TYPE, class B_TYPE, class C_TYPE, class RESULT_TYPE,
+	          class FUNC = std::function<RESULT_TYPE(A_TYPE)>>
+	static void ExecuteTernary(Vector &a, Vector &b, Vector &c, Vector &result, idx_t count, FUNC fun) {
+		ExecuteTernaryInternal<A_TYPE, B_TYPE, C_TYPE, RESULT_TYPE, FUNC>(a, b, c, result, count, fun);
+	}
+	template <class A_TYPE, class B_TYPE, class C_TYPE, class D_TYPE, class RESULT_TYPE,
+	          class FUNC = std::function<RESULT_TYPE(A_TYPE)>>
+	static void ExecuteQuaternary(Vector &a, Vector &b, Vector &c, Vector &d, Vector &result, idx_t count, FUNC fun) {
+		ExecuteQuaternaryInternal<A_TYPE, B_TYPE, C_TYPE, D_TYPE, RESULT_TYPE, FUNC>(a, b, c, d, result, count, fun);
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/vector_operations/ternary_executor.hpp

@@ -16,6 +16,14 @@
 
 namespace duckdb {
 
+template <class OP>
+struct TernaryStandardOperatorWrapper {
+	template <class FUN, class A_TYPE, class B_TYPE, class C_TYPE, class RESULT_TYPE>
+	static inline RESULT_TYPE Operation(FUN fun, A_TYPE a, B_TYPE b, C_TYPE c, ValidityMask &mask, idx_t idx) {
+		return OP::template Operation<A_TYPE, B_TYPE, C_TYPE, RESULT_TYPE>(a, b, c);
+	}
+};
+
 struct TernaryLambdaWrapper {
 	template <class FUN, class A_TYPE, class B_TYPE, class C_TYPE, class RESULT_TYPE>
 	static inline RESULT_TYPE Operation(FUN fun, A_TYPE a, B_TYPE b, C_TYPE c, ValidityMask &mask, idx_t idx) {
@@ -99,6 +107,12 @@ public:
 		ExecuteGeneric<A_TYPE, B_TYPE, C_TYPE, RESULT_TYPE, TernaryLambdaWrapper, FUN>(a, b, c, result, count, fun);
 	}
 
+	template <class A_TYPE, class B_TYPE, class C_TYPE, class RESULT_TYPE, class OP>
+	static void ExecuteStandard(Vector &a, Vector &b, Vector &c, Vector &result, idx_t count) {
+		ExecuteGeneric<A_TYPE, B_TYPE, C_TYPE, RESULT_TYPE, TernaryStandardOperatorWrapper<OP>, bool>(a, b, c, result,
+		                                                                                              count, false);
+	}
+
 	template <class A_TYPE, class B_TYPE, class C_TYPE, class RESULT_TYPE,
 	          class FUN = std::function<RESULT_TYPE(A_TYPE, B_TYPE, C_TYPE, ValidityMask &, idx_t)>>
 	static void ExecuteWithNulls(Vector &a, Vector &b, Vector &c, Vector &result, idx_t count, FUN fun) {

package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp

@@ -68,8 +68,9 @@ public:
 		FindFileSystem(directory)->RemoveDirectory(directory);
 	}
 
-	bool ListFiles(const string &directory, const std::function<void(const string &, bool)> &callback) override {
-		return FindFileSystem(directory)->ListFiles(directory, callback);
+	bool ListFiles(const string &directory, const std::function<void(const string &, bool)> &callback,
+	               FileOpener *opener = nullptr) override {
+		return FindFileSystem(directory)->ListFiles(directory, callback, opener);
 	}
 
 	void MoveFile(const string &source, const string &target) override {
@@ -92,11 +93,29 @@ public:
 	}
 
 	void RegisterSubSystem(unique_ptr<FileSystem> fs) override {
-		sub_systems.push_back(move(fs));
+		sub_systems.push_back(std::move(fs));
+	}
+
+	void UnregisterSubSystem(const string &name) override {
+		for (auto sub_system = sub_systems.begin(); sub_system != sub_systems.end(); sub_system++) {
+			if (sub_system->get()->GetName() == name) {
+				sub_systems.erase(sub_system);
+				return;
+			}
+		}
+		throw InvalidInputException("Could not find filesystem with name %s", name);
 	}
 
 	void RegisterSubSystem(FileCompressionType compression_type, unique_ptr<FileSystem> fs) override {
-		compressed_fs[compression_type] = move(fs);
+		compressed_fs[compression_type] = std::move(fs);
+	}
+
+	vector<string> ListSubSystems() override {
+		vector<string> names(sub_systems.size());
+		for (idx_t i = 0; i < sub_systems.size(); i++) {
+			names[i] = sub_systems[i]->GetName();
+		}
+		return names;
 	}
 
 	std::string GetName() const override {

package/src/duckdb/src/include/duckdb/execution/executor.hpp

@@ -89,6 +89,9 @@ public:
 	//! Returns the query result - can only be used if `HasResultCollector` returns true
 	unique_ptr<QueryResult> GetResult();
 
+	//! Returns true if all pipelines have been completed
+	bool ExecutionIsFinished();
+
 private:
 	void InitializeInternal(PhysicalOperator *physical_plan);
 

package/src/duckdb/src/include/duckdb/execution/expression_executor_state.hpp

@@ -37,7 +37,7 @@ public:
 	void Finalize();
 	Allocator &GetAllocator();
 	bool HasContext();
-	ClientContext &GetContext();
+	DUCKDB_API ClientContext &GetContext();
 
 	void Verify(ExpressionExecutorState &root);
 };
@@ -49,7 +49,7 @@ struct ExecuteFunctionState : public ExpressionState {
 	unique_ptr<FunctionLocalState> local_state;
 
 public:
-	static FunctionLocalState *GetFunctionState(ExpressionState &state) {
+	DUCKDB_API static FunctionLocalState *GetFunctionState(ExpressionState &state) {
 		return ((ExecuteFunctionState &)state).local_state.get();
 	}
 };

package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp

@@ -26,6 +26,8 @@
 
 namespace duckdb {
 
+class ConflictManager;
+
 struct ARTIndexScanState : public IndexScanState {
 	ARTIndexScanState() : checked(false), result_index(0) {
 	}
@@ -41,7 +43,7 @@ struct ARTIndexScanState : public IndexScanState {
 	idx_t result_index = 0;
 };
 
-enum VerifyExistenceType : uint8_t {
+enum class VerifyExistenceType : uint8_t {
 	APPEND = 0,    // for purpose to append into table
 	APPEND_FK = 1, // for purpose to append into table has foreign key
 	DELETE_FK = 2  // for purpose to delete from table related to foreign key
@@ -49,27 +51,26 @@ enum VerifyExistenceType : uint8_t {
 
 class ART : public Index {
 public:
+	//! Constructs an ART containing the bound expressions, which are resolved during index construction
 	ART(const vector<column_t> &column_ids, TableIOManager &table_io_manager,
 	    const vector<unique_ptr<Expression>> &unbound_expressions, IndexConstraintType constraint_type,
-	    AttachedDatabase &db, idx_t block_id = DConstants::INVALID_INDEX,
+	    AttachedDatabase &db, bool track_memory, idx_t block_id = DConstants::INVALID_INDEX,
 	    idx_t block_offset = DConstants::INVALID_INDEX);
 	~ART() override;
 
 	//! Root of the tree
 	Node *tree;
 
-	AttachedDatabase &db;
-
 public:
 	//! Initialize a scan on the index with the given expression and column ids
 	//! to fetch from the base table for a single predicate
-	unique_ptr<IndexScanState> InitializeScanSinglePredicate(Transaction &transaction, Value value,
+	unique_ptr<IndexScanState> InitializeScanSinglePredicate(const Transaction &transaction, const Value &value,
 	                                                         ExpressionType expression_type) override;
 
 	//! Initialize a scan on the index with the given expression and column ids
 	//! to fetch from the base table for two predicates
-	unique_ptr<IndexScanState> InitializeScanTwoPredicates(Transaction &transaction, Value low_value,
-	                                                       ExpressionType low_expression_type, Value high_value,
+	unique_ptr<IndexScanState> InitializeScanTwoPredicates(Transaction &transaction, const Value &low_value,
+	                                                       ExpressionType low_expression_type, const Value &high_value,
 	                                                       ExpressionType high_expression_type) override;
 
 	//! Perform a lookup on the index
@@ -79,17 +80,19 @@ public:
 	bool Append(IndexLock &lock, DataChunk &entries, Vector &row_identifiers) override;
 	//! Verify that data can be appended to the index
 	void VerifyAppend(DataChunk &chunk) override;
+	//! Verify that data can be appended to the index
+	void VerifyAppend(DataChunk &chunk, ConflictManager &conflict_manager) override;
 	//! Verify that data can be appended to the index for foreign key constraint
-	void VerifyAppendForeignKey(DataChunk &chunk, string *err_msg_ptr) override;
+	void VerifyAppendForeignKey(DataChunk &chunk) override;
 	//! Verify that data can be delete from the index for foreign key constraint
-	void VerifyDeleteForeignKey(DataChunk &chunk, string *err_msg_ptr) override;
+	void VerifyDeleteForeignKey(DataChunk &chunk) override;
 	//! Delete entries in the index
 	void Delete(IndexLock &lock, DataChunk &entries, Vector &row_identifiers) override;
-	//! Insert data into the index.
+	//! Insert data into the index
 	bool Insert(IndexLock &lock, DataChunk &data, Vector &row_ids) override;
 
-	//! Construct ARTs from sorted chunks and merge them.
-	void ConstructAndMerge(IndexLock &lock, PayloadScanner &scanner, Allocator &allocator) override;
+	//! Construct an ART from a vector of sorted keys
+	bool ConstructFromSorted(idx_t count, vector<Key> &keys, Vector &row_identifiers);
 
 	//! Search Equal and fetches the row IDs
 	bool SearchEqual(Key &key, idx_t max_count, vector<row_t> &result_ids);
@@ -102,8 +105,16 @@ public:
 	bool MergeIndexes(IndexLock &state, Index *other_index) override;
 	//! Generate ART keys for an input chunk
 	static void GenerateKeys(ArenaAllocator &allocator, DataChunk &input, vector<Key> &keys);
+
+	//! Generate a string containing all the expressions and their respective values that violate a constraint
+	string GenerateErrorKeyName(DataChunk &input, idx_t row);
+	//! Generate the matching error message for a constraint violation
+	string GenerateConstraintErrorMessage(VerifyExistenceType verify_type, const string &key_name);
+
 	//! Returns the string representation of an ART
 	string ToString() override;
+	//! Verifies that the memory_size value of the ART matches its actual size
+	void Verify() override;
 
 private:
 	//! Insert a row id into a leaf node
@@ -114,6 +125,9 @@ private:
 	//! Erase element from leaf (if leaf has more than one value) or eliminate the leaf itself
 	void Erase(Node *&node, Key &key, idx_t depth, row_t row_id);
 
+	//! Perform 'Lookup' for an entire chunk, marking which succeeded
+	void LookupValues(DataChunk &input, ConflictManager &conflict_manager) final override;
+
 	//! Find the node with a matching key, optimistic version
 	Leaf *Lookup(Node *node, Key &key, idx_t depth);
 
@@ -122,14 +136,6 @@ private:
 	                vector<row_t> &result_ids);
 	bool SearchCloseRange(ARTIndexScanState *state, Key &lower_bound, Key &upper_bound, bool left_inclusive,
 	                      bool right_inclusive, idx_t max_count, vector<row_t> &result_ids);
-
-	void VerifyExistence(DataChunk &chunk, VerifyExistenceType verify_type, string *err_msg_ptr = nullptr);
-
-private:
-	//! The estimated ART memory consumption
-	idx_t estimated_art_size;
-	//! The estimated memory consumption of a single key
-	idx_t estimated_key_size;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp

@@ -15,36 +15,45 @@ namespace duckdb {
 
 class Leaf : public Node {
 public:
+	explicit Leaf();
 	Leaf(Key &value, uint32_t depth, row_t row_id);
 	Leaf(Key &value, uint32_t depth, row_t *row_ids, idx_t num_elements);
 	Leaf(row_t *row_ids, idx_t num_elements, Prefix &prefix);
 	Leaf(row_t row_id, Prefix &prefix);
 	~Leaf();
 
-	row_t GetRowId(idx_t index);
+	//! Get the row ID at idx
+	row_t GetRowId(idx_t idx);
+	//! Get the maximum capacity of the leaf, must not match with its count
 	idx_t GetCapacity() const;
+	//! Returns whether a leaf holds exactly one inlined row ID or multiple row IDs
 	bool IsInlined() const;
+	//! Returns a pointer to all row IDs of the leaf
 	row_t *GetRowIds();
 
 public:
+	static Leaf *New();
 	static Leaf *New(Key &value, uint32_t depth, row_t row_id);
 	static Leaf *New(Key &value, uint32_t depth, row_t *row_ids, idx_t num_elements);
 	static Leaf *New(row_t *row_ids, idx_t num_elements, Prefix &prefix);
 	static Leaf *New(row_t row_id, Prefix &prefix);
-	//! Insert a row_id into a leaf
-	void Insert(row_t row_id);
-	//! Remove a row_id from a leaf
-	void Remove(row_t row_id);
+
+	//! Returns the memory size of the leaf
+	idx_t MemorySize(ART &art, const bool &recurse) override;
+	//! Insert a row ID into a leaf
+	void Insert(ART &art, row_t row_id);
+	//! Remove a row ID from a leaf
+	void Remove(ART &art, row_t row_id);
 
 	//! Returns the string representation of a leaf
 	static string ToString(Node *node);
 	//! Merge two NLeaf nodes
-	static void Merge(Node *&l_node, Node *&r_node);
+	static void Merge(ART &art, Node *&l_node, Node *&r_node);
 
 	//! Serialize a leaf
 	BlockPointer Serialize(duckdb::MetaBlockWriter &writer);
-	// Deserialize a leaf
-	static Leaf *Deserialize(duckdb::MetaBlockReader &reader);
+	//! Deserialize a leaf
+	void Deserialize(ART &art, duckdb::MetaBlockReader &reader);
 
 private:
 	union {

package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp

@@ -37,10 +37,12 @@ struct InternalType {
 };
 
 struct MergeInfo {
-	MergeInfo(ART *l_art, ART *r_art, Node *&l_node, Node *&r_node)
-	    : l_art(l_art), r_art(r_art), l_node(l_node), r_node(r_node) {};
+	MergeInfo(ART *l_art, ART *r_art, ART *root_l_art, ART *root_r_art, Node *&l_node, Node *&r_node)
+	    : l_art(l_art), r_art(r_art), root_l_art(root_l_art), root_r_art(root_r_art), l_node(l_node), r_node(r_node) {};
 	ART *l_art;
 	ART *r_art;
+	ART *root_l_art;
+	ART *root_r_art;
 	Node *&l_node;
 	Node *&r_node;
 };
@@ -57,6 +59,8 @@ struct ParentsOfNodes {
 class Node {
 public:
 	static const uint8_t EMPTY_MARKER = 48;
+	static const uint8_t NODE_48_SHRINK_THRESHOLD = 12;
+	static const uint8_t NODE_256_SHRINK_THRESHOLD = 36;
 
 public:
 	explicit Node(NodeType type);
@@ -71,6 +75,8 @@ public:
 	Prefix prefix;
 
 	static void Delete(Node *node);
+	//! Returns the memory size of the node
+	virtual idx_t MemorySize(ART &art, const bool &recurse);
 	//! Get the position of a child corresponding exactly to the specific byte, returns DConstants::INVALID_INDEX if not
 	//! exists
 	virtual idx_t GetChildPos(uint8_t k) {
@@ -88,31 +94,37 @@ public:
 	virtual idx_t GetNextPos(idx_t pos) {
 		return DConstants::INVALID_INDEX;
 	}
+	//! Get the next position and byte in the node, or DConstants::INVALID_INDEX if there is no next position. if pos ==
+	//! DConstants::INVALID_INDEX, then the first valid position in the node is returned
+	virtual idx_t GetNextPosAndByte(idx_t pos, uint8_t &byte) {
+		return DConstants::INVALID_INDEX;
+	}
 	//! Get the child at the specified position in the node. pos should be between [0, count). Throws an assertion if
 	//! the element is not found
 	virtual Node *GetChild(ART &art, idx_t pos);
 	//! Replaces the pointer to a child node
 	virtual void ReplaceChildPointer(idx_t pos, Node *node);
+	//! Returns whether the child at pos is in memory
+	virtual bool ChildIsInMemory(idx_t pos);
 
 	//! Insert a new child node at key_byte into the node
-	static void InsertChild(Node *&node, uint8_t key_byte, Node *new_child);
+	static void InsertChild(ART &art, Node *&node, uint8_t key_byte, Node *new_child);
 	//! Erase child node entry from node
-	static void EraseChild(Node *&node, idx_t pos, ART &art);
+	static void EraseChild(ART &art, Node *&node, idx_t pos);
 	//! Get the corresponding node type for the provided size
 	static NodeType GetTypeBySize(idx_t size);
 	//! Create a new node of the specified type
-	static void New(NodeType &type, Node *&node);
+	static void New(const NodeType &type, Node *&node);
 
 	//! Returns the string representation of a node
 	string ToString(ART &art);
 	//! Serialize this node
 	BlockPointer Serialize(ART &art, duckdb::MetaBlockWriter &writer);
+	//! Returns the memory size of the node
+	idx_t RecursiveMemorySize(ART &art);
 
 	//! Deserialize this node
 	static Node *Deserialize(ART &art, idx_t block_id, idx_t offset);
-	//! Merge r_node into l_node at the specified byte
-	static bool MergeAtByte(MergeInfo &info, idx_t depth, idx_t &l_child_pos, idx_t &r_pos, uint8_t &key_byte,
-	                        Node *&l_parent, idx_t l_pos);
 	//! Merge two ART
 	static bool MergeARTs(ART *l_art, ART *r_art);
 
@@ -120,7 +132,7 @@ private:
 	//! Serialize internal nodes
 	BlockPointer SerializeInternal(ART &art, duckdb::MetaBlockWriter &writer, InternalType &internal_type);
 	//! Deserialize internal nodes
-	void DeserializeInternal(duckdb::MetaBlockReader &reader);
+	void DeserializeInternal(ART &art, duckdb::MetaBlockReader &reader);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp

@@ -14,12 +14,17 @@ namespace duckdb {
 
 class Node16 : public Node {
 public:
+	//! Empty Node16
 	explicit Node16();
+	//! Array containing all partial key bytes
 	uint8_t key[16];
+	//! ART pointers to the child nodes
 	ARTPointer children[16];
 
 public:
 	static Node16 *New();
+	//! Returns the memory size of the Node16
+	idx_t MemorySize(ART &art, const bool &recurse) override;
 	//! Get position of a specific byte, returns DConstants::INVALID_INDEX if not exists
 	idx_t GetChildPos(uint8_t k) override;
 	//! Get the position of the first child that is greater or equal to the specific byte, or DConstants::INVALID_INDEX
@@ -29,18 +34,23 @@ public:
 	idx_t GetMin() override;
 	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position
 	idx_t GetNextPos(idx_t pos) override;
+	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position
+	idx_t GetNextPosAndByte(idx_t pos, uint8_t &byte) override;
 	//! Get Node16 child
 	Node *GetChild(ART &art, idx_t pos) override;
 	//! Replace child pointer
 	void ReplaceChildPointer(idx_t pos, Node *node) override;
+	//! Returns whether the child at pos is in memory
+	bool ChildIsInMemory(idx_t pos) override;
 
 	//! Insert a new child node at key_byte into the Node16
-	static void InsertChild(Node *&node, uint8_t key_byte, Node *new_child);
+	static void InsertChild(ART &art, Node *&node, uint8_t key_byte, Node *new_child);
 	//! Erase the child at pos and (if necessary) shrink to Node4
-	static void EraseChild(Node *&node, int pos, ART &art);
-	//! Merge Node16 into l_node
-	static bool Merge(MergeInfo &info, idx_t depth, Node *&l_parent, idx_t l_pos);
+	static void EraseChild(ART &art, Node *&node, idx_t pos);
+
 	//! Returns the size (maximum capacity) of the Node16
-	static idx_t GetSize();
+	static constexpr idx_t GetSize() {
+		return 16;
+	}
 };
 } // namespace duckdb