duckdb 0.8.2-dev1764.0 → 0.8.2-dev1859.0

package/README.md

@@ -100,6 +100,13 @@ var stmt = con.prepare('select ?::INTEGER as fortytwo', function(err, stmt) {
 });
 ```
 
+## Supported Node versions
+We actively support only LTS and In-Support Node versions, as per July 2023, they are: Node 16, Node 18 and Node 20.
+Release schedule for Node.js can be checked here: https://github.com/nodejs/release#release-schedule.
+
+We currently bundle and test DuckDB also for Node 10, 12, 14, 17 and 19. We plan of going so going forward as long as the tooling supports it.
+As per July 2023, Node 15 has been removed from the supported versions.
+
 ## Development
 
 ### First install:

package/binding.gyp

@@ -45,6 +45,7 @@
                 "src/duckdb/ub_src_core_functions_scalar_bit.cpp", 
                 "src/duckdb/ub_src_core_functions_scalar_blob.cpp", 
                 "src/duckdb/ub_src_core_functions_scalar_date.cpp", 
+                "src/duckdb/ub_src_core_functions_scalar_debug.cpp", 
                 "src/duckdb/ub_src_core_functions_scalar_enum.cpp", 
                 "src/duckdb/ub_src_core_functions_scalar_generic.cpp", 
                 "src/duckdb/ub_src_core_functions_scalar_list.cpp", 

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.2-dev1764.0",
+  "version": "0.8.2-dev1859.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/extension/parquet/include/parquet_reader.hpp

@@ -93,6 +93,7 @@ public:
 	shared_ptr<ParquetFileMetadataCache> metadata;
 	ParquetOptions parquet_options;
 	MultiFileReaderData reader_data;
+	unique_ptr<ColumnReader> root_reader;
 
 public:
 	void InitializeScan(ParquetReaderScanState &state, vector<idx_t> groups_to_read);

package/src/duckdb/extension/parquet/parquet_extension.cpp

@@ -116,6 +116,11 @@ struct ParquetWriteBindData : public TableFunctionData {
 	vector<string> column_names;
 	duckdb_parquet::format::CompressionCodec::type codec = duckdb_parquet::format::CompressionCodec::SNAPPY;
 	idx_t row_group_size = RowGroup::ROW_GROUP_SIZE;
+
+	//! If row_group_size_bytes is not set, we default to row_group_size * BYTES_PER_ROW
+	static constexpr const idx_t BYTES_PER_ROW = 1024;
+	idx_t row_group_size_bytes;
+
 	ChildFieldIDs field_ids;
 };
 
@@ -741,33 +746,39 @@ static void GetFieldIDs(const Value &field_ids_value, ChildFieldIDs &field_ids,
 unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, CopyInfo &info, vector<string> &names,
                                           vector<LogicalType> &sql_types) {
 	D_ASSERT(names.size() == sql_types.size());
+	bool row_group_size_bytes_set = false;
 	auto bind_data = make_uniq<ParquetWriteBindData>();
 	for (auto &option : info.options) {
-		auto loption = StringUtil::Lower(option.first);
+		const auto loption = StringUtil::Lower(option.first);
+		if (option.second.size() != 1) {
+			// All parquet write options require exactly one argument
+			throw BinderException("%s requires exactly one argument", StringUtil::Upper(loption));
+		}
 		if (loption == "row_group_size" || loption == "chunk_size") {
 			bind_data->row_group_size = option.second[0].GetValue<uint64_t>();
+		} else if (loption == "row_group_size_bytes") {
+			auto roption = option.second[0];
+			if (roption.GetTypeMutable().id() == LogicalTypeId::VARCHAR) {
+				bind_data->row_group_size_bytes = DBConfig::ParseMemoryLimit(roption.ToString());
+			} else {
+				bind_data->row_group_size_bytes = option.second[0].GetValue<uint64_t>();
+			}
+			row_group_size_bytes_set = true;
 		} else if (loption == "compression" || loption == "codec") {
-			if (!option.second.empty()) {
-				auto roption = StringUtil::Lower(option.second[0].ToString());
-				if (roption == "uncompressed") {
-					bind_data->codec = duckdb_parquet::format::CompressionCodec::UNCOMPRESSED;
-					continue;
-				} else if (roption == "snappy") {
-					bind_data->codec = duckdb_parquet::format::CompressionCodec::SNAPPY;
-					continue;
-				} else if (roption == "gzip") {
-					bind_data->codec = duckdb_parquet::format::CompressionCodec::GZIP;
-					continue;
-				} else if (roption == "zstd") {
-					bind_data->codec = duckdb_parquet::format::CompressionCodec::ZSTD;
-					continue;
-				}
+			const auto roption = StringUtil::Lower(option.second[0].ToString());
+			if (roption == "uncompressed") {
+				bind_data->codec = duckdb_parquet::format::CompressionCodec::UNCOMPRESSED;
+			} else if (roption == "snappy") {
+				bind_data->codec = duckdb_parquet::format::CompressionCodec::SNAPPY;
+			} else if (roption == "gzip") {
+				bind_data->codec = duckdb_parquet::format::CompressionCodec::GZIP;
+			} else if (roption == "zstd") {
+				bind_data->codec = duckdb_parquet::format::CompressionCodec::ZSTD;
+			} else {
+				throw BinderException("Expected %s argument to be either [uncompressed, snappy, gzip or zstd]",
+				                      loption);
 			}
-			throw BinderException("Expected %s argument to be either [uncompressed, snappy, gzip or zstd]", loption);
 		} else if (loption == "field_ids") {
-			if (option.second.size() != 1) {
-				throw BinderException("FIELD_IDS requires exactly one argument");
-			}
 			if (option.second[0].type().id() == LogicalTypeId::VARCHAR &&
 			    StringUtil::Lower(StringValue::Get(option.second[0])) == "auto") {
 				idx_t field_id = 0;
@@ -788,6 +799,9 @@ unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, CopyInfo &info
 			throw NotImplementedException("Unrecognized option for PARQUET: %s", option.first.c_str());
 		}
 	}
+	if (!row_group_size_bytes_set) {
+		bind_data->row_group_size_bytes = bind_data->row_group_size * ParquetWriteBindData::BYTES_PER_ROW;
+	}
 	bind_data->sql_types = sql_types;
 	bind_data->column_names = names;
 	return std::move(bind_data);
@@ -812,8 +826,10 @@ void ParquetWriteSink(ExecutionContext &context, FunctionData &bind_data_p, Glob
 
 	// append data to the local (buffered) chunk collection
 	local_state.buffer.Append(local_state.append_state, input);
-	if (local_state.buffer.Count() > bind_data.row_group_size) {
-		// if the chunk collection exceeds a certain size we flush it to the parquet file
+
+	if (local_state.buffer.Count() > bind_data.row_group_size ||
+	    local_state.buffer.SizeInBytes() > bind_data.row_group_size_bytes) {
+		// if the chunk collection exceeds a certain size (rows/bytes) we flush it to the parquet file
 		local_state.append_state.current_chunk_state.handles.clear();
 		global_state.writer->Flush(local_state.buffer);
 		local_state.buffer.InitializeAppend(local_state.append_state);

package/src/duckdb/extension/parquet/parquet_reader.cpp

@@ -399,8 +399,7 @@ void ParquetReader::InitializeSchema() {
 	if (file_meta_data->schema.size() < 2) {
 		throw FormatException("Need at least one non-root column in the file");
 	}
-	auto root_reader = CreateReader();
-
+	root_reader = CreateReader();
 	auto &root_type = root_reader->Type();
 	auto &child_types = StructType::GetChildTypes(root_type);
 	D_ASSERT(root_type.id() == LogicalTypeId::STRUCT);
@@ -450,7 +449,6 @@ ParquetReader::ParquetReader(ClientContext &context_p, string file_name_p, Parqu
 			ObjectCache::GetObjectCache(context_p).Put(file_name, metadata);
 		}
 	}
-
 	InitializeSchema();
 }
 
@@ -483,7 +481,6 @@ unique_ptr<BaseStatistics> ParquetReader::ReadStatistics(const string &name) {
 
 	unique_ptr<BaseStatistics> column_stats;
 	auto file_meta_data = GetFileMetadata();
-	auto root_reader = CreateReader();
 	auto column_reader = root_reader->Cast<StructColumnReader>().GetChildReader(file_col_idx);
 
 	for (idx_t row_group_idx = 0; row_group_idx < file_meta_data->row_groups.size(); row_group_idx++) {

package/src/duckdb/src/common/constants.cpp

@@ -7,7 +7,8 @@
 namespace duckdb {
 
 constexpr const idx_t DConstants::INVALID_INDEX;
-const row_t MAX_ROW_ID = 4611686018427388000ULL; // 2^62
+const row_t MAX_ROW_ID = 36028797018960000ULL;       // 2^55
+const row_t MAX_ROW_ID_LOCAL = 72057594037920000ULL; // 2^56
 const column_t COLUMN_IDENTIFIER_ROW_ID = (column_t)-1;
 const sel_t ZERO_VECTOR[STANDARD_VECTOR_SIZE] = {0};
 const double PI = 3.141592653589793;

package/src/duckdb/src/common/enum_util.cpp

@@ -3112,8 +3112,6 @@ const char* EnumUtil::ToChars<NType>(NType value) {
 	switch(value) {
 	case NType::PREFIX:
 		return "PREFIX";
-	case NType::LEAF_SEGMENT:
-		return "LEAF_SEGMENT";
 	case NType::LEAF:
 		return "LEAF";
 	case NType::NODE_4:
@@ -3124,6 +3122,8 @@ const char* EnumUtil::ToChars<NType>(NType value) {
 		return "NODE_48";
 	case NType::NODE_256:
 		return "NODE_256";
+	case NType::LEAF_INLINED:
+		return "LEAF_INLINED";
 	default:
 		throw NotImplementedException(StringUtil::Format("Enum value: '%d' not implemented", value));
 	}
@@ -3134,9 +3134,6 @@ NType EnumUtil::FromString<NType>(const char *value) {
 	if (StringUtil::Equals(value, "PREFIX")) {
 		return NType::PREFIX;
 	}
-	if (StringUtil::Equals(value, "LEAF_SEGMENT")) {
-		return NType::LEAF_SEGMENT;
-	}
 	if (StringUtil::Equals(value, "LEAF")) {
 		return NType::LEAF;
 	}
@@ -3152,6 +3149,9 @@ NType EnumUtil::FromString<NType>(const char *value) {
 	if (StringUtil::Equals(value, "NODE_256")) {
 		return NType::NODE_256;
 	}
+	if (StringUtil::Equals(value, "LEAF_INLINED")) {
+		return NType::LEAF_INLINED;
+	}
 	throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
 }
 

package/src/duckdb/src/common/sort/sort_state.cpp

@@ -315,7 +315,7 @@ void LocalSortState::ReOrder(SortedData &sd, data_ptr_t sorting_ptr, RowDataColl
 		sd.data_blocks.back()->block->SetSwizzling(nullptr);
 		// Create a single heap block to store the ordered heap
 		idx_t total_byte_offset =
-		    std::accumulate(heap.blocks.begin(), heap.blocks.end(), 0,
+		    std::accumulate(heap.blocks.begin(), heap.blocks.end(), (idx_t)0,
 		                    [](idx_t a, const unique_ptr<RowDataBlock> &b) { return a + b->byte_offset; });
 		idx_t heap_block_size = MaxValue(total_byte_offset, (idx_t)Storage::BLOCK_SIZE);
 		auto ordered_heap_block = make_uniq<RowDataBlock>(*buffer_manager, heap_block_size, 1);

package/src/duckdb/src/common/sort/sorted_block.cpp

@@ -85,7 +85,7 @@ SortedBlock::SortedBlock(BufferManager &buffer_manager, GlobalSortState &state)
 }
 
 idx_t SortedBlock::Count() const {
-	idx_t count = std::accumulate(radix_sorting_data.begin(), radix_sorting_data.end(), 0,
+	idx_t count = std::accumulate(radix_sorting_data.begin(), radix_sorting_data.end(), (idx_t)0,
 	                              [](idx_t a, const unique_ptr<RowDataBlock> &b) { return a + b->count; });
 	if (!sort_layout.all_constant) {
 		D_ASSERT(count == blob_sorting_data->Count());

package/src/duckdb/src/common/types/column/column_data_collection.cpp

@@ -100,6 +100,14 @@ Allocator &ColumnDataCollection::GetAllocator() const {
 	return allocator->GetAllocator();
 }
 
+idx_t ColumnDataCollection::SizeInBytes() const {
+	idx_t total_size = 0;
+	for (const auto &segment : segments) {
+		total_size += segment->SizeInBytes();
+	}
+	return total_size;
+}
+
 //===--------------------------------------------------------------------===//
 // ColumnDataRow
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp

@@ -243,6 +243,11 @@ idx_t ColumnDataCollectionSegment::ChunkCount() const {
 	return chunk_data.size();
 }
 
+idx_t ColumnDataCollectionSegment::SizeInBytes() const {
+	D_ASSERT(!allocator->IsShared());
+	return allocator->SizeInBytes() + heap->SizeInBytes();
+}
+
 void ColumnDataCollectionSegment::FetchChunk(idx_t chunk_idx, DataChunk &result) {
 	vector<column_t> column_ids;
 	column_ids.reserve(types.size());

package/src/duckdb/src/common/types/string_heap.cpp

@@ -55,4 +55,8 @@ string_t StringHeap::EmptyString(idx_t len) {
 	return string_t(insert_pos, len);
 }
 
+idx_t StringHeap::SizeInBytes() const {
+	return allocator.SizeInBytes();
+}
+
 } // namespace duckdb

package/src/duckdb/src/core_functions/function_list.cpp

@@ -17,6 +17,7 @@
 #include "duckdb/core_functions/scalar/string_functions.hpp"
 #include "duckdb/core_functions/scalar/struct_functions.hpp"
 #include "duckdb/core_functions/scalar/union_functions.hpp"
+#include "duckdb/core_functions/scalar/debug_functions.hpp"
 
 namespace duckdb {
 
@@ -339,6 +340,7 @@ static StaticFunctionDefinition internal_functions[] = {
 	DUCKDB_AGGREGATE_FUNCTION(VarPopFun),
 	DUCKDB_AGGREGATE_FUNCTION(VarSampFun),
 	DUCKDB_AGGREGATE_FUNCTION_ALIAS(VarianceFun),
+	DUCKDB_SCALAR_FUNCTION(VectorTypeFun),
 	DUCKDB_SCALAR_FUNCTION(VersionFun),
 	DUCKDB_SCALAR_FUNCTION_SET(WeekFun),
 	DUCKDB_SCALAR_FUNCTION_SET(WeekDayFun),

package/src/duckdb/src/core_functions/scalar/debug/vector_type.cpp

@@ -0,0 +1,23 @@
+#include "duckdb/core_functions/scalar/debug_functions.hpp"
+
+#include "duckdb/common/exception.hpp"
+#include "duckdb/common/vector_operations/vector_operations.hpp"
+#include "duckdb/planner/expression/bound_function_expression.hpp"
+#include "duckdb/common/enum_util.hpp"
+
+namespace duckdb {
+
+static void VectorTypeFunction(DataChunk &input, ExpressionState &state, Vector &result) {
+	result.SetVectorType(VectorType::CONSTANT_VECTOR);
+	auto data = ConstantVector::GetData<string_t>(result);
+	data[0] = StringVector::AddString(result, EnumUtil::ToString(input.data[0].GetVectorType()));
+}
+
+ScalarFunction VectorTypeFun::GetFunction() {
+	return ScalarFunction("vector_type",        // name of the function
+	                      {LogicalType::ANY},   // argument list
+	                      LogicalType::VARCHAR, // return type
+	                      VectorTypeFunction);
+}
+
+} // namespace duckdb

package/src/duckdb/src/execution/index/art/art.cpp

@@ -5,7 +5,6 @@
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/storage/arena_allocator.hpp"
 #include "duckdb/execution/index/art/art_key.hpp"
-#include "duckdb/execution/index/art/leaf_segment.hpp"
 #include "duckdb/execution/index/art/prefix.hpp"
 #include "duckdb/execution/index/art/leaf.hpp"
 #include "duckdb/execution/index/art/node4.hpp"
@@ -44,7 +43,6 @@ ART::ART(const vector<column_t> &column_ids, TableIOManager &table_io_manager,
 
 	// initialize all allocators
 	allocators.emplace_back(make_uniq<FixedSizeAllocator>(sizeof(Prefix), buffer_manager.GetBufferAllocator()));
-	allocators.emplace_back(make_uniq<FixedSizeAllocator>(sizeof(LeafSegment), buffer_manager.GetBufferAllocator()));
 	allocators.emplace_back(make_uniq<FixedSizeAllocator>(sizeof(Leaf), buffer_manager.GetBufferAllocator()));
 	allocators.emplace_back(make_uniq<FixedSizeAllocator>(sizeof(Node4), buffer_manager.GetBufferAllocator()));
 	allocators.emplace_back(make_uniq<FixedSizeAllocator>(sizeof(Node16), buffer_manager.GetBufferAllocator()));
@@ -54,8 +52,8 @@ ART::ART(const vector<column_t> &column_ids, TableIOManager &table_io_manager,
 	// set the root node of the tree
 	tree = make_uniq<Node>();
 	if (block_id != DConstants::INVALID_INDEX) {
-		tree->buffer_id = block_id;
-		tree->offset = block_offset;
+		tree->SetSerialized();
+		tree->SetPtr(block_id, block_offset);
 		tree->Deserialize(*this);
 	}
 	serialized_data_pointer = BlockPointer(block_id, block_offset);
@@ -308,7 +306,7 @@ bool Construct(ART &art, vector<ARTKey> &keys, row_t *row_ids, Node &node, KeySe
 		reference<Node> ref_node(node);
 		Prefix::New(art, ref_node, start_key, prefix_start, start_key.len - prefix_start);
 		if (single_row_id) {
-			Leaf::New(art, ref_node, row_ids[key_section.start]);
+			Leaf::New(ref_node, row_ids[key_section.start]);
 		} else {
 			Leaf::New(art, ref_node, row_ids + key_section.start, num_row_ids);
 		}
@@ -358,19 +356,9 @@ bool ART::ConstructFromSorted(idx_t count, vector<ARTKey> &keys, Vector &row_ide
 	D_ASSERT(!VerifyAndToStringInternal(true).empty());
 	for (idx_t i = 0; i < count; i++) {
 		D_ASSERT(!keys[i].Empty());
-		auto leaf_node = Lookup(*tree, keys[i], 0);
-		D_ASSERT(leaf_node.IsSet());
-		auto &leaf = Leaf::Get(*this, leaf_node);
-
-		if (leaf.IsInlined()) {
-			D_ASSERT(row_ids[i] == leaf.row_ids.inlined);
-			continue;
-		}
-
-		D_ASSERT(leaf.row_ids.ptr.IsSet());
-		Node leaf_segment = leaf.row_ids.ptr;
-		auto position = leaf.FindRowId(*this, leaf_segment, row_ids[i]);
-		D_ASSERT(position != (uint32_t)DConstants::INVALID_INDEX);
+		auto leaf = Lookup(*tree, keys[i], 0);
+		D_ASSERT(leaf.IsSet());
+		D_ASSERT(Leaf::ContainsRowId(*this, leaf, row_ids[i]));
 	}
 #endif
 
@@ -431,19 +419,9 @@ PreservedError ART::Insert(IndexLock &lock, DataChunk &input, Vector &row_ids) {
 			continue;
 		}
 
-		auto leaf_node = Lookup(*tree, keys[i], 0);
-		D_ASSERT(leaf_node.IsSet());
-		auto &leaf = Leaf::Get(*this, leaf_node);
-
-		if (leaf.IsInlined()) {
-			D_ASSERT(row_identifiers[i] == leaf.row_ids.inlined);
-			continue;
-		}
-
-		D_ASSERT(leaf.row_ids.ptr.IsSet());
-		Node leaf_segment = leaf.row_ids.ptr;
-		auto position = leaf.FindRowId(*this, leaf_segment, row_identifiers[i]);
-		D_ASSERT(position != (uint32_t)DConstants::INVALID_INDEX);
+		auto leaf = Lookup(*tree, keys[i], 0);
+		D_ASSERT(leaf.IsSet());
+		D_ASSERT(Leaf::ContainsRowId(*this, leaf, row_identifiers[i]));
 	}
 #endif
 
@@ -471,19 +449,13 @@ void ART::VerifyAppend(DataChunk &chunk, ConflictManager &conflict_manager) {
 	CheckConstraintsForChunk(chunk, conflict_manager);
 }
 
-bool ART::InsertToLeaf(Node &leaf_node, const row_t &row_id) {
+bool ART::InsertToLeaf(Node &leaf, const row_t &row_id) {
 
-	auto &leaf = Leaf::Get(*this, leaf_node);
-
-#ifdef DEBUG
-	for (idx_t k = 0; k < leaf.count; k++) {
-		D_ASSERT(leaf.GetRowId(*this, k) != row_id);
-	}
-#endif
-	if (IsUnique() && leaf.count != 0) {
+	if (IsUnique()) {
 		return false;
 	}
-	leaf.Insert(*this, row_id);
+
+	Leaf::Insert(*this, leaf, row_id);
 	return true;
 }
 
@@ -494,14 +466,14 @@ bool ART::Insert(Node &node, const ARTKey &key, idx_t depth, const row_t &row_id
 		D_ASSERT(depth <= key.len);
 		reference<Node> ref_node(node);
 		Prefix::New(*this, ref_node, key, depth, key.len - depth);
-		Leaf::New(*this, ref_node, row_id);
+		Leaf::New(ref_node, row_id);
 		return true;
 	}
 
-	auto node_type = node.DecodeARTNodeType();
+	auto node_type = node.GetType();
 
 	// insert the row ID into this leaf
-	if (node_type == NType::LEAF) {
+	if (node_type == NType::LEAF || node_type == NType::LEAF_INLINED) {
 		return InsertToLeaf(node, row_id);
 	}
 
@@ -518,13 +490,11 @@ bool ART::Insert(Node &node, const ARTKey &key, idx_t depth, const row_t &row_id
 
 		// insert a new leaf node at key[depth]
 		Node leaf_node;
+		reference<Node> ref_node(leaf_node);
 		if (depth + 1 < key.len) {
-			reference<Node> ref_node(leaf_node);
 			Prefix::New(*this, ref_node, key, depth + 1, key.len - depth - 1);
-			Leaf::New(*this, ref_node, row_id);
-		} else {
-			Leaf::New(*this, leaf_node, row_id);
 		}
+		Leaf::New(ref_node, row_id);
 		Node::InsertChild(*this, node, key[depth], leaf_node);
 		return true;
 	}
@@ -534,7 +504,7 @@ bool ART::Insert(Node &node, const ARTKey &key, idx_t depth, const row_t &row_id
 	auto mismatch_position = Prefix::Traverse(*this, next_node, key, depth);
 
 	// prefix matches key
-	if (next_node.get().DecodeARTNodeType() != NType::PREFIX) {
+	if (next_node.get().GetType() != NType::PREFIX) {
 		return Insert(next_node, key, depth, row_id);
 	}
 
@@ -550,13 +520,11 @@ bool ART::Insert(Node &node, const ARTKey &key, idx_t depth, const row_t &row_id
 
 	// insert new leaf
 	Node leaf_node;
+	reference<Node> ref_node(leaf_node);
 	if (depth + 1 < key.len) {
-		reference<Node> ref_node(leaf_node);
 		Prefix::New(*this, ref_node, key, depth + 1, key.len - depth - 1);
-		Leaf::New(*this, ref_node, row_id);
-	} else {
-		Leaf::New(*this, leaf_node, row_id);
 	}
+	Leaf::New(ref_node, row_id);
 	Node4::InsertChild(*this, next_node, key[depth], leaf_node);
 	return true;
 }
@@ -596,19 +564,9 @@ void ART::Delete(IndexLock &state, DataChunk &input, Vector &row_ids) {
 			continue;
 		}
 
-		auto node = Lookup(*tree, keys[i], 0);
-		if (node.IsSet()) {
-			auto &leaf = Leaf::Get(*this, node);
-
-			if (leaf.IsInlined()) {
-				D_ASSERT(row_identifiers[i] != leaf.row_ids.inlined);
-				continue;
-			}
-
-			D_ASSERT(leaf.row_ids.ptr.IsSet());
-			Node leaf_segment = leaf.row_ids.ptr;
-			auto position = leaf.FindRowId(*this, leaf_segment, row_identifiers[i]);
-			D_ASSERT(position == (uint32_t)DConstants::INVALID_INDEX);
+		auto leaf = Lookup(*tree, keys[i], 0);
+		if (leaf.IsSet()) {
+			D_ASSERT(!Leaf::ContainsRowId(*this, leaf, row_identifiers[i]));
 		}
 	}
 #endif
@@ -622,21 +580,17 @@ void ART::Erase(Node &node, const ARTKey &key, idx_t depth, const row_t &row_id)
 
 	// handle prefix
 	reference<Node> next_node(node);
-	if (next_node.get().DecodeARTNodeType() == NType::PREFIX) {
+	if (next_node.get().GetType() == NType::PREFIX) {
 		Prefix::Traverse(*this, next_node, key, depth);
-		if (next_node.get().DecodeARTNodeType() == NType::PREFIX) {
+		if (next_node.get().GetType() == NType::PREFIX) {
 			return;
 		}
 	}
 
 	// delete a row ID from a leaf (root is leaf with possible prefix nodes)
-	if (next_node.get().DecodeARTNodeType() == NType::LEAF) {
-		auto &leaf = Leaf::Get(*this, next_node.get());
-		leaf.Remove(*this, row_id);
-
-		if (leaf.count == 0) {
+	if (next_node.get().GetType() == NType::LEAF || next_node.get().GetType() == NType::LEAF_INLINED) {
+		if (Leaf::Remove(*this, next_node, row_id)) {
 			Node::Free(*this, node);
-			node.Reset();
 		}
 		return;
 	}
@@ -648,20 +602,16 @@ void ART::Erase(Node &node, const ARTKey &key, idx_t depth, const row_t &row_id)
 
 		auto temp_depth = depth + 1;
 		reference<Node> child_node(*child);
-		if (child_node.get().DecodeARTNodeType() == NType::PREFIX) {
+		if (child_node.get().GetType() == NType::PREFIX) {
 			Prefix::Traverse(*this, child_node, key, temp_depth);
-			if (child_node.get().DecodeARTNodeType() == NType::PREFIX) {
+			if (child_node.get().GetType() == NType::PREFIX) {
 				return;
 			}
 		}
 
-		if (child_node.get().DecodeARTNodeType() == NType::LEAF) {
+		if (child_node.get().GetType() == NType::LEAF || child_node.get().GetType() == NType::LEAF_INLINED) {
 			// leaf found, remove entry
-			auto &leaf = Leaf::Get(*this, child_node.get());
-			leaf.Remove(*this, row_id);
-
-			if (leaf.count == 0) {
-				// leaf is empty, delete leaf, decrement node counter and maybe shrink node
+			if (Leaf::Remove(*this, child_node, row_id)) {
 				Node::DeleteChild(*this, next_node, node, key[depth]);
 			}
 			return;
@@ -713,20 +663,11 @@ static ARTKey CreateKey(ArenaAllocator &allocator, PhysicalType type, Value &val
 
 bool ART::SearchEqual(ARTKey &key, idx_t max_count, vector<row_t> &result_ids) {
 
-	auto leaf_node = Lookup(*tree, key, 0);
-	if (!leaf_node.IsSet()) {
+	auto leaf = Lookup(*tree, key, 0);
+	if (!leaf.IsSet()) {
 		return true;
 	}
-
-	auto &leaf = Leaf::Get(*this, leaf_node);
-	if (leaf.count > max_count) {
-		return false;
-	}
-	for (idx_t i = 0; i < leaf.count; i++) {
-		row_t row_id = leaf.GetRowId(*this, i);
-		result_ids.push_back(row_id);
-	}
-	return true;
+	return Leaf::GetRowIds(*this, leaf, result_ids, max_count);
 }
 
 void ART::SearchEqualJoinNoFetch(ARTKey &key, idx_t &result_size) {
@@ -738,8 +679,10 @@ void ART::SearchEqualJoinNoFetch(ARTKey &key, idx_t &result_size) {
 		return;
 	}
 
-	auto &leaf = Leaf::Get(*this, leaf_node);
-	result_size = leaf.count;
+	// we only perform index joins on PK/FK columns
+	D_ASSERT(leaf_node.GetType() == NType::LEAF_INLINED);
+	result_size = 1;
+	return;
 }
 
 //===--------------------------------------------------------------------===//
@@ -752,14 +695,14 @@ Node ART::Lookup(Node node, const ARTKey &key, idx_t depth) {
 
 		// traverse prefix, if exists
 		reference<Node> next_node(node);
-		if (next_node.get().DecodeARTNodeType() == NType::PREFIX) {
+		if (next_node.get().GetType() == NType::PREFIX) {
 			Prefix::Traverse(*this, next_node, key, depth);
-			if (next_node.get().DecodeARTNodeType() == NType::PREFIX) {
+			if (next_node.get().GetType() == NType::PREFIX) {
 				return Node();
 			}
 		}
 
-		if (next_node.get().DecodeARTNodeType() == NType::LEAF) {
+		if (next_node.get().GetType() == NType::LEAF || next_node.get().GetType() == NType::LEAF_INLINED) {
 			return next_node.get();
 		}
 
@@ -994,20 +937,18 @@ void ART::CheckConstraintsForChunk(DataChunk &input, ConflictManager &conflict_m
 			continue;
 		}
 
-		auto leaf_node = Lookup(*tree, keys[i], 0);
-		if (!leaf_node.IsSet()) {
+		auto leaf = Lookup(*tree, keys[i], 0);
+		if (!leaf.IsSet()) {
 			if (conflict_manager.AddMiss(i)) {
 				found_conflict = i;
 			}
 			continue;
 		}
 
-		// When we find a node, we need to update the 'matches' and 'row_ids'
-		// NOTE: Leafs can have more than one row_id, but for UNIQUE/PRIMARY KEY they will only have one
-		Leaf &leaf = Leaf::Get(*this, leaf_node);
-		D_ASSERT(leaf.count == 1);
-		auto row_id = leaf.GetRowId(*this, 0);
-		if (conflict_manager.AddHit(i, row_id)) {
+		// when we find a node, we need to update the 'matches' and 'row_ids'
+		// NOTE: leaves can have more than one row_id, but for UNIQUE/PRIMARY KEY they will only have one
+		D_ASSERT(leaf.GetType() == NType::LEAF_INLINED);
+		if (conflict_manager.AddHit(i, leaf.GetRowId())) {
 			found_conflict = i;
 		}
 	}
@@ -1086,7 +1027,7 @@ void ART::Vacuum(IndexLock &state) {
 	}
 
 	// traverse the allocated memory of the tree to perform a vacuum
-	Node::Vacuum(*this, *tree, flags);
+	tree->Vacuum(*this, flags);
 
 	// finalize the vacuum operation
 	FinalizeVacuum(flags);

package/src/duckdb/src/execution/index/art/art_key.cpp

@@ -72,17 +72,6 @@ bool ARTKey::operator>(const ARTKey &k) const {
 	return len > k.len;
 }
 
-bool ARTKey::operator<(const ARTKey &k) const {
-	for (uint32_t i = 0; i < MinValue<uint32_t>(len, k.len); i++) {
-		if (data[i] < k.data[i]) {
-			return true;
-		} else if (data[i] > k.data[i]) {
-			return false;
-		}
-	}
-	return len < k.len;
-}
-
 bool ARTKey::operator>=(const ARTKey &k) const {
 	for (uint32_t i = 0; i < MinValue<uint32_t>(len, k.len); i++) {
 		if (data[i] > k.data[i]) {