duckdb 0.8.2-dev2509.0 → 0.8.2-dev2669.0

package/src/duckdb/src/execution/index/art/prefix.cpp

@@ -55,9 +55,35 @@ void Prefix::New(ART &art, reference<Node> &node, const ARTKey &key, const uint3
 
 void Prefix::Free(ART &art, Node &node) {
 
-	D_ASSERT(node.IsSet() && !node.IsSerialized());
-	auto &child = Prefix::Get(art, node).ptr;
-	Node::Free(art, child);
+	Node current_node = node;
+	Node next_node;
+	while (current_node.IsSet() && !current_node.IsSerialized() && current_node.GetType() == NType::PREFIX) {
+		next_node = Prefix::Get(art, current_node).ptr;
+		Node::GetAllocator(art, NType::PREFIX).Free(current_node);
+		current_node = next_node;
+	}
+
+	Node::Free(art, current_node);
+	node.Reset();
+}
+
+void Prefix::InitializeMerge(ART &art, Node &node, const ARTFlags &flags) {
+
+	auto merge_buffer_count = flags.merge_buffer_counts[(uint8_t)NType::PREFIX - 1];
+
+	Node next_node = node;
+	reference<Prefix> prefix = Prefix::Get(art, next_node);
+
+	while (next_node.GetType() == NType::PREFIX) {
+		next_node = prefix.get().ptr;
+		if (prefix.get().ptr.GetType() == NType::PREFIX) {
+			prefix.get().ptr.AddToBufferID(merge_buffer_count);
+			prefix = Prefix::Get(art, next_node);
+		}
+	}
+
+	node.AddToBufferID(merge_buffer_count);
+	prefix.get().ptr.InitializeMerge(art, flags);
 }
 
 void Prefix::Concatenate(ART &art, Node &prefix_node, const uint8_t byte, Node &child_prefix_node) {
@@ -280,19 +306,28 @@ string Prefix::VerifyAndToString(ART &art, Node &node, const bool only_verify) {
 	return str + node_ref.get().VerifyAndToString(art, only_verify);
 }
 
-BlockPointer Prefix::Serialize(ART &art, MetaBlockWriter &writer) {
+BlockPointer Prefix::Serialize(ART &art, Node &node, MetaBlockWriter &writer) {
 
-	// recurse into the child and retrieve its block pointer
-	auto child_block_pointer = ptr.Serialize(art, writer);
+	reference<Node> first_non_prefix(node);
+	idx_t total_count = Prefix::TotalCount(art, first_non_prefix);
+	auto child_block_pointer = first_non_prefix.get().Serialize(art, writer);
 
 	// get pointer and write fields
 	auto block_pointer = writer.GetBlockPointer();
 	writer.Write(NType::PREFIX);
-	writer.Write<uint8_t>(data[Node::PREFIX_SIZE]);
+	writer.Write<idx_t>(total_count);
+
+	reference<Node> current_node(node);
+	while (current_node.get().GetType() == NType::PREFIX) {
 
-	// write prefix bytes
-	for (idx_t i = 0; i < data[Node::PREFIX_SIZE]; i++) {
-		writer.Write(data[i]);
+		// write prefix bytes
+		D_ASSERT(!current_node.get().IsSerialized());
+		auto &prefix = Prefix::Get(art, current_node);
+		for (idx_t i = 0; i < prefix.data[Node::PREFIX_SIZE]; i++) {
+			writer.Write(prefix.data[i]);
+		}
+
+		current_node = prefix.ptr;
 	}
 
 	// write child block pointer
@@ -302,17 +337,48 @@ BlockPointer Prefix::Serialize(ART &art, MetaBlockWriter &writer) {
 	return block_pointer;
 }
 
-void Prefix::Deserialize(MetaBlockReader &reader) {
+void Prefix::Deserialize(ART &art, Node &node, MetaBlockReader &reader) {
+
+	auto total_count = reader.Read<idx_t>();
+	reference<Node> current_node(node);
+
+	while (total_count) {
+		current_node.get() = Node::GetAllocator(art, NType::PREFIX).New();
+		current_node.get().SetType((uint8_t)NType::PREFIX);
 
-	data[Node::PREFIX_SIZE] = reader.Read<uint8_t>();
+		auto &prefix = Prefix::Get(art, current_node);
+		prefix.data[Node::PREFIX_SIZE] = MinValue((idx_t)Node::PREFIX_SIZE, total_count);
 
-	// read bytes
-	for (idx_t i = 0; i < data[Node::PREFIX_SIZE]; i++) {
-		data[i] = reader.Read<uint8_t>();
+		// read bytes
+		for (idx_t i = 0; i < prefix.data[Node::PREFIX_SIZE]; i++) {
+			prefix.data[i] = reader.Read<uint8_t>();
+		}
+
+		total_count -= prefix.data[Node::PREFIX_SIZE];
+		current_node = prefix.ptr;
+		prefix.ptr.Reset();
 	}
 
 	// read child block pointer
-	ptr = Node(reader);
+	current_node.get() = Node(reader);
+}
+
+void Prefix::Vacuum(ART &art, Node &node, const ARTFlags &flags) {
+
+	bool flag_set = flags.vacuum_flags[(uint8_t)NType::PREFIX - 1];
+	auto &allocator = Node::GetAllocator(art, NType::PREFIX);
+
+	reference<Node> node_ref(node);
+	while (!node_ref.get().IsSerialized() && node_ref.get().GetType() == NType::PREFIX) {
+		if (flag_set && allocator.NeedsVacuum(node_ref)) {
+			node_ref.get() = allocator.VacuumPointer(node_ref);
+			node_ref.get().SetType((uint8_t)NType::PREFIX);
+		}
+		auto &prefix = Prefix::Get(art, node_ref);
+		node_ref = prefix.ptr;
+	}
+
+	node_ref.get().Vacuum(art, flags);
 }
 
 Prefix &Prefix::Append(ART &art, const uint8_t byte) {
@@ -356,4 +422,22 @@ void Prefix::Append(ART &art, Node other_prefix) {
 	D_ASSERT(prefix.get().ptr.GetType() != NType::PREFIX);
 }
 
+idx_t Prefix::TotalCount(ART &art, reference<Node> &node) {
+
+	// NOTE: first prefix in the prefix chain is already deserialized
+	D_ASSERT(node.get().IsSet() && !node.get().IsSerialized());
+
+	idx_t count = 0;
+	while (node.get().GetType() == NType::PREFIX) {
+		auto &prefix = Prefix::Get(art, node);
+		count += prefix.data[Node::PREFIX_SIZE];
+
+		if (prefix.ptr.IsSerialized()) {
+			prefix.ptr.Deserialize(art);
+		}
+		node = prefix.ptr;
+	}
+	return count;
+}
+
 } // namespace duckdb

package/src/duckdb/src/execution/operator/schema/physical_create_index.cpp

@@ -4,7 +4,9 @@
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/duck_index_entry.hpp"
 #include "duckdb/main/client_context.hpp"
+#include "duckdb/storage/index.hpp"
 #include "duckdb/storage/storage_manager.hpp"
+#include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/main/database_manager.hpp"
 #include "duckdb/execution/index/art/art_key.hpp"
 #include "duckdb/execution/index/art/node.hpp"
@@ -15,10 +17,10 @@ namespace duckdb {
 PhysicalCreateIndex::PhysicalCreateIndex(LogicalOperator &op, TableCatalogEntry &table_p,
                                          const vector<column_t> &column_ids, unique_ptr<CreateIndexInfo> info,
                                          vector<unique_ptr<Expression>> unbound_expressions,
-                                         idx_t estimated_cardinality)
+                                         idx_t estimated_cardinality, const bool sorted)
     : PhysicalOperator(PhysicalOperatorType::CREATE_INDEX, op.types, estimated_cardinality),
-      table(table_p.Cast<DuckTableEntry>()), info(std::move(info)),
-      unbound_expressions(std::move(unbound_expressions)) {
+      table(table_p.Cast<DuckTableEntry>()), info(std::move(info)), unbound_expressions(std::move(unbound_expressions)),
+      sorted(sorted) {
 	// convert virtual column ids to storage column ids
 	for (auto &column_id : column_ids) {
 		storage_ids.push_back(table.GetColumns().LogicalToPhysical(LogicalIndex(column_id)).index);
@@ -86,43 +88,65 @@ unique_ptr<LocalSinkState> PhysicalCreateIndex::GetLocalSinkState(ExecutionConte
 	return std::move(state);
 }
 
-SinkResultType PhysicalCreateIndex::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const {
+SinkResultType PhysicalCreateIndex::SinkUnsorted(Vector &row_identifiers, OperatorSinkInput &input) const {
 
-	D_ASSERT(chunk.ColumnCount() >= 2);
-	auto &lstate = input.local_state.Cast<CreateIndexLocalSinkState>();
-	auto &row_identifiers = chunk.data[chunk.ColumnCount() - 1];
+	auto &l_state = input.local_state.Cast<CreateIndexLocalSinkState>();
+	auto count = l_state.key_chunk.size();
 
-	// generate the keys for the given input
-	lstate.key_chunk.ReferenceColumns(chunk, lstate.key_column_ids);
-	lstate.arena_allocator.Reset();
-	ART::GenerateKeys(lstate.arena_allocator, lstate.key_chunk, lstate.keys);
+	// get the corresponding row IDs
+	row_identifiers.Flatten(count);
+	auto row_ids = FlatVector::GetData<row_t>(row_identifiers);
+
+	// insert the row IDs
+	auto &art = l_state.local_index->Cast<ART>();
+	for (idx_t i = 0; i < count; i++) {
+		if (!art.Insert(*art.tree, l_state.keys[i], 0, row_ids[i])) {
+			throw ConstraintException("Data contains duplicates on indexed column(s)");
+		}
+	}
 
+	return SinkResultType::NEED_MORE_INPUT;
+}
+
+SinkResultType PhysicalCreateIndex::SinkSorted(Vector &row_identifiers, OperatorSinkInput &input) const {
+
+	auto &l_state = input.local_state.Cast<CreateIndexLocalSinkState>();
 	auto &storage = table.GetStorage();
-	auto art = make_uniq<ART>(lstate.local_index->column_ids, lstate.local_index->table_io_manager,
-	                          lstate.local_index->unbound_expressions, lstate.local_index->constraint_type, storage.db);
-	if (!art->ConstructFromSorted(lstate.key_chunk.size(), lstate.keys, row_identifiers)) {
+	auto &l_index = l_state.local_index;
+
+	// create an ART from the chunk
+	auto art = make_uniq<ART>(l_index->column_ids, l_index->table_io_manager, l_index->unbound_expressions,
+	                          l_index->constraint_type, storage.db, l_index->Cast<ART>().allocators);
+	if (!art->ConstructFromSorted(l_state.key_chunk.size(), l_state.keys, row_identifiers)) {
 		throw ConstraintException("Data contains duplicates on indexed column(s)");
 	}
 
 	// merge into the local ART
-	if (!lstate.local_index->MergeIndexes(*art)) {
+	if (!l_index->MergeIndexes(*art)) {
 		throw ConstraintException("Data contains duplicates on indexed column(s)");
 	}
 
-#ifdef DEBUG
-	// ensure that all row IDs of this chunk exist in the ART
-	auto &local_art = lstate.local_index->Cast<ART>();
-	auto row_ids = FlatVector::GetData<row_t>(row_identifiers);
-	for (idx_t i = 0; i < lstate.key_chunk.size(); i++) {
-		auto leaf = local_art.Lookup(*local_art.tree, lstate.keys[i], 0);
-		D_ASSERT(leaf.IsSet());
-		D_ASSERT(Leaf::ContainsRowId(local_art, leaf, row_ids[i]));
-	}
-#endif
-
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
+SinkResultType PhysicalCreateIndex::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const {
+
+	D_ASSERT(chunk.ColumnCount() >= 2);
+
+	// generate the keys for the given input
+	auto &l_state = input.local_state.Cast<CreateIndexLocalSinkState>();
+	l_state.key_chunk.ReferenceColumns(chunk, l_state.key_column_ids);
+	l_state.arena_allocator.Reset();
+	ART::GenerateKeys(l_state.arena_allocator, l_state.key_chunk, l_state.keys);
+
+	// insert the keys and their corresponding row IDs
+	auto &row_identifiers = chunk.data[chunk.ColumnCount() - 1];
+	if (sorted) {
+		return SinkSorted(row_identifiers, input);
+	}
+	return SinkUnsorted(row_identifiers, input);
+}
+
 SinkCombineResultType PhysicalCreateIndex::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
 
 	auto &gstate = input.global_state.Cast<CreateIndexGlobalSinkState>();
@@ -133,18 +157,17 @@ SinkCombineResultType PhysicalCreateIndex::Combine(ExecutionContext &context, Op
 		throw ConstraintException("Data contains duplicates on indexed column(s)");
 	}
 
-	// vacuum excess memory
-	gstate.global_index->Vacuum();
-
 	return SinkCombineResultType::FINISHED;
 }
 
 SinkFinalizeType PhysicalCreateIndex::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
                                                OperatorSinkFinalizeInput &input) const {
 
-	// here, we just set the resulting global index as the newly created index of the table
-
+	// here, we set the resulting global index as the newly created index of the table
 	auto &state = input.global_state.Cast<CreateIndexGlobalSinkState>();
+
+	// vacuum excess memory and verify
+	state.global_index->Vacuum();
 	D_ASSERT(!state.global_index->VerifyAndToString(true).empty());
 
 	auto &storage = table.GetStorage();

package/src/duckdb/src/execution/physical_plan/plan_create_index.cpp

@@ -68,27 +68,44 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCreateInde
 	null_filter->types.emplace_back(LogicalType::ROW_TYPE);
 	null_filter->children.push_back(std::move(projection));
 
-	// order operator
-
-	vector<BoundOrderByNode> orders;
-	vector<idx_t> projections;
-	for (idx_t i = 0; i < new_column_types.size() - 1; i++) {
-		auto col_expr = make_uniq_base<Expression, BoundReferenceExpression>(new_column_types[i], i);
-		orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_FIRST, std::move(col_expr));
-		projections.emplace_back(i);
+	// determine if we sort the data prior to index creation
+	// we don't sort, if either VARCHAR or compound key
+	auto perform_sorting = true;
+	if (op.unbound_expressions.size() > 1) {
+		perform_sorting = false;
+	} else if (op.unbound_expressions[0]->return_type.InternalType() == PhysicalType::VARCHAR) {
+		perform_sorting = false;
 	}
-	projections.emplace_back(new_column_types.size() - 1);
-
-	auto physical_order =
-	    make_uniq<PhysicalOrder>(new_column_types, std::move(orders), std::move(projections), op.estimated_cardinality);
-	physical_order->children.push_back(std::move(null_filter));
 
 	// actual physical create index operator
 
 	auto physical_create_index =
 	    make_uniq<PhysicalCreateIndex>(op, op.table, op.info->column_ids, std::move(op.info),
-	                                   std::move(op.unbound_expressions), op.estimated_cardinality);
-	physical_create_index->children.push_back(std::move(physical_order));
+	                                   std::move(op.unbound_expressions), op.estimated_cardinality, perform_sorting);
+
+	if (perform_sorting) {
+
+		// optional order operator
+		vector<BoundOrderByNode> orders;
+		vector<idx_t> projections;
+		for (idx_t i = 0; i < new_column_types.size() - 1; i++) {
+			auto col_expr = make_uniq_base<Expression, BoundReferenceExpression>(new_column_types[i], i);
+			orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_FIRST, std::move(col_expr));
+			projections.emplace_back(i);
+		}
+		projections.emplace_back(new_column_types.size() - 1);
+
+		auto physical_order = make_uniq<PhysicalOrder>(new_column_types, std::move(orders), std::move(projections),
+		                                               op.estimated_cardinality);
+		physical_order->children.push_back(std::move(null_filter));
+
+		physical_create_index->children.push_back(std::move(physical_order));
+	} else {
+
+		// no ordering
+		physical_create_index->children.push_back(std::move(null_filter));
+	}
+
 	return std::move(physical_create_index);
 }
 

package/src/duckdb/src/function/table/arrow/arrow_duck_schema.cpp

@@ -0,0 +1,57 @@
+#include "duckdb/function/table/arrow/arrow_duck_schema.hpp"
+#include "duckdb/common/arrow/arrow.hpp"
+#include "duckdb/common/exception.hpp"
+
+namespace duckdb {
+
+void ArrowTableType::AddColumn(idx_t index, unique_ptr<ArrowType> type) {
+	D_ASSERT(arrow_convert_data.find(index) == arrow_convert_data.end());
+	arrow_convert_data.emplace(std::make_pair(index, std::move(type)));
+}
+
+const arrow_column_map_t &ArrowTableType::GetColumns() const {
+	return arrow_convert_data;
+}
+
+void ArrowType::AddChild(unique_ptr<ArrowType> child) {
+	children.emplace_back(std::move(child));
+}
+
+void ArrowType::AssignChildren(vector<unique_ptr<ArrowType>> children) {
+	D_ASSERT(this->children.empty());
+	this->children = std::move(children);
+}
+
+void ArrowType::SetDictionary(unique_ptr<ArrowType> dictionary) {
+	D_ASSERT(!this->dictionary_type);
+	dictionary_type = std::move(dictionary);
+}
+
+const ArrowType &ArrowType::GetDictionary() const {
+	D_ASSERT(dictionary_type);
+	return *dictionary_type;
+}
+
+const LogicalType &ArrowType::GetDuckType() const {
+	return type;
+}
+
+ArrowVariableSizeType ArrowType::GetSizeType() const {
+	return size_type;
+}
+
+ArrowDateTimeType ArrowType::GetDateTimeType() const {
+	return date_time_precision;
+}
+
+const ArrowType &ArrowType::operator[](idx_t index) const {
+	D_ASSERT(index < children.size());
+	return *children[index];
+}
+
+idx_t ArrowType::FixedSize() const {
+	D_ASSERT(size_type == ArrowVariableSizeType::FIXED_SIZE);
+	return fixed_size;
+}
+
+} // namespace duckdb