duckdb 0.6.2-dev1070.0 → 0.6.2-dev1092.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.6.2-dev1070.0",
+  "version": "0.6.2-dev1092.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/execution/index/art/art.cpp

@@ -15,16 +15,20 @@ namespace duckdb {
 ART::ART(const vector<column_t> &column_ids, TableIOManager &table_io_manager,
          const vector<unique_ptr<Expression>> &unbound_expressions, IndexConstraintType constraint_type,
          AttachedDatabase &db, idx_t block_id, idx_t block_offset)
+
     : Index(IndexType::ART, table_io_manager, column_ids, unbound_expressions, constraint_type), db(db),
       estimated_art_size(0), estimated_key_size(16) {
+
 	if (!Radix::IsLittleEndian()) {
 		throw NotImplementedException("ART indexes are not supported on big endian architectures");
 	}
+
 	if (block_id != DConstants::INVALID_INDEX) {
 		tree = Node::Deserialize(*this, block_id, block_offset);
 	} else {
 		tree = nullptr;
 	}
+
 	serialized_data_pointer = BlockPointer(block_id, block_offset);
 	for (idx_t i = 0; i < types.size(); i++) {
 		switch (types[i]) {
@@ -272,11 +276,17 @@ void Construct(vector<Key> &keys, row_t *row_ids, Node *&node, KeySection &key_s
 		auto num_row_ids = key_section.end - key_section.start + 1;
 
 		// check for possible constraint violation
-		if (has_constraint && num_row_ids != 1) {
+		auto single_row_id = num_row_ids == 1;
+		if (has_constraint && !single_row_id) {
 			throw ConstraintException("New data contains duplicates on indexed column(s)");
 		}
 
+		if (single_row_id) {
+			node = Leaf::New(start_key, prefix_start, row_ids[key_section.start]);
+			return;
+		}
 		node = Leaf::New(start_key, prefix_start, row_ids + key_section.start, num_row_ids);
+
 	} else { // create a new node and recurse
 
 		// we will find at least two child entries of this node, otherwise we'd have reached a leaf
@@ -298,61 +308,15 @@ void Construct(vector<Key> &keys, row_t *row_ids, Node *&node, KeySection &key_s
 	}
 }
 
-void ART::ConstructAndMerge(IndexLock &lock, PayloadScanner &scanner, Allocator &allocator) {
-
-	auto payload_types = logical_types;
-	payload_types.emplace_back(LogicalType::ROW_TYPE);
-
-	ArenaAllocator arena_allocator(BufferAllocator::Get(db));
-	vector<Key> keys(STANDARD_VECTOR_SIZE);
-
-	auto temp_art = make_unique<ART>(this->column_ids, this->table_io_manager, this->unbound_expressions,
-	                                 this->constraint_type, this->db);
+void ART::ConstructFromSorted(idx_t count, vector<Key> &keys, Vector &row_identifiers) {
 
-	for (;;) {
-		DataChunk ordered_chunk;
-		ordered_chunk.Initialize(allocator, payload_types);
-		ordered_chunk.SetCardinality(0);
-		scanner.Scan(ordered_chunk);
-		if (ordered_chunk.size() == 0) {
-			break;
-		}
-
-		// get the key chunk and the row_identifiers vector
-		DataChunk row_id_chunk;
-		ordered_chunk.Split(row_id_chunk, ordered_chunk.ColumnCount() - 1);
-		auto &row_identifiers = row_id_chunk.data[0];
-
-		D_ASSERT(row_identifiers.GetType().InternalType() == ROW_TYPE);
-		D_ASSERT(logical_types[0] == ordered_chunk.data[0].GetType());
-
-		// generate the keys for the given input
-		arena_allocator.Reset();
-		GenerateKeys(arena_allocator, ordered_chunk, keys);
-
-		// prepare the row_identifiers
-		row_identifiers.Flatten(ordered_chunk.size());
-		auto row_ids = FlatVector::GetData<row_t>(row_identifiers);
+	// prepare the row_identifiers
+	row_identifiers.Flatten(count);
+	auto row_ids = FlatVector::GetData<row_t>(row_identifiers);
 
-		// construct the ART of this chunk
-		auto art = make_unique<ART>(this->column_ids, this->table_io_manager, this->unbound_expressions,
-		                            this->constraint_type, this->db);
-		auto key_section = KeySection(0, ordered_chunk.size() - 1, 0, 0);
-		auto has_constraint = IsUnique();
-		Construct(keys, row_ids, art->tree, key_section, has_constraint);
-
-		// merge art into temp_art
-		if (!temp_art->MergeIndexes(lock, art.get())) {
-			throw ConstraintException("Data contains duplicates on indexed column(s)");
-		}
-	}
-
-	// NOTE: currently this code is only used for index creation, so we can assume that there are no
-	// duplicate violations between the existing index and the new data,
-	// so we do not need to revert any changes
-	if (!this->MergeIndexes(lock, temp_art.get())) {
-		throw ConstraintException("Data contains duplicates on indexed column(s)");
-	}
+	auto key_section = KeySection(0, count - 1, 0, 0);
+	auto has_constraint = IsUnique();
+	Construct(keys, row_ids, this->tree, key_section, has_constraint);
 }
 
 bool ART::Insert(IndexLock &lock, DataChunk &input, Vector &row_ids) {

package/src/duckdb/src/execution/index/art/leaf.cpp

@@ -158,10 +158,17 @@ void Leaf::Merge(Node *&l_node, Node *&r_node) {
 	Leaf *l_n = (Leaf *)l_node;
 	Leaf *r_n = (Leaf *)r_node;
 
-	// append non-duplicate row_ids to l_n
-	for (idx_t i = 0; i < r_n->count; i++) {
-		l_n->Insert(r_n->GetRowId(i));
+	auto l_capacity = l_n->GetCapacity();
+	auto l_row_ids = l_n->GetRowIds();
+	auto r_row_ids = r_n->GetRowIds();
+
+	if (l_n->count + r_n->count > l_capacity) {
+		auto new_capacity = NextPowerOfTwo(l_n->count + r_n->count);
+		l_row_ids = l_n->Resize(l_row_ids, l_n->count, new_capacity);
 	}
+
+	memcpy(l_row_ids + l_n->count, r_row_ids, r_n->count * sizeof(row_t));
+	l_n->count += r_n->count;
 }
 
 BlockPointer Leaf::Serialize(duckdb::MetaBlockWriter &writer) {

package/src/duckdb/src/execution/index/art/node.cpp

@@ -314,6 +314,9 @@ void UpdateParentsOfNodes(Node *&l_node, Node *&r_node, ParentsOfNodes &parents)
 	}
 }
 
+// forward declaration
+bool ResolvePrefixesAndMerge(MergeInfo &info, idx_t depth, ParentsOfNodes &parents);
+
 bool Merge(MergeInfo &info, idx_t depth, ParentsOfNodes &parents) {
 
 	// always try to merge the smaller node into the bigger node
@@ -327,16 +330,7 @@ bool Merge(MergeInfo &info, idx_t depth, ParentsOfNodes &parents) {
 		UpdateParentsOfNodes(info.l_node, info.r_node, parents);
 	}
 
-	switch (info.r_node->type) {
-	case NodeType::N256:
-		return Node256::Merge(info, depth, parents.l_parent, parents.l_pos);
-	case NodeType::N48:
-		return Node48::Merge(info, depth, parents.l_parent, parents.l_pos);
-	case NodeType::N16:
-		return Node16::Merge(info, depth, parents.l_parent, parents.l_pos);
-	case NodeType::N4:
-		return Node4::Merge(info, depth, parents.l_parent, parents.l_pos);
-	case NodeType::NLeaf:
+	if (info.r_node->type == NodeType::NLeaf) {
 		D_ASSERT(info.l_node->type == NodeType::NLeaf);
 		D_ASSERT(info.r_node->type == NodeType::NLeaf);
 		if (info.l_art->IsUnique()) {
@@ -345,7 +339,37 @@ bool Merge(MergeInfo &info, idx_t depth, ParentsOfNodes &parents) {
 		Leaf::Merge(info.l_node, info.r_node);
 		return true;
 	}
-	throw InternalException("Invalid node type for right node in merge.");
+
+	uint8_t key_byte;
+	idx_t r_child_pos = DConstants::INVALID_INDEX;
+
+	while (true) {
+		r_child_pos = info.r_node->GetNextPosAndByte(r_child_pos, key_byte);
+		if (r_child_pos == DConstants::INVALID_INDEX) {
+			break;
+		}
+		auto r_child = info.r_node->GetChild(*info.r_art, r_child_pos);
+		auto l_child_pos = info.l_node->GetChildPos(key_byte);
+
+		if (l_child_pos == DConstants::INVALID_INDEX) {
+			// insert child at empty position
+			Node::InsertChild(info.l_node, key_byte, r_child);
+			if (parents.l_parent) {
+				parents.l_parent->ReplaceChildPointer(parents.l_pos, info.l_node);
+			}
+			info.r_node->ReplaceChildPointer(r_child_pos, nullptr);
+
+		} else {
+			// recurse
+			auto l_child = info.l_node->GetChild(*info.l_art, l_child_pos);
+			MergeInfo child_info(info.l_art, info.r_art, l_child, r_child);
+			ParentsOfNodes child_parents(info.l_node, l_child_pos, info.r_node, r_child_pos);
+			if (!ResolvePrefixesAndMerge(child_info, depth + 1, child_parents)) {
+				return false;
+			}
+		}
+	}
+	return true;
 }
 
 bool ResolvePrefixesAndMerge(MergeInfo &info, idx_t depth, ParentsOfNodes &parents) {
@@ -355,22 +379,25 @@ bool ResolvePrefixesAndMerge(MergeInfo &info, idx_t depth, ParentsOfNodes &paren
 
 	// NOTE: we always merge into the left ART
 	D_ASSERT(l_node);
+	auto l_prefix_size = l_node->prefix.Size();
+	auto r_prefix_size = r_node->prefix.Size();
 
 	// make sure that r_node has the longer (or equally long) prefix
-	if (l_node->prefix.Size() > r_node->prefix.Size()) {
+	if (l_prefix_size > r_prefix_size) {
 		swap(info.l_art, info.r_art);
 		swap(l_node, r_node);
+		swap(l_prefix_size, r_prefix_size);
 		UpdateParentsOfNodes(l_node, r_node, parents);
 	}
 
 	auto mismatch_pos = l_node->prefix.MismatchPosition(r_node->prefix);
 
 	// both nodes have no prefix or the same prefix
-	if (mismatch_pos == l_node->prefix.Size() && l_node->prefix.Size() == r_node->prefix.Size()) {
+	if (mismatch_pos == l_prefix_size && l_prefix_size == r_prefix_size) {
 		return Merge(info, depth + mismatch_pos, parents);
 	}
 
-	if (mismatch_pos == l_node->prefix.Size()) {
+	if (mismatch_pos == l_prefix_size) {
 		// r_node's prefix contains l_node's prefix
 		// l_node cannot be a leaf, otherwise the key represented by l_node would be a subset of another key
 		// which is not possible by our construction
@@ -418,28 +445,6 @@ bool ResolvePrefixesAndMerge(MergeInfo &info, idx_t depth, ParentsOfNodes &paren
 	return true;
 }
 
-bool Node::MergeAtByte(MergeInfo &info, idx_t depth, idx_t &l_child_pos, idx_t &r_pos, uint8_t &key_byte,
-                       Node *&l_parent, idx_t l_pos) {
-
-	auto r_child = info.r_node->GetChild(*info.r_art, r_pos);
-
-	// insert child at empty position
-	if (l_child_pos == DConstants::INVALID_INDEX) {
-		Node::InsertChild(info.l_node, key_byte, r_child);
-		if (l_parent) {
-			l_parent->ReplaceChildPointer(l_pos, info.l_node);
-		}
-		info.r_node->ReplaceChildPointer(r_pos, nullptr);
-		return true;
-	}
-
-	// recurse
-	auto l_child = info.l_node->GetChild(*info.l_art, l_child_pos);
-	MergeInfo child_info(info.l_art, info.r_art, l_child, r_child);
-	ParentsOfNodes child_parents(info.l_node, l_child_pos, info.r_node, r_pos);
-	return ResolvePrefixesAndMerge(child_info, depth + 1, child_parents);
-}
-
 bool Node::MergeARTs(ART *l_art, ART *r_art) {
 
 	Node *null_parent = nullptr;

package/src/duckdb/src/execution/index/art/node16.cpp

@@ -47,6 +47,19 @@ idx_t Node16::GetNextPos(idx_t pos) {
 	return pos < count ? pos : DConstants::INVALID_INDEX;
 }
 
+idx_t Node16::GetNextPosAndByte(idx_t pos, uint8_t &byte) {
+	if (pos == DConstants::INVALID_INDEX) {
+		byte = key[0];
+		return 0;
+	}
+	pos++;
+	if (pos < count) {
+		byte = key[pos];
+		return pos;
+	}
+	return DConstants::INVALID_INDEX;
+}
+
 Node *Node16::GetChild(ART &art, idx_t pos) {
 	D_ASSERT(pos < count);
 	return children[pos].Unswizzle(art);
@@ -124,20 +137,6 @@ void Node16::EraseChild(Node *&node, int pos, ART &art) {
 	}
 }
 
-bool Node16::Merge(MergeInfo &info, idx_t depth, Node *&l_parent, idx_t l_pos) {
-
-	Node16 *r_n = (Node16 *)info.r_node;
-
-	for (idx_t i = 0; i < info.r_node->count; i++) {
-
-		auto l_child_pos = info.l_node->GetChildPos(r_n->key[i]);
-		if (!Node::MergeAtByte(info, depth, l_child_pos, i, r_n->key[i], l_parent, l_pos)) {
-			return false;
-		}
-	}
-	return true;
-}
-
 idx_t Node16::GetSize() {
 	return 16;
 }

package/src/duckdb/src/execution/index/art/node256.cpp

@@ -47,6 +47,16 @@ idx_t Node256::GetNextPos(idx_t pos) {
 	return Node::GetNextPos(pos);
 }
 
+idx_t Node256::GetNextPosAndByte(idx_t pos, uint8_t &byte) {
+	for (pos == DConstants::INVALID_INDEX ? pos = 0 : pos++; pos < 256; pos++) {
+		if (children[pos]) {
+			byte = uint8_t(pos);
+			return pos;
+		}
+	}
+	return Node::GetNextPos(pos);
+}
+
 Node *Node256::GetChild(ART &art, idx_t pos) {
 	return children[pos].Unswizzle(art);
 }
@@ -82,21 +92,6 @@ void Node256::EraseChild(Node *&node, int pos, ART &art) {
 	}
 }
 
-bool Node256::Merge(MergeInfo &info, idx_t depth, Node *&l_parent, idx_t l_pos) {
-
-	for (idx_t i = 0; i < 256; i++) {
-		if (info.r_node->GetChildPos(i) != DConstants::INVALID_INDEX) {
-
-			auto l_child_pos = info.l_node->GetChildPos(i);
-			auto key_byte = (uint8_t)i;
-			if (!Node::MergeAtByte(info, depth, l_child_pos, i, key_byte, l_parent, l_pos)) {
-				return false;
-			}
-		}
-	}
-	return true;
-}
-
 idx_t Node256::GetSize() {
 	return 256;
 }

package/src/duckdb/src/execution/index/art/node4.cpp

@@ -45,6 +45,19 @@ idx_t Node4::GetNextPos(idx_t pos) {
 	return pos < count ? pos : DConstants::INVALID_INDEX;
 }
 
+idx_t Node4::GetNextPosAndByte(idx_t pos, uint8_t &byte) {
+	if (pos == DConstants::INVALID_INDEX) {
+		byte = key[0];
+		return 0;
+	}
+	pos++;
+	if (pos < count) {
+		byte = key[pos];
+		return pos;
+	}
+	return DConstants::INVALID_INDEX;
+}
+
 Node *Node4::GetChild(ART &art, idx_t pos) {
 	D_ASSERT(pos < count);
 	return children[pos].Unswizzle(art);
@@ -117,20 +130,6 @@ void Node4::EraseChild(Node *&node, int pos, ART &art) {
 	}
 }
 
-bool Node4::Merge(MergeInfo &info, idx_t depth, Node *&l_parent, idx_t l_pos) {
-
-	Node4 *r_n = (Node4 *)info.r_node;
-
-	for (idx_t i = 0; i < info.r_node->count; i++) {
-
-		auto l_child_pos = info.l_node->GetChildPos(r_n->key[i]);
-		if (!Node::MergeAtByte(info, depth, l_child_pos, i, r_n->key[i], l_parent, l_pos)) {
-			return false;
-		}
-	}
-	return true;
-}
-
 idx_t Node4::GetSize() {
 	return 4;
 }

package/src/duckdb/src/execution/index/art/node48.cpp

@@ -51,6 +51,16 @@ idx_t Node48::GetNextPos(idx_t pos) {
 	return Node::GetNextPos(pos);
 }
 
+idx_t Node48::GetNextPosAndByte(idx_t pos, uint8_t &byte) {
+	for (pos == DConstants::INVALID_INDEX ? pos = 0 : pos++; pos < 256; pos++) {
+		if (child_index[pos] != Node::EMPTY_MARKER) {
+			byte = uint8_t(pos);
+			return pos;
+		}
+	}
+	return Node::GetNextPos(pos);
+}
+
 Node *Node48::GetChild(ART &art, idx_t pos) {
 	D_ASSERT(child_index[pos] != Node::EMPTY_MARKER);
 	return children[child_index[pos]].Unswizzle(art);
@@ -114,23 +124,6 @@ void Node48::EraseChild(Node *&node, int pos, ART &art) {
 	}
 }
 
-bool Node48::Merge(MergeInfo &info, idx_t depth, Node *&l_parent, idx_t l_pos) {
-
-	Node48 *r_n = (Node48 *)info.r_node;
-
-	for (idx_t i = 0; i < 256; i++) {
-		if (r_n->child_index[i] != Node::EMPTY_MARKER) {
-
-			auto l_child_pos = info.l_node->GetChildPos(i);
-			auto key_byte = (uint8_t)i;
-			if (!Node::MergeAtByte(info, depth, l_child_pos, i, key_byte, l_parent, l_pos)) {
-				return false;
-			}
-		}
-	}
-	return true;
-}
-
 idx_t Node48::GetSize() {
 	return 48;
 }

package/src/duckdb/src/execution/index/art/prefix.cpp

@@ -2,10 +2,6 @@
 
 namespace duckdb {
 
-uint32_t Prefix::Size() const {
-	return size;
-}
-
 bool Prefix::IsInlined() const {
 	return size <= PREFIX_INLINE_BYTES;
 }

package/src/duckdb/src/execution/operator/schema/physical_create_index.cpp

@@ -1,9 +1,6 @@
 #include "duckdb/execution/operator/schema/physical_create_index.hpp"
 
 #include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp"
-#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp"
-#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
-#include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/storage/storage_manager.hpp"
 #include "duckdb/main/database_manager.hpp"
@@ -22,20 +19,13 @@ public:
 
 class CreateIndexLocalSinkState : public LocalSinkState {
 public:
-	explicit CreateIndexLocalSinkState(const vector<unique_ptr<Expression>> &expressions) : executor(expressions) {
-	}
+	explicit CreateIndexLocalSinkState(ClientContext &context) : arena_allocator(Allocator::Get(context)) {};
 
-	//! Local indexes build from chunks of the scanned data
 	unique_ptr<Index> local_index;
-
+	ArenaAllocator arena_allocator;
+	vector<Key> keys;
 	DataChunk key_chunk;
-	unique_ptr<GlobalSortState> global_sort_state;
-	LocalSortState local_sort_state;
-
-	RowLayout payload_layout;
-	vector<LogicalType> payload_types;
-
-	ExpressionExecutor executor;
+	vector<column_t> key_column_ids;
 };
 
 unique_ptr<GlobalSinkState> PhysicalCreateIndex::GetGlobalSinkState(ClientContext &context) const {
@@ -51,14 +41,12 @@ unique_ptr<GlobalSinkState> PhysicalCreateIndex::GetGlobalSinkState(ClientContex
 	default:
 		throw InternalException("Unimplemented index type");
 	}
-
 	return (std::move(state));
 }
 
 unique_ptr<LocalSinkState> PhysicalCreateIndex::GetLocalSinkState(ExecutionContext &context) const {
-	auto &allocator = Allocator::Get(table.storage->db);
 
-	auto state = make_unique<CreateIndexLocalSinkState>(expressions);
+	auto state = make_unique<CreateIndexLocalSinkState>(context.client);
 
 	// create the local index
 	switch (info->index_type) {
@@ -70,82 +58,49 @@ unique_ptr<LocalSinkState> PhysicalCreateIndex::GetLocalSinkState(ExecutionConte
 	default:
 		throw InternalException("Unimplemented index type");
 	}
+	state->keys = vector<Key>(STANDARD_VECTOR_SIZE);
+	state->key_chunk.Initialize(Allocator::Get(context.client), state->local_index->logical_types);
 
-	state->key_chunk.Initialize(allocator, state->local_index->logical_types);
-
-	// ordering of the entries of the index
-	vector<BoundOrderByNode> orders;
-	for (idx_t i = 0; i < state->local_index->logical_types.size(); i++) {
-		auto col_expr = make_unique_base<Expression, BoundReferenceExpression>(state->local_index->logical_types[i], i);
-		orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_FIRST, std::move(col_expr));
+	for (idx_t i = 0; i < state->key_chunk.ColumnCount(); i++) {
+		state->key_column_ids.push_back(i);
 	}
-
-	// row layout of the global sort state
-	state->payload_types = state->local_index->logical_types;
-	state->payload_types.emplace_back(LogicalType::ROW_TYPE);
-	state->payload_layout.Initialize(state->payload_types);
-
-	// initialize global and local sort state
-	auto &buffer_manager = BufferManager::GetBufferManager(table.storage->db);
-	state->global_sort_state = make_unique<GlobalSortState>(buffer_manager, orders, state->payload_layout);
-	state->local_sort_state.Initialize(*state->global_sort_state, buffer_manager);
-
 	return std::move(state);
 }
 
 SinkResultType PhysicalCreateIndex::Sink(ExecutionContext &context, GlobalSinkState &gstate_p, LocalSinkState &lstate_p,
                                          DataChunk &input) const {
 
-	// here, we sink all incoming data into the local_sink_state after executing the expression executor
+	D_ASSERT(input.ColumnCount() >= 2);
 	auto &lstate = (CreateIndexLocalSinkState &)lstate_p;
+	auto &row_identifiers = input.data[input.ColumnCount() - 1];
 
-	// resolve the expressions for this chunk
-	D_ASSERT(!lstate.executor.HasContext());
-	lstate.key_chunk.Reset();
-	lstate.executor.Execute(input, lstate.key_chunk);
-
-	// create the payload chunk
-	DataChunk payload_chunk;
-	payload_chunk.InitializeEmpty(lstate.payload_types);
-	for (idx_t i = 0; i < lstate.local_index->logical_types.size(); i++) {
-		payload_chunk.data[i].Reference(lstate.key_chunk.data[i]);
-	}
-	payload_chunk.data[lstate.payload_types.size() - 1].Reference(input.data[input.ColumnCount() - 1]);
-	payload_chunk.SetCardinality(input.size());
+	// generate the keys for the given input
+	lstate.key_chunk.ReferenceColumns(input, lstate.key_column_ids);
+	lstate.arena_allocator.Reset();
+	ART::GenerateKeys(lstate.arena_allocator, lstate.key_chunk, lstate.keys);
 
-	// sink the chunks into the local sort state
-	lstate.local_sort_state.SinkChunk(lstate.key_chunk, payload_chunk);
+	auto art = make_unique<ART>(lstate.local_index->column_ids, lstate.local_index->table_io_manager,
+	                            lstate.local_index->unbound_expressions, lstate.local_index->constraint_type,
+	                            table.storage->db);
+	art->ConstructFromSorted(lstate.key_chunk.size(), lstate.keys, row_identifiers);
 
+	// merge into the local ART
+	{
+		IndexLock local_lock;
+		lstate.local_index->InitializeLock(local_lock);
+		if (!lstate.local_index->MergeIndexes(local_lock, art.get())) {
+			throw ConstraintException("Data contains duplicates on indexed column(s)");
+		}
+	}
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
 void PhysicalCreateIndex::Combine(ExecutionContext &context, GlobalSinkState &gstate_p,
                                   LocalSinkState &lstate_p) const {
 
-	// here, we take the sunk data chunks and sort them
-	// then, we scan the sorted data and build a local index from it
-	// finally, we merge the local index into the global index
-
 	auto &gstate = (CreateIndexGlobalSinkState &)gstate_p;
 	auto &lstate = (CreateIndexLocalSinkState &)lstate_p;
 
-	auto &allocator = Allocator::Get(table.storage->db);
-
-	// add local state to global state, which sorts the data
-	lstate.global_sort_state->AddLocalState(lstate.local_sort_state);
-	lstate.global_sort_state->PrepareMergePhase();
-
-	// scan the sorted row data and construct the index from it
-	{
-		IndexLock local_lock;
-		lstate.local_index->InitializeLock(local_lock);
-		if (!lstate.global_sort_state->sorted_blocks.empty()) {
-			PayloadScanner scanner(*lstate.global_sort_state->sorted_blocks[0]->payload_data,
-			                       *lstate.global_sort_state);
-			lstate.local_index->ConstructAndMerge(local_lock, scanner, allocator);
-		}
-	}
-
 	// merge the local index into the global index
 	gstate.global_index->MergeIndexes(lstate.local_index.get());
 }

package/src/duckdb/src/execution/physical_plan/plan_create_index.cpp

@@ -1,7 +1,9 @@
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
+#include "duckdb/execution/operator/projection/physical_projection.hpp"
 #include "duckdb/execution/operator/filter/physical_filter.hpp"
 #include "duckdb/execution/operator/scan/physical_table_scan.hpp"
 #include "duckdb/execution/operator/schema/physical_create_index.hpp"
+#include "duckdb/execution/operator/order/physical_order.hpp"
 #include "duckdb/execution/physical_plan_generator.hpp"
 #include "duckdb/function/table/table_scan.hpp"
 #include "duckdb/planner/filter/null_filter.hpp"
@@ -12,9 +14,26 @@ namespace duckdb {
 
 unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCreateIndex &op) {
 
+	// generate a physical plan for the parallel index creation which consists of the following operators
+	// table scan - projection (for expression execution) - filter (NOT NULL) - order - create index
+
 	D_ASSERT(op.children.empty());
 
+	// validate that all expressions contain valid scalar functions
+	// e.g. get_current_timestamp(), random(), and sequence values are not allowed as ART keys
+	// because they make deletions and lookups unfeasible
+	for (idx_t i = 0; i < op.unbound_expressions.size(); i++) {
+		auto &expr = op.unbound_expressions[i];
+		if (expr->expression_class == ExpressionClass::BOUND_FUNCTION) {
+			auto &func_expr = (BoundFunctionExpression &)*expr;
+			if (func_expr.function.side_effects == FunctionSideEffects::HAS_SIDE_EFFECTS) {
+				throw BinderException("Index keys cannot contain the \"%s\" function.", func_expr.function.name);
+			}
+		}
+	}
+
 	// table scan operator for index key columns and row IDs
+
 	unique_ptr<TableFilterSet> table_filters;
 	op.info->column_ids.emplace_back(COLUMN_IDENTIFIER_ROW_ID);
 
@@ -31,30 +50,61 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCreateInde
 	D_ASSERT(op.info->scan_types.size() - 1 <= op.info->names.size());
 	D_ASSERT(op.info->scan_types.size() - 1 <= op.info->column_ids.size());
 
+	// projection to execute expressions on the key columns
+
+	vector<LogicalType> new_column_types;
+	vector<unique_ptr<Expression>> select_list;
+	for (idx_t i = 0; i < op.expressions.size(); i++) {
+		new_column_types.push_back(op.expressions[i]->return_type);
+		select_list.push_back(std::move(op.expressions[i]));
+	}
+	new_column_types.emplace_back(LogicalType::ROW_TYPE);
+	select_list.push_back(make_unique<BoundReferenceExpression>(LogicalType::ROW_TYPE, op.info->scan_types.size() - 1));
+
+	auto projection =
+	    make_unique<PhysicalProjection>(new_column_types, std::move(select_list), op.estimated_cardinality);
+	projection->children.push_back(std::move(table_scan));
+
 	// filter operator for IS_NOT_NULL on each key column
+
 	vector<LogicalType> filter_types;
 	vector<unique_ptr<Expression>> filter_select_list;
 
-	for (idx_t i = 0; i < op.info->scan_types.size() - 1; i++) {
-		filter_types.push_back(op.info->scan_types[i]);
+	for (idx_t i = 0; i < new_column_types.size() - 1; i++) {
+		filter_types.push_back(new_column_types[i]);
 		auto is_not_null_expr =
 		    make_unique<BoundOperatorExpression>(ExpressionType::OPERATOR_IS_NOT_NULL, LogicalType::BOOLEAN);
-		auto bound_ref =
-		    make_unique<BoundReferenceExpression>(op.info->names[op.info->column_ids[i]], op.info->scan_types[i], i);
+		auto bound_ref = make_unique<BoundReferenceExpression>(new_column_types[i], i);
 		is_not_null_expr->children.push_back(std::move(bound_ref));
 		filter_select_list.push_back(std::move(is_not_null_expr));
 	}
 
 	auto null_filter =
-	    make_unique<PhysicalFilter>(std::move(filter_types), std::move(filter_select_list), STANDARD_VECTOR_SIZE);
+	    make_unique<PhysicalFilter>(std::move(filter_types), std::move(filter_select_list), op.estimated_cardinality);
 	null_filter->types.emplace_back(LogicalType::ROW_TYPE);
-	null_filter->children.push_back(std::move(table_scan));
+	null_filter->children.push_back(std::move(projection));
+
+	// order operator
+
+	vector<BoundOrderByNode> orders;
+	vector<idx_t> projections;
+	for (idx_t i = 0; i < new_column_types.size() - 1; i++) {
+		auto col_expr = make_unique_base<Expression, BoundReferenceExpression>(new_column_types[i], i);
+		orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_FIRST, std::move(col_expr));
+		projections.emplace_back(i);
+	}
+	projections.emplace_back(new_column_types.size() - 1);
+
+	auto physical_order = make_unique<PhysicalOrder>(new_column_types, std::move(orders), std::move(projections),
+	                                                 op.estimated_cardinality);
+	physical_order->children.push_back(std::move(null_filter));
 
 	// actual physical create index operator
-	auto physical_create_index = make_unique<PhysicalCreateIndex>(
-	    op, op.table, op.info->column_ids, std::move(op.expressions), std::move(op.info),
-	    std::move(op.unbound_expressions), op.estimated_cardinality);
-	physical_create_index->children.push_back(std::move(null_filter));
+
+	auto physical_create_index =
+	    make_unique<PhysicalCreateIndex>(op, op.table, op.info->column_ids, std::move(op.info),
+	                                     std::move(op.unbound_expressions), op.estimated_cardinality);
+	physical_create_index->children.push_back(std::move(physical_order));
 	return std::move(physical_create_index);
 }
 

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.6.2-dev1070"
+#define DUCKDB_VERSION "0.6.2-dev1092"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "e0cff6a3dd"
+#define DUCKDB_SOURCE_ID "123b6e7bbd"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp

@@ -49,6 +49,7 @@ enum VerifyExistenceType : uint8_t {
 
 class ART : public Index {
 public:
+	//! Constructs an ART containing the bound expressions, which are resolved during index construction
 	ART(const vector<column_t> &column_ids, TableIOManager &table_io_manager,
 	    const vector<unique_ptr<Expression>> &unbound_expressions, IndexConstraintType constraint_type,
 	    AttachedDatabase &db, idx_t block_id = DConstants::INVALID_INDEX,
@@ -57,7 +58,7 @@ public:
 
 	//! Root of the tree
 	Node *tree;
-
+	//! Attached database
 	AttachedDatabase &db;
 
 public:
@@ -88,8 +89,7 @@ public:
 	//! Insert data into the index.
 	bool Insert(IndexLock &lock, DataChunk &data, Vector &row_ids) override;
 
-	//! Construct ARTs from sorted chunks and merge them.
-	void ConstructAndMerge(IndexLock &lock, PayloadScanner &scanner, Allocator &allocator) override;
+	void ConstructFromSorted(idx_t count, vector<Key> &keys, Vector &row_identifiers);
 
 	//! Search Equal and fetches the row IDs
 	bool SearchEqual(Key &key, idx_t max_count, vector<row_t> &result_ids);

package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp

@@ -88,6 +88,11 @@ public:
 	virtual idx_t GetNextPos(idx_t pos) {
 		return DConstants::INVALID_INDEX;
 	}
+	//! Get the next position and byte in the node, or DConstants::INVALID_INDEX if there is no next position. if pos ==
+	//! DConstants::INVALID_INDEX, then the first valid position in the node is returned
+	virtual idx_t GetNextPosAndByte(idx_t pos, uint8_t &byte) {
+		return DConstants::INVALID_INDEX;
+	}
 	//! Get the child at the specified position in the node. pos should be between [0, count). Throws an assertion if
 	//! the element is not found
 	virtual Node *GetChild(ART &art, idx_t pos);
@@ -110,9 +115,6 @@ public:
 
 	//! Deserialize this node
 	static Node *Deserialize(ART &art, idx_t block_id, idx_t offset);
-	//! Merge r_node into l_node at the specified byte
-	static bool MergeAtByte(MergeInfo &info, idx_t depth, idx_t &l_child_pos, idx_t &r_pos, uint8_t &key_byte,
-	                        Node *&l_parent, idx_t l_pos);
 	//! Merge two ART
 	static bool MergeARTs(ART *l_art, ART *r_art);
 

package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp

@@ -29,6 +29,8 @@ public:
 	idx_t GetMin() override;
 	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position
 	idx_t GetNextPos(idx_t pos) override;
+	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position
+	idx_t GetNextPosAndByte(idx_t pos, uint8_t &byte) override;
 	//! Get Node16 child
 	Node *GetChild(ART &art, idx_t pos) override;
 	//! Replace child pointer
@@ -38,8 +40,6 @@ public:
 	static void InsertChild(Node *&node, uint8_t key_byte, Node *new_child);
 	//! Erase the child at pos and (if necessary) shrink to Node4
 	static void EraseChild(Node *&node, int pos, ART &art);
-	//! Merge Node16 into l_node
-	static bool Merge(MergeInfo &info, idx_t depth, Node *&l_parent, idx_t l_pos);
 	//! Returns the size (maximum capacity) of the Node16
 	static idx_t GetSize();
 };

package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp

@@ -28,6 +28,8 @@ public:
 	idx_t GetMin() override;
 	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position
 	idx_t GetNextPos(idx_t pos) override;
+	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position
+	idx_t GetNextPosAndByte(idx_t pos, uint8_t &byte) override;
 	//! Get Node256 child
 	Node *GetChild(ART &art, idx_t pos) override;
 	//! Replace child pointer
@@ -37,8 +39,6 @@ public:
 	static void InsertChild(Node *&node, uint8_t key_byte, Node *new_child);
 	//! Erase the child at pos and (if necessary) shrink to Node48
 	static void EraseChild(Node *&node, int pos, ART &art);
-	//! Merge Node256 into l_node
-	static bool Merge(MergeInfo &info, idx_t depth, Node *&l_parent, idx_t l_pos);
 	//! Returns the size (maximum capacity) of the Node256
 	static idx_t GetSize();
 };

package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp

@@ -31,6 +31,8 @@ public:
 	idx_t GetMin() override;
 	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position
 	idx_t GetNextPos(idx_t pos) override;
+	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position
+	idx_t GetNextPosAndByte(idx_t pos, uint8_t &byte) override;
 	//! Get Node4 child
 	Node *GetChild(ART &art, idx_t pos) override;
 	//! Replace child pointer
@@ -40,8 +42,6 @@ public:
 	static void InsertChild(Node *&node, uint8_t key_byte, Node *new_child);
 	//! Erase the child at pos and (if necessary) merge with last child
 	static void EraseChild(Node *&node, int pos, ART &art);
-	//! Merge Node4 into l_node
-	static bool Merge(MergeInfo &info, idx_t depth, Node *&l_parent, idx_t l_pos);
 	//! Returns the size (maximum capacity) of the Node4
 	static idx_t GetSize();
 };

package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp

@@ -29,6 +29,8 @@ public:
 	idx_t GetMin() override;
 	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position
 	idx_t GetNextPos(idx_t pos) override;
+	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position
+	idx_t GetNextPosAndByte(idx_t pos, uint8_t &byte) override;
 	//! Get Node48 child
 	Node *GetChild(ART &art, idx_t pos) override;
 	//! Replace child pointer
@@ -38,8 +40,6 @@ public:
 	static void InsertChild(Node *&node, uint8_t key_byte, Node *new_child);
 	//! Erase the child at pos and (if necessary) shrink to Node16
 	static void EraseChild(Node *&node, int pos, ART &art);
-	//! Merge Node48 into l_node
-	static bool Merge(MergeInfo &info, idx_t depth, Node *&l_parent, idx_t l_pos);
 	//! Returns the size (maximum capacity) of the Node48
 	static idx_t GetSize();
 };

package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp

@@ -23,8 +23,10 @@ public:
 	Prefix(Prefix &other_prefix, uint32_t size);
 	~Prefix();
 
-	// Returns the Prefix's size
-	uint32_t Size() const;
+	//! Returns the Prefix's size
+	inline uint32_t Size() const {
+		return size;
+	}
 	//! Return a pointer to the prefix data
 	uint8_t *GetPrefixData();
 	const uint8_t *GetPrefixData() const;

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_index.hpp

@@ -18,15 +18,14 @@
 
 namespace duckdb {
 
-//! Physically CREATE INDEX statement
+//! Physical CREATE (UNIQUE) INDEX statement
 class PhysicalCreateIndex : public PhysicalOperator {
 public:
 	PhysicalCreateIndex(LogicalOperator &op, TableCatalogEntry &table, vector<column_t> column_ids,
-	                    vector<unique_ptr<Expression>> expressions, unique_ptr<CreateIndexInfo> info,
-	                    vector<unique_ptr<Expression>> unbound_expressions, idx_t estimated_cardinality)
+	                    unique_ptr<CreateIndexInfo> info, vector<unique_ptr<Expression>> unbound_expressions,
+	                    idx_t estimated_cardinality)
 	    : PhysicalOperator(PhysicalOperatorType::CREATE_INDEX, op.types, estimated_cardinality), table(table),
-	      expressions(std::move(expressions)), info(std::move(info)),
-	      unbound_expressions(std::move(unbound_expressions)) {
+	      info(std::move(info)), unbound_expressions(std::move(unbound_expressions)) {
 
 		// convert virtual column ids to storage column ids
 		for (auto &column_id : column_ids) {
@@ -38,21 +37,20 @@ public:
 	TableCatalogEntry &table;
 	//! The list of column IDs required for the index
 	vector<column_t> storage_ids;
-	//! Set of expressions to index by
-	vector<unique_ptr<Expression>> expressions;
 	//! Info for index creation
 	unique_ptr<CreateIndexInfo> info;
 	//! Unbound expressions to be used in the optimizer
 	vector<unique_ptr<Expression>> unbound_expressions;
 
 public:
-	// Source interface
+	//! Source interface, NOP for this operator
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
 public:
-	// Sink interface
+	//! Sink interface, thread-local sink states
 	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
+	//! Sink interface, global sink state
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
 
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &gstate_p, LocalSinkState &lstate_p,

package/src/duckdb/src/include/duckdb/main/prepared_statement.hpp

@@ -60,6 +60,8 @@ public:
 	DUCKDB_API const vector<LogicalType> &GetTypes();
 	//! Returns the result names of the prepared statement
 	DUCKDB_API const vector<string> &GetNames();
+	//! Returns the map of parameter index to the expected type of parameter
+	DUCKDB_API vector<LogicalType> GetExpectedParameterTypes() const;
 
 	//! Create a pending query result of the prepared statement with the given set of arguments
 	template <typename... Args>

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -39,7 +39,7 @@ public:
 	TableIOManager &table_io_manager;
 	//! Column identifiers to extract from the base table
 	vector<column_t> column_ids;
-	//! unordered_set of column_ids used by the index
+	//! Unordered_set of column_ids used by the index
 	unordered_set<column_t> column_id_set;
 	//! Unbound expressions used by the index
 	vector<unique_ptr<Expression>> unbound_expressions;
@@ -47,7 +47,7 @@ public:
 	vector<PhysicalType> types;
 	//! The logical types of the expressions
 	vector<LogicalType> logical_types;
-	//! constraint type
+	//! Index constraint type (primary key, foreign key, ...)
 	IndexConstraintType constraint_type;
 
 public:
@@ -83,8 +83,6 @@ public:
 
 	//! Insert data into the index. Does not lock the index.
 	virtual bool Insert(IndexLock &lock, DataChunk &input, Vector &row_identifiers) = 0;
-	//! Construct an index from sorted chunks of keys.
-	virtual void ConstructAndMerge(IndexLock &lock, PayloadScanner &scanner, Allocator &allocator) = 0;
 
 	//! Merge other_index into this index.
 	virtual bool MergeIndexes(IndexLock &state, Index *other_index) = 0;

package/src/duckdb/src/main/prepared_statement.cpp

@@ -52,6 +52,19 @@ const vector<string> &PreparedStatement::GetNames() {
 	return data->names;
 }
 
+vector<LogicalType> PreparedStatement::GetExpectedParameterTypes() const {
+	D_ASSERT(data);
+	vector<LogicalType> expected_types(data->value_map.size());
+	for (auto &it : data->value_map) {
+		D_ASSERT(it.first >= 1);
+		idx_t param_index = it.first - 1;
+		D_ASSERT(param_index < expected_types.size());
+		D_ASSERT(it.second);
+		expected_types[param_index] = it.second->value.type();
+	}
+	return expected_types;
+}
+
 unique_ptr<QueryResult> PreparedStatement::Execute(vector<Value> &values, bool allow_stream_result) {
 	auto pending = PendingQuery(values, allow_stream_result);
 	if (pending->HasError()) {

package/src/duckdb/src/storage/index.cpp

@@ -10,7 +10,9 @@ namespace duckdb {
 
 Index::Index(IndexType type, TableIOManager &table_io_manager, const vector<column_t> &column_ids_p,
              const vector<unique_ptr<Expression>> &unbound_expressions, IndexConstraintType constraint_type_p)
+
     : type(type), table_io_manager(table_io_manager), column_ids(column_ids_p), constraint_type(constraint_type_p) {
+
 	for (auto &expr : unbound_expressions) {
 		types.push_back(expr->return_type.InternalType());
 		logical_types.push_back(expr->return_type);