duckdb 0.8.2-dev37.0 → 0.8.2-dev3989.0

package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp

@@ -1,9 +1,11 @@
 #include "duckdb/execution/operator/persistent/physical_batch_copy_to_file.hpp"
+
+#include "duckdb/common/allocator.hpp"
+#include "duckdb/common/types/batched_data_collection.hpp"
+#include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/execution/operator/persistent/physical_copy_to_file.hpp"
 #include "duckdb/parallel/base_pipeline_event.hpp"
-#include "duckdb/common/vector_operations/vector_operations.hpp"
-#include "duckdb/common/types/batched_data_collection.hpp"
-#include "duckdb/common/allocator.hpp"
+
 #include <algorithm>
 
 namespace duckdb {
@@ -67,7 +69,7 @@ public:
 	optional_idx batch_index;
 
 	void InitializeCollection(ClientContext &context, const PhysicalOperator &op) {
-		collection = make_uniq<ColumnDataCollection>(Allocator::Get(context), op.children[0]->types);
+		collection = make_uniq<ColumnDataCollection>(BufferAllocator::Get(context), op.children[0]->types);
 		collection->InitializeAppend(append_state);
 	}
 };
@@ -87,11 +89,13 @@ SinkResultType PhysicalBatchCopyToFile::Sink(ExecutionContext &context, DataChun
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
-void PhysicalBatchCopyToFile::Combine(ExecutionContext &context, GlobalSinkState &gstate_p,
-                                      LocalSinkState &lstate) const {
-	auto &state = lstate.Cast<BatchCopyToLocalState>();
-	auto &gstate = gstate_p.Cast<BatchCopyToGlobalState>();
+SinkCombineResultType PhysicalBatchCopyToFile::Combine(ExecutionContext &context,
+                                                       OperatorSinkCombineInput &input) const {
+	auto &state = input.local_state.Cast<BatchCopyToLocalState>();
+	auto &gstate = input.global_state.Cast<BatchCopyToGlobalState>();
 	gstate.rows_copied += state.rows_copied;
+
+	return SinkCombineResultType::FINISHED;
 }
 
 //===--------------------------------------------------------------------===//
@@ -112,8 +116,8 @@ SinkFinalizeType PhysicalBatchCopyToFile::FinalFlush(ClientContext &context, Glo
 }
 
 SinkFinalizeType PhysicalBatchCopyToFile::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-                                                   GlobalSinkState &gstate_p) const {
-	FinalFlush(context, gstate_p);
+                                                   OperatorSinkFinalizeInput &input) const {
+	FinalFlush(context, input.global_state);
 	return SinkFinalizeType::READY;
 }
 

package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp

@@ -327,10 +327,11 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, DataChunk &c
 		// no collection yet: create a new one
 		lstate.CreateNewCollection(table, insert_types);
 		lstate.writer = &table.GetStorage().CreateOptimisticWriter(context.client);
-	} else if (lstate.current_index != batch_index) {
+	}
+
+	if (lstate.current_index != batch_index) {
 		throw InternalException("Current batch differs from batch - but NextBatch was not called!?");
 	}
-	lstate.current_index = batch_index;
 
 	table.GetStorage().VerifyAppendConstraints(table, context.client, lstate.insert_chunk);
 
@@ -343,16 +344,15 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, DataChunk &c
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
-void PhysicalBatchInsert::Combine(ExecutionContext &context, GlobalSinkState &gstate_p,
-                                  LocalSinkState &lstate_p) const {
-	auto &gstate = gstate_p.Cast<BatchInsertGlobalState>();
-	auto &lstate = lstate_p.Cast<BatchInsertLocalState>();
+SinkCombineResultType PhysicalBatchInsert::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
+	auto &gstate = input.global_state.Cast<BatchInsertGlobalState>();
+	auto &lstate = input.local_state.Cast<BatchInsertLocalState>();
 	auto &client_profiler = QueryProfiler::Get(context.client);
 	context.thread.profiler.Flush(*this, lstate.default_executor, "default_executor", 1);
 	client_profiler.Flush(context.thread.profiler);
 
 	if (!lstate.current_collection) {
-		return;
+		return SinkCombineResultType::FINISHED;
 	}
 
 	if (lstate.current_collection->GetTotalRows() > 0) {
@@ -365,11 +365,13 @@ void PhysicalBatchInsert::Combine(ExecutionContext &context, GlobalSinkState &gs
 		lock_guard<mutex> l(gstate.lock);
 		gstate.table.GetStorage().FinalizeOptimisticWriter(context.client, *lstate.writer);
 	}
+
+	return SinkCombineResultType::FINISHED;
 }
 
 SinkFinalizeType PhysicalBatchInsert::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-                                               GlobalSinkState &gstate_p) const {
-	auto &gstate = gstate_p.Cast<BatchInsertGlobalState>();
+                                               OperatorSinkFinalizeInput &input) const {
+	auto &gstate = input.global_state.Cast<BatchInsertGlobalState>();
 
 	// in the finalize, do a final pass over all of the collections we created and try to merge smaller collections
 	// together

package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp

@@ -97,9 +97,9 @@ static string CreateDirRecursive(const vector<idx_t> &cols, const vector<string>
 	return path;
 }
 
-void PhysicalCopyToFile::Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const {
-	auto &g = gstate.Cast<CopyToFunctionGlobalState>();
-	auto &l = lstate.Cast<CopyToFunctionLocalState>();
+SinkCombineResultType PhysicalCopyToFile::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
+	auto &g = input.global_state.Cast<CopyToFunctionGlobalState>();
+	auto &l = input.local_state.Cast<CopyToFunctionLocalState>();
 
 	if (partition_output) {
 		auto &fs = FileSystem::GetFileSystem(context.client);
@@ -108,7 +108,7 @@ void PhysicalCopyToFile::Combine(ExecutionContext &context, GlobalSinkState &gst
 		auto partition_key_map = l.part_buffer->GetReverseMap();
 
 		string trimmed_path = file_path;
-		StringUtil::RTrim(trimmed_path, fs.PathSeparator());
+		StringUtil::RTrim(trimmed_path, fs.PathSeparator(trimmed_path));
 
 		for (idx_t i = 0; i < partitions.size(); i++) {
 			string hive_path =
@@ -130,7 +130,7 @@ void PhysicalCopyToFile::Combine(ExecutionContext &context, GlobalSinkState &gst
 			function.copy_to_finalize(context.client, *bind_data, *fun_data_global);
 		}
 
-		return;
+		return SinkCombineResultType::FINISHED;
 	}
 
 	if (function.copy_to_combine) {
@@ -141,11 +141,13 @@ void PhysicalCopyToFile::Combine(ExecutionContext &context, GlobalSinkState &gst
 			function.copy_to_finalize(context.client, *bind_data, *l.global_state);
 		}
 	}
+
+	return SinkCombineResultType::FINISHED;
 }
 
 SinkFinalizeType PhysicalCopyToFile::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-                                              GlobalSinkState &gstate_p) const {
-	auto &gstate = gstate_p.Cast<CopyToFunctionGlobalState>();
+                                              OperatorSinkFinalizeInput &input) const {
+	auto &gstate = input.global_state.Cast<CopyToFunctionGlobalState>();
 	if (per_thread_output || partition_output) {
 		// already happened in combine
 		return SinkFinalizeType::READY;

package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp

@@ -116,7 +116,7 @@ public:
 	optional_idx batch_index;
 
 	void InitializeCollection(ClientContext &context, const PhysicalOperator &op) {
-		collection = make_uniq<ColumnDataCollection>(Allocator::Get(context), op.children[0]->types);
+		collection = make_uniq<ColumnDataCollection>(BufferAllocator::Get(context), op.children[0]->types);
 		collection->InitializeAppend(append_state);
 	}
 };
@@ -136,10 +136,10 @@ SinkResultType PhysicalFixedBatchCopy::Sink(ExecutionContext &context, DataChunk
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
-void PhysicalFixedBatchCopy::Combine(ExecutionContext &context, GlobalSinkState &gstate_p,
-                                     LocalSinkState &lstate) const {
-	auto &state = lstate.Cast<FixedBatchCopyLocalState>();
-	auto &gstate = gstate_p.Cast<FixedBatchCopyGlobalState>();
+SinkCombineResultType PhysicalFixedBatchCopy::Combine(ExecutionContext &context,
+                                                      OperatorSinkCombineInput &input) const {
+	auto &state = input.local_state.Cast<FixedBatchCopyLocalState>();
+	auto &gstate = input.global_state.Cast<FixedBatchCopyGlobalState>();
 	gstate.rows_copied += state.rows_copied;
 	if (!gstate.any_finished) {
 		// signal that this thread is finished processing batches and that we should move on to Finalize
@@ -147,6 +147,8 @@ void PhysicalFixedBatchCopy::Combine(ExecutionContext &context, GlobalSinkState
 		gstate.any_finished = true;
 	}
 	ExecuteTasks(context.client, gstate);
+
+	return SinkCombineResultType::FINISHED;
 }
 
 //===--------------------------------------------------------------------===//
@@ -225,16 +227,16 @@ SinkFinalizeType PhysicalFixedBatchCopy::FinalFlush(ClientContext &context, Glob
 }
 
 SinkFinalizeType PhysicalFixedBatchCopy::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-                                                  GlobalSinkState &gstate_p) const {
-	auto &gstate = gstate_p.Cast<FixedBatchCopyGlobalState>();
+                                                  OperatorSinkFinalizeInput &input) const {
+	auto &gstate = input.global_state.Cast<FixedBatchCopyGlobalState>();
 	idx_t min_batch_index = idx_t(NumericLimits<int64_t>::Maximum());
 	// repartition any remaining batches
-	RepartitionBatches(context, gstate_p, min_batch_index, true);
+	RepartitionBatches(context, input.global_state, min_batch_index, true);
 	// check if we have multiple tasks to execute
 	if (gstate.TaskCount() <= 1) {
 		// we don't - just execute the remaining task and finish flushing to disk
-		ExecuteTasks(context, gstate_p);
-		FinalFlush(context, gstate_p);
+		ExecuteTasks(context, input.global_state);
+		FinalFlush(context, input.global_state);
 		return SinkFinalizeType::READY;
 	}
 	// we have multiple tasks remaining - launch an event to execute the tasks in parallel
@@ -353,7 +355,7 @@ void PhysicalFixedBatchCopy::RepartitionBatches(ClientContext &context, GlobalSi
 			} else {
 				// the collection is too large for a batch - we need to repartition
 				// create an empty collection
-				current_collection = make_uniq<ColumnDataCollection>(Allocator::Get(context), children[0]->types);
+				current_collection = make_uniq<ColumnDataCollection>(BufferAllocator::Get(context), children[0]->types);
 			}
 			if (current_collection) {
 				current_collection->InitializeAppend(append_state);
@@ -373,7 +375,7 @@ void PhysicalFixedBatchCopy::RepartitionBatches(ClientContext &context, GlobalSi
 			}
 			// the collection is full - move it to the result and create a new one
 			gstate.AddTask(make_uniq<PrepareBatchTask>(gstate.scheduled_batch_index++, std::move(current_collection)));
-			current_collection = make_uniq<ColumnDataCollection>(Allocator::Get(context), children[0]->types);
+			current_collection = make_uniq<ColumnDataCollection>(BufferAllocator::Get(context), children[0]->types);
 			current_collection->InitializeAppend(append_state);
 		}
 	}

package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp

@@ -41,7 +41,7 @@ PhysicalInsert::PhysicalInsert(vector<LogicalType> types_p, TableCatalogEntry &t
 		return;
 	}
 
-	D_ASSERT(set_expressions.size() == set_columns.size());
+	D_ASSERT(this->set_expressions.size() == this->set_columns.size());
 
 	// One or more columns are referenced from the existing table,
 	// we use the 'insert_types' to figure out which types these columns have
@@ -463,19 +463,17 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, DataChunk &chunk,
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
-void PhysicalInsert::Combine(ExecutionContext &context, GlobalSinkState &gstate_p, LocalSinkState &lstate_p) const {
-	auto &gstate = gstate_p.Cast<InsertGlobalState>();
-	auto &lstate = lstate_p.Cast<InsertLocalState>();
+SinkCombineResultType PhysicalInsert::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
+	auto &gstate = input.global_state.Cast<InsertGlobalState>();
+	auto &lstate = input.local_state.Cast<InsertLocalState>();
 	auto &client_profiler = QueryProfiler::Get(context.client);
 	context.thread.profiler.Flush(*this, lstate.default_executor, "default_executor", 1);
 	client_profiler.Flush(context.thread.profiler);
 
-	if (!parallel) {
-		return;
-	}
-	if (!lstate.local_collection) {
-		return;
+	if (!parallel || !lstate.local_collection) {
+		return SinkCombineResultType::FINISHED;
 	}
+
 	// parallel append: finalize the append
 	TransactionData tdata(0, 0);
 	lstate.local_collection->FinalizeAppend(tdata, lstate.local_append_state);
@@ -500,11 +498,13 @@ void PhysicalInsert::Combine(ExecutionContext &context, GlobalSinkState &gstate_
 		gstate.table.GetStorage().FinalizeOptimisticWriter(context.client, *lstate.writer);
 		gstate.table.GetStorage().LocalMerge(context.client, *lstate.local_collection);
 	}
+
+	return SinkCombineResultType::FINISHED;
 }
 
 SinkFinalizeType PhysicalInsert::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-                                          GlobalSinkState &state) const {
-	auto &gstate = state.Cast<InsertGlobalState>();
+                                          OperatorSinkFinalizeInput &input) const {
+	auto &gstate = input.global_state.Cast<InsertGlobalState>();
 	if (!parallel && gstate.initialized) {
 		auto &table = gstate.table;
 		auto &storage = table.GetStorage();

package/src/duckdb/src/execution/operator/persistent/physical_update.cpp

@@ -140,11 +140,13 @@ unique_ptr<LocalSinkState> PhysicalUpdate::GetLocalSinkState(ExecutionContext &c
 	return make_uniq<UpdateLocalState>(context.client, expressions, table.GetTypes(), bound_defaults);
 }
 
-void PhysicalUpdate::Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const {
-	auto &state = lstate.Cast<UpdateLocalState>();
+SinkCombineResultType PhysicalUpdate::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
+	auto &state = input.local_state.Cast<UpdateLocalState>();
 	auto &client_profiler = QueryProfiler::Get(context.client);
 	context.thread.profiler.Flush(*this, state.default_executor, "default_executor", 1);
 	client_profiler.Flush(context.thread.profiler);
+
+	return SinkCombineResultType::FINISHED;
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp

@@ -16,6 +16,7 @@ PhysicalPivot::PhysicalPivot(vector<LogicalType> types_p, unique_ptr<PhysicalOpe
 		pivot_map[bound_pivot.pivot_values[p]] = bound_pivot.group_count + p;
 	}
 	// extract the empty aggregate expressions
+	ArenaAllocator allocator(Allocator::DefaultAllocator());
 	for (auto &aggr_expr : bound_pivot.aggregates) {
 		auto &aggr = aggr_expr->Cast<BoundAggregateExpression>();
 		// for each aggregate, initialize an empty aggregate state and finalize it immediately
@@ -23,7 +24,7 @@ PhysicalPivot::PhysicalPivot(vector<LogicalType> types_p, unique_ptr<PhysicalOpe
 		aggr.function.initialize(state.get());
 		Vector state_vector(Value::POINTER(CastPointerToValue(state.get())));
 		Vector result_vector(aggr_expr->return_type);
-		AggregateInputData aggr_input_data(aggr.bind_info.get(), Allocator::DefaultAllocator());
+		AggregateInputData aggr_input_data(aggr.bind_info.get(), allocator);
 		aggr.function.finalize(state_vector, aggr_input_data, result_vector, 1, 0);
 		empty_aggregates.push_back(result_vector.GetValue(0));
 	}

package/src/duckdb/src/execution/operator/projection/physical_unnest.cpp

@@ -308,40 +308,37 @@ OperatorResultType PhysicalUnnest::ExecuteInternal(ExecutionContext &context, Da
 				// UNNEST(NULL)
 				chunk.SetCardinality(0);
 				break;
+			}
 
-			} else {
-
-				auto &vector_data = state.list_vector_data[col_idx];
-				auto current_idx = vector_data.sel->get_index(state.current_row);
-
-				if (!vector_data.validity.RowIsValid(current_idx)) {
-					UnnestNull(0, this_chunk_len, result_vector);
+			auto &vector_data = state.list_vector_data[col_idx];
+			auto current_idx = vector_data.sel->get_index(state.current_row);
 
-				} else {
+			if (!vector_data.validity.RowIsValid(current_idx)) {
+				UnnestNull(0, this_chunk_len, result_vector);
+				continue;
+			}
 
-					auto list_data = UnifiedVectorFormat::GetData<list_entry_t>(vector_data);
-					auto list_entry = list_data[current_idx];
+			auto list_data = UnifiedVectorFormat::GetData<list_entry_t>(vector_data);
+			auto list_entry = list_data[current_idx];
 
-					idx_t list_count = 0;
-					if (state.list_position < list_entry.length) {
-						// there are still list_count elements to unnest
-						list_count = MinValue<idx_t>(this_chunk_len, list_entry.length - state.list_position);
+			idx_t list_count = 0;
+			if (state.list_position < list_entry.length) {
+				// there are still list_count elements to unnest
+				list_count = MinValue<idx_t>(this_chunk_len, list_entry.length - state.list_position);
 
-						auto &list_vector = state.list_data.data[col_idx];
-						auto &child_vector = ListVector::GetEntry(list_vector);
-						auto list_size = ListVector::GetListSize(list_vector);
-						auto &child_vector_data = state.list_child_data[col_idx];
+				auto &list_vector = state.list_data.data[col_idx];
+				auto &child_vector = ListVector::GetEntry(list_vector);
+				auto list_size = ListVector::GetListSize(list_vector);
+				auto &child_vector_data = state.list_child_data[col_idx];
 
-						auto base_offset = list_entry.offset + state.list_position;
-						UnnestVector(child_vector_data, child_vector, list_size, base_offset, base_offset + list_count,
-						             result_vector);
-					}
+				auto base_offset = list_entry.offset + state.list_position;
+				UnnestVector(child_vector_data, child_vector, list_size, base_offset, base_offset + list_count,
+				             result_vector);
+			}
 
-					// fill the rest with NULLs
-					if (list_count != this_chunk_len) {
-						UnnestNull(list_count, this_chunk_len, result_vector);
-					}
-				}
+			// fill the rest with NULLs
+			if (list_count != this_chunk_len) {
+				UnnestNull(list_count, this_chunk_len, result_vector);
 			}
 		}
 

package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp

@@ -64,6 +64,9 @@ void PhysicalColumnDataScan::BuildPipelines(Pipeline &current, MetaPipeline &met
 		state.SetPipelineSource(current, delim_join.distinct->Cast<PhysicalOperator>());
 		return;
 	}
+	case PhysicalOperatorType::CTE_SCAN: {
+		break;
+	}
 	case PhysicalOperatorType::RECURSIVE_CTE_SCAN:
 		if (!meta_pipeline.HasRecursiveCTE()) {
 			throw InternalException("Recursive CTE scan found without recursive CTE node");
@@ -76,4 +79,20 @@ void PhysicalColumnDataScan::BuildPipelines(Pipeline &current, MetaPipeline &met
 	state.SetPipelineSource(current, *this);
 }
 
+string PhysicalColumnDataScan::ParamsToString() const {
+	string result = "";
+	switch (type) {
+	case PhysicalOperatorType::CTE_SCAN:
+	case PhysicalOperatorType::RECURSIVE_CTE_SCAN: {
+		result += "\n[INFOSEPARATOR]\n";
+		result += StringUtil::Format("idx: %llu", cte_index);
+		break;
+	}
+	default:
+		break;
+	}
+
+	return result;
+}
+
 } // namespace duckdb

package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp

@@ -9,24 +9,15 @@
 
 namespace duckdb {
 
-PhysicalTableScan::PhysicalTableScan(vector<LogicalType> types, TableFunction function_p,
-                                     unique_ptr<FunctionData> bind_data_p, vector<column_t> column_ids_p,
-                                     vector<string> names_p, unique_ptr<TableFilterSet> table_filters_p,
-                                     idx_t estimated_cardinality)
-    : PhysicalOperator(PhysicalOperatorType::TABLE_SCAN, std::move(types), estimated_cardinality),
-      function(std::move(function_p)), bind_data(std::move(bind_data_p)), column_ids(std::move(column_ids_p)),
-      names(std::move(names_p)), table_filters(std::move(table_filters_p)) {
-}
-
 PhysicalTableScan::PhysicalTableScan(vector<LogicalType> types, TableFunction function_p,
                                      unique_ptr<FunctionData> bind_data_p, vector<LogicalType> returned_types_p,
                                      vector<column_t> column_ids_p, vector<idx_t> projection_ids_p,
                                      vector<string> names_p, unique_ptr<TableFilterSet> table_filters_p,
-                                     idx_t estimated_cardinality)
+                                     idx_t estimated_cardinality, ExtraOperatorInfo extra_info)
     : PhysicalOperator(PhysicalOperatorType::TABLE_SCAN, std::move(types), estimated_cardinality),
       function(std::move(function_p)), bind_data(std::move(bind_data_p)), returned_types(std::move(returned_types_p)),
       column_ids(std::move(column_ids_p)), projection_ids(std::move(projection_ids_p)), names(std::move(names_p)),
-      table_filters(std::move(table_filters_p)) {
+      table_filters(std::move(table_filters_p)), extra_info(extra_info) {
 }
 
 class TableScanGlobalSourceState : public GlobalSourceState {
@@ -149,8 +140,12 @@ string PhysicalTableScan::ParamsToString() const {
 			}
 		}
 	}
+	if (!extra_info.file_filters.empty()) {
+		result += "\n[INFOSEPARATOR]\n";
+		result += "File Filters: " + extra_info.file_filters;
+	}
 	result += "\n[INFOSEPARATOR]\n";
-	result += StringUtil::Format("EC: %llu", estimated_props->GetCardinality<idx_t>());
+	result += StringUtil::Format("EC: %llu", estimated_cardinality);
 	return result;
 }
 

package/src/duckdb/src/execution/operator/schema/physical_attach.cpp

@@ -66,7 +66,8 @@ SourceResultType PhysicalAttach::GetData(ExecutionContext &context, DataChunk &c
 	const auto &path = info->path;
 
 	if (name.empty()) {
-		name = AttachedDatabase::ExtractDatabaseName(path);
+		auto &fs = FileSystem::GetFileSystem(context.client);
+		name = AttachedDatabase::ExtractDatabaseName(path, fs);
 	}
 	auto &db_manager = DatabaseManager::Get(context.client);
 	auto existing_db = db_manager.GetDatabaseFromPath(context.client, path);

package/src/duckdb/src/execution/operator/schema/physical_create_art_index.cpp

@@ -0,0 +1,198 @@
+#include "duckdb/execution/operator/schema/physical_create_art_index.hpp"
+
+#include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
+#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
+#include "duckdb/catalog/catalog_entry/duck_index_entry.hpp"
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/storage/index.hpp"
+#include "duckdb/storage/storage_manager.hpp"
+#include "duckdb/storage/table/append_state.hpp"
+#include "duckdb/main/database_manager.hpp"
+#include "duckdb/execution/index/art/art_key.hpp"
+#include "duckdb/execution/index/art/node.hpp"
+#include "duckdb/execution/index/art/leaf.hpp"
+
+namespace duckdb {
+
+PhysicalCreateARTIndex::PhysicalCreateARTIndex(LogicalOperator &op, TableCatalogEntry &table_p,
+                                               const vector<column_t> &column_ids, unique_ptr<CreateIndexInfo> info,
+                                               vector<unique_ptr<Expression>> unbound_expressions,
+                                               idx_t estimated_cardinality, const bool sorted)
+    : PhysicalOperator(PhysicalOperatorType::CREATE_INDEX, op.types, estimated_cardinality),
+      table(table_p.Cast<DuckTableEntry>()), info(std::move(info)), unbound_expressions(std::move(unbound_expressions)),
+      sorted(sorted) {
+	// convert virtual column ids to storage column ids
+	for (auto &column_id : column_ids) {
+		storage_ids.push_back(table.GetColumns().LogicalToPhysical(LogicalIndex(column_id)).index);
+	}
+}
+
+//===--------------------------------------------------------------------===//
+// Sink
+//===--------------------------------------------------------------------===//
+
+class CreateARTIndexGlobalSinkState : public GlobalSinkState {
+public:
+	//! Global index to be added to the table
+	unique_ptr<Index> global_index;
+};
+
+class CreateARTIndexLocalSinkState : public LocalSinkState {
+public:
+	explicit CreateARTIndexLocalSinkState(ClientContext &context) : arena_allocator(Allocator::Get(context)) {};
+
+	unique_ptr<Index> local_index;
+	ArenaAllocator arena_allocator;
+	vector<ARTKey> keys;
+	DataChunk key_chunk;
+	vector<column_t> key_column_ids;
+};
+
+unique_ptr<GlobalSinkState> PhysicalCreateARTIndex::GetGlobalSinkState(ClientContext &context) const {
+	auto state = make_uniq<CreateARTIndexGlobalSinkState>();
+
+	// create the global index
+	auto &storage = table.GetStorage();
+	state->global_index = make_uniq<ART>(storage_ids, TableIOManager::Get(storage), unbound_expressions,
+	                                     info->constraint_type, storage.db);
+
+	return (std::move(state));
+}
+
+unique_ptr<LocalSinkState> PhysicalCreateARTIndex::GetLocalSinkState(ExecutionContext &context) const {
+	auto state = make_uniq<CreateARTIndexLocalSinkState>(context.client);
+
+	// create the local index
+
+	auto &storage = table.GetStorage();
+	state->local_index = make_uniq<ART>(storage_ids, TableIOManager::Get(storage), unbound_expressions,
+	                                    info->constraint_type, storage.db);
+
+	state->keys = vector<ARTKey>(STANDARD_VECTOR_SIZE);
+	state->key_chunk.Initialize(Allocator::Get(context.client), state->local_index->logical_types);
+
+	for (idx_t i = 0; i < state->key_chunk.ColumnCount(); i++) {
+		state->key_column_ids.push_back(i);
+	}
+	return std::move(state);
+}
+
+SinkResultType PhysicalCreateARTIndex::SinkUnsorted(Vector &row_identifiers, OperatorSinkInput &input) const {
+
+	auto &l_state = input.local_state.Cast<CreateARTIndexLocalSinkState>();
+	auto count = l_state.key_chunk.size();
+
+	// get the corresponding row IDs
+	row_identifiers.Flatten(count);
+	auto row_ids = FlatVector::GetData<row_t>(row_identifiers);
+
+	// insert the row IDs
+	auto &art = l_state.local_index->Cast<ART>();
+	for (idx_t i = 0; i < count; i++) {
+		if (!art.Insert(*art.tree, l_state.keys[i], 0, row_ids[i])) {
+			throw ConstraintException("Data contains duplicates on indexed column(s)");
+		}
+	}
+
+	return SinkResultType::NEED_MORE_INPUT;
+}
+
+SinkResultType PhysicalCreateARTIndex::SinkSorted(Vector &row_identifiers, OperatorSinkInput &input) const {
+
+	auto &l_state = input.local_state.Cast<CreateARTIndexLocalSinkState>();
+	auto &storage = table.GetStorage();
+	auto &l_index = l_state.local_index;
+
+	// create an ART from the chunk
+	auto art = make_uniq<ART>(l_index->column_ids, l_index->table_io_manager, l_index->unbound_expressions,
+	                          l_index->constraint_type, storage.db, l_index->Cast<ART>().allocators);
+	if (!art->ConstructFromSorted(l_state.key_chunk.size(), l_state.keys, row_identifiers)) {
+		throw ConstraintException("Data contains duplicates on indexed column(s)");
+	}
+
+	// merge into the local ART
+	if (!l_index->MergeIndexes(*art)) {
+		throw ConstraintException("Data contains duplicates on indexed column(s)");
+	}
+
+	return SinkResultType::NEED_MORE_INPUT;
+}
+
+SinkResultType PhysicalCreateARTIndex::Sink(ExecutionContext &context, DataChunk &chunk,
+                                            OperatorSinkInput &input) const {
+
+	D_ASSERT(chunk.ColumnCount() >= 2);
+
+	// generate the keys for the given input
+	auto &l_state = input.local_state.Cast<CreateARTIndexLocalSinkState>();
+	l_state.key_chunk.ReferenceColumns(chunk, l_state.key_column_ids);
+	l_state.arena_allocator.Reset();
+	ART::GenerateKeys(l_state.arena_allocator, l_state.key_chunk, l_state.keys);
+
+	// insert the keys and their corresponding row IDs
+	auto &row_identifiers = chunk.data[chunk.ColumnCount() - 1];
+	if (sorted) {
+		return SinkSorted(row_identifiers, input);
+	}
+	return SinkUnsorted(row_identifiers, input);
+}
+
+SinkCombineResultType PhysicalCreateARTIndex::Combine(ExecutionContext &context,
+                                                      OperatorSinkCombineInput &input) const {
+
+	auto &gstate = input.global_state.Cast<CreateARTIndexGlobalSinkState>();
+	auto &lstate = input.local_state.Cast<CreateARTIndexLocalSinkState>();
+
+	// merge the local index into the global index
+	if (!gstate.global_index->MergeIndexes(*lstate.local_index)) {
+		throw ConstraintException("Data contains duplicates on indexed column(s)");
+	}
+
+	return SinkCombineResultType::FINISHED;
+}
+
+SinkFinalizeType PhysicalCreateARTIndex::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
+                                                  OperatorSinkFinalizeInput &input) const {
+
+	// here, we set the resulting global index as the newly created index of the table
+	auto &state = input.global_state.Cast<CreateARTIndexGlobalSinkState>();
+
+	// vacuum excess memory and verify
+	state.global_index->Vacuum();
+	D_ASSERT(!state.global_index->VerifyAndToString(true).empty());
+
+	auto &storage = table.GetStorage();
+	if (!storage.IsRoot()) {
+		throw TransactionException("Transaction conflict: cannot add an index to a table that has been altered!");
+	}
+
+	auto &schema = table.schema;
+	auto index_entry = schema.CreateIndex(context, *info, table).get();
+	if (!index_entry) {
+		D_ASSERT(info->on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT);
+		// index already exists, but error ignored because of IF NOT EXISTS
+		return SinkFinalizeType::READY;
+	}
+	auto &index = index_entry->Cast<DuckIndexEntry>();
+
+	index.index = state.global_index.get();
+	index.info = storage.info;
+	for (auto &parsed_expr : info->parsed_expressions) {
+		index.parsed_expressions.push_back(parsed_expr->Copy());
+	}
+
+	// add index to storage
+	storage.info->indexes.AddIndex(std::move(state.global_index));
+	return SinkFinalizeType::READY;
+}
+
+//===--------------------------------------------------------------------===//
+// Source
+//===--------------------------------------------------------------------===//
+
+SourceResultType PhysicalCreateARTIndex::GetData(ExecutionContext &context, DataChunk &chunk,
+                                                 OperatorSourceInput &input) const {
+	return SourceResultType::FINISHED;
+}
+
+} // namespace duckdb