duckdb 0.8.2-dev33.0 → 0.8.2-dev3300.0

package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/execution/perfect_aggregate_hashtable.hpp"
-#include "duckdb/execution/expression_executor.hpp"
+
 #include "duckdb/common/row_operations/row_operations.hpp"
+#include "duckdb/execution/expression_executor.hpp"
 
 namespace duckdb {
 
@@ -11,7 +12,8 @@ PerfectAggregateHashTable::PerfectAggregateHashTable(ClientContext &context, All
                                                      vector<Value> group_minima_p, vector<idx_t> required_bits_p)
     : BaseAggregateHashTable(context, allocator, aggregate_objects_p, std::move(payload_types_p)),
       addresses(LogicalType::POINTER), required_bits(std::move(required_bits_p)), total_required_bits(0),
-      group_minima(std::move(group_minima_p)), sel(STANDARD_VECTOR_SIZE), aggregate_allocator(allocator) {
+      group_minima(std::move(group_minima_p)), sel(STANDARD_VECTOR_SIZE),
+      aggregate_allocator(make_uniq<ArenaAllocator>(allocator)) {
 	for (auto &group_bits : required_bits) {
 		total_required_bits += group_bits;
 	}
@@ -93,6 +95,18 @@ static void ComputeGroupLocation(Vector &group, Value &min, uintptr_t *address_d
 	case PhysicalType::INT64:
 		ComputeGroupLocationTemplated<int64_t>(vdata, min, address_data, current_shift, count);
 		break;
+	case PhysicalType::UINT8:
+		ComputeGroupLocationTemplated<uint8_t>(vdata, min, address_data, current_shift, count);
+		break;
+	case PhysicalType::UINT16:
+		ComputeGroupLocationTemplated<uint16_t>(vdata, min, address_data, current_shift, count);
+		break;
+	case PhysicalType::UINT32:
+		ComputeGroupLocationTemplated<uint32_t>(vdata, min, address_data, current_shift, count);
+		break;
+	case PhysicalType::UINT64:
+		ComputeGroupLocationTemplated<uint64_t>(vdata, min, address_data, current_shift, count);
+		break;
 	default:
 		throw InternalException("Unsupported group type for perfect aggregate hash table");
 	}
@@ -123,7 +137,7 @@ void PerfectAggregateHashTable::AddChunk(DataChunk &groups, DataChunk &payload)
 	// after finding the group location we update the aggregates
 	idx_t payload_idx = 0;
 	auto &aggregates = layout.GetAggregates();
-	RowOperationsState row_state(aggregate_allocator.GetAllocator());
+	RowOperationsState row_state(*aggregate_allocator);
 	for (idx_t aggr_idx = 0; aggr_idx < aggregates.size(); aggr_idx++) {
 		auto &aggregate = aggregates[aggr_idx];
 		auto input_count = (idx_t)aggregate.child_count;
@@ -152,7 +166,7 @@ void PerfectAggregateHashTable::Combine(PerfectAggregateHashTable &other) {
 	data_ptr_t source_ptr = other.data;
 	data_ptr_t target_ptr = data;
 	idx_t combine_count = 0;
-	RowOperationsState row_state(aggregate_allocator.GetAllocator());
+	RowOperationsState row_state(*aggregate_allocator);
 	for (idx_t i = 0; i < total_groups; i++) {
 		auto has_entry_source = other.group_is_set[i];
 		// we only have any work to do if the source has an entry for this group
@@ -170,6 +184,11 @@ void PerfectAggregateHashTable::Combine(PerfectAggregateHashTable &other) {
 		target_ptr += tuple_size;
 	}
 	RowOperations::CombineStates(row_state, layout, source_addresses, target_addresses, combine_count);
+
+	// FIXME: after moving the arena allocator, we currently have to ensure that the pointer is not nullptr, because the
+	// FIXME: Destroy()-function of the hash table expects an allocator in some cases (e.g., for sorted aggregates)
+	stored_allocators.push_back(std::move(other.aggregate_allocator));
+	other.aggregate_allocator = make_uniq<ArenaAllocator>(allocator);
 }
 
 template <class T>
@@ -208,6 +227,18 @@ static void ReconstructGroupVector(uint32_t group_values[], Value &min, idx_t re
 	case PhysicalType::INT64:
 		ReconstructGroupVectorTemplated<int64_t>(group_values, min, mask, shift, entry_count, result);
 		break;
+	case PhysicalType::UINT8:
+		ReconstructGroupVectorTemplated<uint8_t>(group_values, min, mask, shift, entry_count, result);
+		break;
+	case PhysicalType::UINT16:
+		ReconstructGroupVectorTemplated<uint16_t>(group_values, min, mask, shift, entry_count, result);
+		break;
+	case PhysicalType::UINT32:
+		ReconstructGroupVectorTemplated<uint32_t>(group_values, min, mask, shift, entry_count, result);
+		break;
+	case PhysicalType::UINT64:
+		ReconstructGroupVectorTemplated<uint64_t>(group_values, min, mask, shift, entry_count, result);
+		break;
 	default:
 		throw InternalException("Invalid type for perfect aggregate HT group");
 	}
@@ -243,7 +274,7 @@ void PerfectAggregateHashTable::Scan(idx_t &scan_position, DataChunk &result) {
 	}
 	// then construct the payloads
 	result.SetCardinality(entry_count);
-	RowOperationsState row_state(aggregate_allocator.GetAllocator());
+	RowOperationsState row_state(*aggregate_allocator);
 	RowOperations::FinalizeStates(row_state, layout, addresses, result, grouping_columns);
 }
 
@@ -264,7 +295,7 @@ void PerfectAggregateHashTable::Destroy() {
 	idx_t count = 0;
 
 	// iterate over all initialised slots of the hash table
-	RowOperationsState row_state(aggregate_allocator.GetAllocator());
+	RowOperationsState row_state(*aggregate_allocator);
 	data_ptr_t payload_ptr = data;
 	for (idx_t i = 0; i < total_groups; i++) {
 		if (group_is_set[i]) {

package/src/duckdb/src/execution/physical_operator.cpp

@@ -97,11 +97,12 @@ SinkResultType PhysicalOperator::Sink(ExecutionContext &context, DataChunk &chun
 
 // LCOV_EXCL_STOP
 
-void PhysicalOperator::Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const {
+SinkCombineResultType PhysicalOperator::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
+	return SinkCombineResultType::FINISHED;
 }
 
 SinkFinalizeType PhysicalOperator::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-                                            GlobalSinkState &gstate) const {
+                                            OperatorSinkFinalizeInput &input) const {
 	return SinkFinalizeType::READY;
 }
 
@@ -124,6 +125,22 @@ idx_t PhysicalOperator::GetMaxThreadMemory(ClientContext &context) {
 	return (max_memory / num_threads) / 4;
 }
 
+bool PhysicalOperator::OperatorCachingAllowed(ExecutionContext &context) {
+	if (!context.client.config.enable_caching_operators) {
+		return false;
+	} else if (!context.pipeline) {
+		return false;
+	} else if (!context.pipeline->GetSink()) {
+		return false;
+	} else if (context.pipeline->GetSink()->RequiresBatchIndex()) {
+		return false;
+	} else if (context.pipeline->IsOrderDependent()) {
+		return false;
+	}
+
+	return true;
+}
+
 //===--------------------------------------------------------------------===//
 // Pipeline Construction
 //===--------------------------------------------------------------------===//
@@ -239,20 +256,7 @@ OperatorResultType CachingPhysicalOperator::Execute(ExecutionContext &context, D
 #if STANDARD_VECTOR_SIZE >= 128
 	if (!state.initialized) {
 		state.initialized = true;
-		state.can_cache_chunk = true;
-
-		if (!context.client.config.enable_caching_operators) {
-			state.can_cache_chunk = false;
-		} else if (!context.pipeline || !caching_supported) {
-			state.can_cache_chunk = false;
-		} else if (!context.pipeline->GetSink()) {
-			// Disabling for pipelines without Sink, i.e. when pulling
-			state.can_cache_chunk = false;
-		} else if (context.pipeline->GetSink()->RequiresBatchIndex()) {
-			state.can_cache_chunk = false;
-		} else if (context.pipeline->IsOrderDependent()) {
-			state.can_cache_chunk = false;
-		}
+		state.can_cache_chunk = caching_supported && PhysicalOperator::OperatorCachingAllowed(context);
 	}
 	if (!state.can_cache_chunk) {
 		return child_result;

package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp

@@ -5,12 +5,12 @@
 #include "duckdb/execution/operator/aggregate/physical_ungrouped_aggregate.hpp"
 #include "duckdb/execution/operator/projection/physical_projection.hpp"
 #include "duckdb/execution/physical_plan_generator.hpp"
+#include "duckdb/function/function_binder.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/parser/expression/comparison_expression.hpp"
 #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
-#include "duckdb/planner/operator/logical_aggregate.hpp"
-#include "duckdb/function/function_binder.hpp"
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
+#include "duckdb/planner/operator/logical_aggregate.hpp"
 
 namespace duckdb {
 
@@ -23,6 +23,11 @@ static uint32_t RequiredBitsForValue(uint32_t n) {
 	return required_bits;
 }
 
+template <class T>
+hugeint_t GetRangeHugeint(const BaseStatistics &nstats) {
+	return Hugeint::Convert(NumericStats::GetMax<T>(nstats)) - Hugeint::Convert(NumericStats::GetMin<T>(nstats));
+}
+
 static bool CanUsePerfectHashAggregate(ClientContext &context, LogicalAggregate &op, vector<idx_t> &bits_per_group) {
 	if (op.grouping_sets.size() > 1 || !op.grouping_functions.empty()) {
 		return false;
@@ -40,6 +45,10 @@ static bool CanUsePerfectHashAggregate(ClientContext &context, LogicalAggregate
 		case PhysicalType::INT16:
 		case PhysicalType::INT32:
 		case PhysicalType::INT64:
+		case PhysicalType::UINT8:
+		case PhysicalType::UINT16:
+		case PhysicalType::UINT32:
+		case PhysicalType::UINT64:
 			break;
 		default:
 			// we only support simple integer types for perfect hashing
@@ -53,6 +62,8 @@ static bool CanUsePerfectHashAggregate(ClientContext &context, LogicalAggregate
 			switch (group_type.InternalType()) {
 			case PhysicalType::INT8:
 			case PhysicalType::INT16:
+			case PhysicalType::UINT8:
+			case PhysicalType::UINT16:
 				break;
 			default:
 				// type is too large and there are no stats: skip perfect hashing
@@ -68,33 +79,55 @@ static bool CanUsePerfectHashAggregate(ClientContext &context, LogicalAggregate
 		if (!NumericStats::HasMinMax(nstats)) {
 			return false;
 		}
+
+		if (NumericStats::Max(*stats) < NumericStats::Min(*stats)) {
+			// May result in underflow
+			return false;
+		}
+
 		// we have a min and a max value for the stats: use that to figure out how many bits we have
 		// we add two here, one for the NULL value, and one to make the computation one-indexed
 		// (e.g. if min and max are the same, we still need one entry in total)
-		int64_t range;
+		hugeint_t range_h;
 		switch (group_type.InternalType()) {
 		case PhysicalType::INT8:
-			range = int64_t(NumericStats::GetMax<int8_t>(nstats)) - int64_t(NumericStats::GetMin<int8_t>(nstats));
+			range_h = GetRangeHugeint<int8_t>(nstats);
 			break;
 		case PhysicalType::INT16:
-			range = int64_t(NumericStats::GetMax<int16_t>(nstats)) - int64_t(NumericStats::GetMin<int16_t>(nstats));
+			range_h = GetRangeHugeint<int16_t>(nstats);
 			break;
 		case PhysicalType::INT32:
-			range = int64_t(NumericStats::GetMax<int32_t>(nstats)) - int64_t(NumericStats::GetMin<int32_t>(nstats));
+			range_h = GetRangeHugeint<int32_t>(nstats);
 			break;
 		case PhysicalType::INT64:
-			if (!TrySubtractOperator::Operation(NumericStats::GetMax<int64_t>(nstats),
-			                                    NumericStats::GetMin<int64_t>(nstats), range)) {
-				return false;
-			}
+			range_h = GetRangeHugeint<int64_t>(nstats);
+			break;
+		case PhysicalType::UINT8:
+			range_h = GetRangeHugeint<uint8_t>(nstats);
+			break;
+		case PhysicalType::UINT16:
+			range_h = GetRangeHugeint<uint16_t>(nstats);
+			break;
+		case PhysicalType::UINT32:
+			range_h = GetRangeHugeint<uint32_t>(nstats);
+			break;
+		case PhysicalType::UINT64:
+			range_h = GetRangeHugeint<uint64_t>(nstats);
 			break;
 		default:
 			throw InternalException("Unsupported type for perfect hash (should be caught before)");
 		}
+
+		uint64_t range;
+		if (!Hugeint::TryCast(range_h, range)) {
+			return false;
+		}
+
 		// bail out on any range bigger than 2^32
 		if (range >= NumericLimits<int32_t>::Maximum()) {
 			return false;
 		}
+
 		range += 2;
 		// figure out how many bits we need
 		idx_t required_bits = RequiredBitsForValue(range);

package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp

@@ -7,11 +7,10 @@
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/planner/expression/bound_window_expression.hpp"
-#include "duckdb/planner/operator/logical_asof_join.hpp"
 
 namespace duckdb {
 
-unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalAsOfJoin &op) {
+unique_ptr<PhysicalOperator> PhysicalPlanGenerator::PlanAsOfJoin(LogicalComparisonJoin &op) {
 	// now visit the children
 	D_ASSERT(op.children.size() == 2);
 	idx_t lhs_cardinality = op.children[0]->EstimateCardinality(context);
@@ -31,6 +30,9 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalAsOfJoin &
 			equi_indexes.emplace_back(c);
 			break;
 		case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
+		case ExpressionType::COMPARE_GREATERTHAN:
+		case ExpressionType::COMPARE_LESSTHANOREQUALTO:
+		case ExpressionType::COMPARE_LESSTHAN:
 			D_ASSERT(asof_idx == op.conditions.size());
 			asof_idx = c;
 			break;
@@ -44,60 +46,80 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalAsOfJoin &
 		return make_uniq<PhysicalAsOfJoin>(op, std::move(left), std::move(right));
 	}
 
+	//	Strip extra column from rhs projections
+	auto &right_projection_map = op.right_projection_map;
+	if (right_projection_map.empty()) {
+		const auto right_count = right->types.size();
+		right_projection_map.reserve(right_count);
+		for (column_t i = 0; i < right_count; ++i) {
+			right_projection_map.emplace_back(i);
+		}
+	}
+
 	//	Debug implementation: IEJoin of Window
-	//	LEAD(asof_column, 1, infinity) OVER (PARTITION BY equi_column... ORDER BY asof_column) AS asof_temp
+	//	LEAD(asof_column, 1, infinity) OVER (PARTITION BY equi_column... ORDER BY asof_column) AS asof_end
 	auto &asof_comp = op.conditions[asof_idx];
 	auto &asof_column = asof_comp.right;
 	auto asof_type = asof_column->return_type;
-	auto asof_temp = make_uniq<BoundWindowExpression>(ExpressionType::WINDOW_LEAD, asof_type, nullptr, nullptr);
-	asof_temp->children.emplace_back(asof_column->Copy());
-	asof_temp->offset_expr = make_uniq<BoundConstantExpression>(Value::BIGINT(1));
+	auto asof_end = make_uniq<BoundWindowExpression>(ExpressionType::WINDOW_LEAD, asof_type, nullptr, nullptr);
+	asof_end->children.emplace_back(asof_column->Copy());
 	// TODO: If infinities are not supported for a type, fake them by looking at LHS statistics?
-	asof_temp->default_expr = make_uniq<BoundConstantExpression>(Value::Infinity(asof_type));
+	asof_end->offset_expr = make_uniq<BoundConstantExpression>(Value::BIGINT(1));
 	for (auto equi_idx : equi_indexes) {
-		asof_temp->partitions.emplace_back(op.conditions[equi_idx].right->Copy());
+		asof_end->partitions.emplace_back(op.conditions[equi_idx].right->Copy());
 	}
-	asof_temp->orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_FIRST, asof_column->Copy());
-	asof_temp->start = WindowBoundary::UNBOUNDED_PRECEDING;
-	asof_temp->end = WindowBoundary::CURRENT_ROW_ROWS;
+	switch (asof_comp.comparison) {
+	case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
+	case ExpressionType::COMPARE_GREATERTHAN:
+		asof_end->orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_FIRST, asof_column->Copy());
+		asof_end->default_expr = make_uniq<BoundConstantExpression>(Value::Infinity(asof_type));
+		break;
+	case ExpressionType::COMPARE_LESSTHANOREQUALTO:
+	case ExpressionType::COMPARE_LESSTHAN:
+		asof_end->orders.emplace_back(OrderType::DESCENDING, OrderByNullType::NULLS_FIRST, asof_column->Copy());
+		asof_end->default_expr = make_uniq<BoundConstantExpression>(Value::NegativeInfinity(asof_type));
+		break;
+	default:
+		throw InternalException("Invalid ASOF JOIN ordering for WINDOW");
+	}
+
+	asof_end->start = WindowBoundary::UNBOUNDED_PRECEDING;
+	asof_end->end = WindowBoundary::CURRENT_ROW_ROWS;
 
 	vector<unique_ptr<Expression>> window_select;
-	window_select.emplace_back(std::move(asof_temp));
+	window_select.emplace_back(std::move(asof_end));
 
-	auto window_types = right->types;
+	auto &window_types = op.children[1]->types;
 	window_types.emplace_back(asof_type);
 
 	auto window = make_uniq<PhysicalWindow>(window_types, std::move(window_select), rhs_cardinality);
 	window->children.emplace_back(std::move(right));
 
-	// IEJoin(left, window, conditions || asof_column < asof_temp)
+	// IEJoin(left, window, conditions || asof_comp ~op asof_end)
 	JoinCondition asof_upper;
 	asof_upper.left = asof_comp.left->Copy();
 	asof_upper.right = make_uniq<BoundReferenceExpression>(asof_type, window_types.size() - 1);
-	asof_upper.comparison = ExpressionType::COMPARE_LESSTHAN;
-
-	// We have an equality condition, so we may have to deal with projection maps.
-	// IEJoin does not (currently) support them, so we have to do it manually
-	auto proj_types = op.types;
-	op.types.clear();
-
-	auto lhs_types = op.children[0]->types;
-	op.types = lhs_types;
-
-	auto rhs_types = op.children[1]->types;
-	op.types.insert(op.types.end(), rhs_types.begin(), rhs_types.end());
+	switch (asof_comp.comparison) {
+	case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
+		asof_upper.comparison = ExpressionType::COMPARE_LESSTHAN;
+		break;
+	case ExpressionType::COMPARE_GREATERTHAN:
+		asof_upper.comparison = ExpressionType::COMPARE_LESSTHANOREQUALTO;
+		break;
+	case ExpressionType::COMPARE_LESSTHANOREQUALTO:
+		asof_upper.comparison = ExpressionType::COMPARE_GREATERTHAN;
+		break;
+	case ExpressionType::COMPARE_LESSTHAN:
+		asof_upper.comparison = ExpressionType::COMPARE_GREATERTHANOREQUALTO;
+		break;
+	default:
+		throw InternalException("Invalid ASOF JOIN comparison for IEJoin");
+	}
 
-	op.types.emplace_back(asof_type);
 	op.conditions.emplace_back(std::move(asof_upper));
-	auto iejoin = make_uniq<PhysicalIEJoin>(op, std::move(left), std::move(window), std::move(op.conditions),
-	                                        op.join_type, op.estimated_cardinality);
-
-	//	Project away asof_temp and anything from the projection maps
-	auto proj = PhysicalProjection::CreateJoinProjection(proj_types, lhs_types, rhs_types, op.left_projection_map,
-	                                                     op.right_projection_map, lhs_cardinality);
-	proj->children.push_back(std::move(iejoin));
 
-	return proj;
+	return make_uniq<PhysicalIEJoin>(op, std::move(left), std::move(window), std::move(op.conditions), op.join_type,
+	                                 lhs_cardinality);
 }
 
 } // namespace duckdb

package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp

@@ -237,7 +237,7 @@ static void RewriteJoinCondition(Expression &expr, idx_t offset) {
 	ExpressionIterator::EnumerateChildren(expr, [&](Expression &child) { RewriteJoinCondition(child, offset); });
 }
 
-unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparisonJoin &op) {
+unique_ptr<PhysicalOperator> PhysicalPlanGenerator::PlanComparisonJoin(LogicalComparisonJoin &op) {
 	// now visit the children
 	D_ASSERT(op.children.size() == 2);
 	idx_t lhs_cardinality = op.children[0]->EstimateCardinality(context);
@@ -278,8 +278,24 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparison
 		}
 	}
 
+	bool can_merge = has_range > 0;
+	bool can_iejoin = has_range >= 2 && recursive_cte_tables.empty();
+	switch (op.join_type) {
+	case JoinType::SEMI:
+	case JoinType::ANTI:
+	case JoinType::MARK:
+		can_merge = can_merge && op.conditions.size() == 1;
+		can_iejoin = false;
+		break;
+	default:
+		break;
+	}
+
+	//	TODO: Extend PWMJ to handle all comparisons and projection maps
+	const auto prefer_range_joins = (ClientConfig::GetConfig(context).prefer_range_joins && can_iejoin);
+
 	unique_ptr<PhysicalOperator> plan;
-	if (has_equality) {
+	if (has_equality && !prefer_range_joins) {
 		// check if we can use an index join
 		if (PlanIndexJoin(context, op, plan, left, right)) {
 			return plan;
@@ -289,22 +305,10 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparison
 		CheckForPerfectJoinOpt(op, perfect_join_stats);
 		plan = make_uniq<PhysicalHashJoin>(op, std::move(left), std::move(right), std::move(op.conditions),
 		                                   op.join_type, op.left_projection_map, op.right_projection_map,
-		                                   std::move(op.delim_types), op.estimated_cardinality, perfect_join_stats);
+		                                   std::move(op.mark_types), op.estimated_cardinality, perfect_join_stats);
 
 	} else {
 		static constexpr const idx_t NESTED_LOOP_JOIN_THRESHOLD = 5;
-		bool can_merge = has_range > 0;
-		bool can_iejoin = has_range >= 2 && recursive_cte_tables.empty();
-		switch (op.join_type) {
-		case JoinType::SEMI:
-		case JoinType::ANTI:
-		case JoinType::MARK:
-			can_merge = can_merge && op.conditions.size() == 1;
-			can_iejoin = false;
-			break;
-		default:
-			break;
-		}
 		if (left->estimated_cardinality <= NESTED_LOOP_JOIN_THRESHOLD ||
 		    right->estimated_cardinality <= NESTED_LOOP_JOIN_THRESHOLD) {
 			can_iejoin = false;
@@ -334,4 +338,17 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparison
 	return plan;
 }
 
+unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparisonJoin &op) {
+	switch (op.type) {
+	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
+		return PlanAsOfJoin(op);
+	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
+		return PlanComparisonJoin(op);
+	case LogicalOperatorType::LOGICAL_DELIM_JOIN:
+		return PlanDelimJoin(op);
+	default:
+		throw InternalException("Unrecognized operator type for LogicalComparisonJoin");
+	}
+}
+
 } // namespace duckdb

package/src/duckdb/src/execution/physical_plan/plan_create_index.cpp

@@ -2,11 +2,11 @@
 #include "duckdb/execution/operator/projection/physical_projection.hpp"
 #include "duckdb/execution/operator/filter/physical_filter.hpp"
 #include "duckdb/execution/operator/scan/physical_table_scan.hpp"
-#include "duckdb/execution/operator/schema/physical_create_index.hpp"
+#include "duckdb/execution/operator/schema/physical_create_art_index.hpp"
 #include "duckdb/execution/operator/order/physical_order.hpp"
 #include "duckdb/execution/physical_plan_generator.hpp"
-#include "duckdb/function/table/table_scan.hpp"
 #include "duckdb/planner/operator/logical_create_index.hpp"
+#include "duckdb/planner/operator/logical_get.hpp"
 #include "duckdb/planner/expression/bound_operator_expression.hpp"
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/planner/table_filter.hpp"
@@ -14,14 +14,13 @@
 namespace duckdb {
 
 unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCreateIndex &op) {
-
 	// generate a physical plan for the parallel index creation which consists of the following operators
 	// table scan - projection (for expression execution) - filter (NOT NULL) - order - create index
-
-	D_ASSERT(op.children.empty());
+	D_ASSERT(op.children.size() == 1);
+	auto table_scan = CreatePlan(*op.children[0]);
 
 	// validate that all expressions contain valid scalar functions
-	// e.g. get_current_timestamp(), random(), and sequence values are not allowed as ART keys
+	// e.g. get_current_timestamp(), random(), and sequence values are not allowed as index keys
 	// because they make deletions and lookups unfeasible
 	for (idx_t i = 0; i < op.unbound_expressions.size(); i++) {
 		auto &expr = op.unbound_expressions[i];
@@ -31,20 +30,15 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCreateInde
 		}
 	}
 
-	// table scan operator for index key columns and row IDs
-
-	unique_ptr<TableFilterSet> table_filters;
-	op.info->column_ids.emplace_back(COLUMN_IDENTIFIER_ROW_ID);
-
-	auto &bind_data = op.bind_data->Cast<TableScanBindData>();
-	bind_data.is_create_index = true;
-
-	auto table_scan =
-	    make_uniq<PhysicalTableScan>(op.info->scan_types, op.function, std::move(op.bind_data), op.info->column_ids,
-	                                 op.info->names, std::move(table_filters), op.estimated_cardinality);
+	// If we get here without the plan and the index type is not ART, we throw an exception
+	// because we don't support any other index type yet. However an operator extension could have
+	// replaced this part of the plan with a different index creation operator.
+	if (op.info->index_type != IndexType::ART) {
+		throw BinderException("Index type not supported");
+	}
 
+	// table scan operator for index key columns and row IDs
 	dependencies.AddDependency(op.table);
-	op.info->column_ids.pop_back();
 
 	D_ASSERT(op.info->scan_types.size() - 1 <= op.info->names.size());
 	D_ASSERT(op.info->scan_types.size() - 1 <= op.info->column_ids.size());
@@ -82,27 +76,44 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCreateInde
 	null_filter->types.emplace_back(LogicalType::ROW_TYPE);
 	null_filter->children.push_back(std::move(projection));
 
-	// order operator
-
-	vector<BoundOrderByNode> orders;
-	vector<idx_t> projections;
-	for (idx_t i = 0; i < new_column_types.size() - 1; i++) {
-		auto col_expr = make_uniq_base<Expression, BoundReferenceExpression>(new_column_types[i], i);
-		orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_FIRST, std::move(col_expr));
-		projections.emplace_back(i);
+	// determine if we sort the data prior to index creation
+	// we don't sort, if either VARCHAR or compound key
+	auto perform_sorting = true;
+	if (op.unbound_expressions.size() > 1) {
+		perform_sorting = false;
+	} else if (op.unbound_expressions[0]->return_type.InternalType() == PhysicalType::VARCHAR) {
+		perform_sorting = false;
 	}
-	projections.emplace_back(new_column_types.size() - 1);
-
-	auto physical_order =
-	    make_uniq<PhysicalOrder>(new_column_types, std::move(orders), std::move(projections), op.estimated_cardinality);
-	physical_order->children.push_back(std::move(null_filter));
 
 	// actual physical create index operator
 
 	auto physical_create_index =
-	    make_uniq<PhysicalCreateIndex>(op, op.table, op.info->column_ids, std::move(op.info),
-	                                   std::move(op.unbound_expressions), op.estimated_cardinality);
-	physical_create_index->children.push_back(std::move(physical_order));
+	    make_uniq<PhysicalCreateARTIndex>(op, op.table, op.info->column_ids, std::move(op.info),
+	                                      std::move(op.unbound_expressions), op.estimated_cardinality, perform_sorting);
+
+	if (perform_sorting) {
+
+		// optional order operator
+		vector<BoundOrderByNode> orders;
+		vector<idx_t> projections;
+		for (idx_t i = 0; i < new_column_types.size() - 1; i++) {
+			auto col_expr = make_uniq_base<Expression, BoundReferenceExpression>(new_column_types[i], i);
+			orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_FIRST, std::move(col_expr));
+			projections.emplace_back(i);
+		}
+		projections.emplace_back(new_column_types.size() - 1);
+
+		auto physical_order = make_uniq<PhysicalOrder>(new_column_types, std::move(orders), std::move(projections),
+		                                               op.estimated_cardinality);
+		physical_order->children.push_back(std::move(null_filter));
+
+		physical_create_index->children.push_back(std::move(physical_order));
+	} else {
+
+		// no ordering
+		physical_create_index->children.push_back(std::move(null_filter));
+	}
+
 	return std::move(physical_create_index);
 }
 

package/src/duckdb/src/execution/physical_plan/plan_cte.cpp

@@ -0,0 +1,33 @@
+#include "duckdb/common/types/column/column_data_collection.hpp"
+#include "duckdb/execution/operator/scan/physical_column_data_scan.hpp"
+#include "duckdb/execution/operator/set/physical_cte.hpp"
+#include "duckdb/execution/physical_plan_generator.hpp"
+#include "duckdb/planner/expression/bound_reference_expression.hpp"
+#include "duckdb/planner/operator/logical_cteref.hpp"
+#include "duckdb/planner/operator/logical_materialized_cte.hpp"
+
+namespace duckdb {
+
+unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalMaterializedCTE &op) {
+	D_ASSERT(op.children.size() == 2);
+
+	// Create the working_table that the PhysicalCTE will use for evaluation.
+	auto working_table = std::make_shared<ColumnDataCollection>(context, op.children[0]->types);
+
+	// Add the ColumnDataCollection to the context of this PhysicalPlanGenerator
+	recursive_cte_tables[op.table_index] = working_table;
+
+	// Create the plan for the left side. This is the materialization.
+	auto left = CreatePlan(*op.children[0]);
+	// Initialize an empty vector to collect the scan operators.
+	materialized_ctes.insert(op.table_index);
+	auto right = CreatePlan(*op.children[1]);
+
+	auto cte = make_uniq<PhysicalCTE>(op.ctename, op.table_index, op.children[1]->types, std::move(left),
+	                                  std::move(right), op.estimated_cardinality);
+	cte->working_table = working_table;
+
+	return std::move(cte);
+}
+
+} // namespace duckdb

package/src/duckdb/src/execution/physical_plan/plan_delim_join.cpp

@@ -1,10 +1,7 @@
-#include "duckdb/execution/aggregate_hashtable.hpp"
 #include "duckdb/execution/operator/join/physical_delim_join.hpp"
 #include "duckdb/execution/operator/join/physical_hash_join.hpp"
 #include "duckdb/execution/operator/projection/physical_projection.hpp"
 #include "duckdb/execution/physical_plan_generator.hpp"
-#include "duckdb/planner/operator/logical_delim_join.hpp"
-#include "duckdb/planner/expression/bound_aggregate_expression.hpp"
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp"
 
@@ -19,9 +16,9 @@ static void GatherDelimScans(const PhysicalOperator &op, vector<const_reference<
 	}
 }
 
-unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalDelimJoin &op) {
+unique_ptr<PhysicalOperator> PhysicalPlanGenerator::PlanDelimJoin(LogicalComparisonJoin &op) {
 	// first create the underlying join
-	auto plan = CreatePlan(op.Cast<LogicalComparisonJoin>());
+	auto plan = PlanComparisonJoin(op);
 	// this should create a join, not a cross product
 	D_ASSERT(plan && plan->type != PhysicalOperatorType::CROSS_PRODUCT);
 	// duplicate eliminated join