duckdb 0.8.2-dev37.0 → 0.8.2-dev3989.0

package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp

@@ -237,7 +237,7 @@ static void RewriteJoinCondition(Expression &expr, idx_t offset) {
 	ExpressionIterator::EnumerateChildren(expr, [&](Expression &child) { RewriteJoinCondition(child, offset); });
 }
 
-unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparisonJoin &op) {
+unique_ptr<PhysicalOperator> PhysicalPlanGenerator::PlanComparisonJoin(LogicalComparisonJoin &op) {
 	// now visit the children
 	D_ASSERT(op.children.size() == 2);
 	idx_t lhs_cardinality = op.children[0]->EstimateCardinality(context);
@@ -278,8 +278,24 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparison
 		}
 	}
 
+	bool can_merge = has_range > 0;
+	bool can_iejoin = has_range >= 2 && recursive_cte_tables.empty();
+	switch (op.join_type) {
+	case JoinType::SEMI:
+	case JoinType::ANTI:
+	case JoinType::MARK:
+		can_merge = can_merge && op.conditions.size() == 1;
+		can_iejoin = false;
+		break;
+	default:
+		break;
+	}
+
+	//	TODO: Extend PWMJ to handle all comparisons and projection maps
+	const auto prefer_range_joins = (ClientConfig::GetConfig(context).prefer_range_joins && can_iejoin);
+
 	unique_ptr<PhysicalOperator> plan;
-	if (has_equality) {
+	if (has_equality && !prefer_range_joins) {
 		// check if we can use an index join
 		if (PlanIndexJoin(context, op, plan, left, right)) {
 			return plan;
@@ -289,22 +305,10 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparison
 		CheckForPerfectJoinOpt(op, perfect_join_stats);
 		plan = make_uniq<PhysicalHashJoin>(op, std::move(left), std::move(right), std::move(op.conditions),
 		                                   op.join_type, op.left_projection_map, op.right_projection_map,
-		                                   std::move(op.delim_types), op.estimated_cardinality, perfect_join_stats);
+		                                   std::move(op.mark_types), op.estimated_cardinality, perfect_join_stats);
 
 	} else {
 		static constexpr const idx_t NESTED_LOOP_JOIN_THRESHOLD = 5;
-		bool can_merge = has_range > 0;
-		bool can_iejoin = has_range >= 2 && recursive_cte_tables.empty();
-		switch (op.join_type) {
-		case JoinType::SEMI:
-		case JoinType::ANTI:
-		case JoinType::MARK:
-			can_merge = can_merge && op.conditions.size() == 1;
-			can_iejoin = false;
-			break;
-		default:
-			break;
-		}
 		if (left->estimated_cardinality <= NESTED_LOOP_JOIN_THRESHOLD ||
 		    right->estimated_cardinality <= NESTED_LOOP_JOIN_THRESHOLD) {
 			can_iejoin = false;
@@ -334,4 +338,17 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparison
 	return plan;
 }
 
+unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparisonJoin &op) {
+	switch (op.type) {
+	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
+		return PlanAsOfJoin(op);
+	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
+		return PlanComparisonJoin(op);
+	case LogicalOperatorType::LOGICAL_DELIM_JOIN:
+		return PlanDelimJoin(op);
+	default:
+		throw InternalException("Unrecognized operator type for LogicalComparisonJoin");
+	}
+}
+
 } // namespace duckdb

package/src/duckdb/src/execution/physical_plan/plan_create_index.cpp

@@ -2,11 +2,11 @@
 #include "duckdb/execution/operator/projection/physical_projection.hpp"
 #include "duckdb/execution/operator/filter/physical_filter.hpp"
 #include "duckdb/execution/operator/scan/physical_table_scan.hpp"
-#include "duckdb/execution/operator/schema/physical_create_index.hpp"
+#include "duckdb/execution/operator/schema/physical_create_art_index.hpp"
 #include "duckdb/execution/operator/order/physical_order.hpp"
 #include "duckdb/execution/physical_plan_generator.hpp"
-#include "duckdb/function/table/table_scan.hpp"
 #include "duckdb/planner/operator/logical_create_index.hpp"
+#include "duckdb/planner/operator/logical_get.hpp"
 #include "duckdb/planner/expression/bound_operator_expression.hpp"
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/planner/table_filter.hpp"
@@ -14,14 +14,13 @@
 namespace duckdb {
 
 unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCreateIndex &op) {
-
 	// generate a physical plan for the parallel index creation which consists of the following operators
 	// table scan - projection (for expression execution) - filter (NOT NULL) - order - create index
-
-	D_ASSERT(op.children.empty());
+	D_ASSERT(op.children.size() == 1);
+	auto table_scan = CreatePlan(*op.children[0]);
 
 	// validate that all expressions contain valid scalar functions
-	// e.g. get_current_timestamp(), random(), and sequence values are not allowed as ART keys
+	// e.g. get_current_timestamp(), random(), and sequence values are not allowed as index keys
 	// because they make deletions and lookups unfeasible
 	for (idx_t i = 0; i < op.unbound_expressions.size(); i++) {
 		auto &expr = op.unbound_expressions[i];
@@ -31,20 +30,15 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCreateInde
 		}
 	}
 
-	// table scan operator for index key columns and row IDs
-
-	unique_ptr<TableFilterSet> table_filters;
-	op.info->column_ids.emplace_back(COLUMN_IDENTIFIER_ROW_ID);
-
-	auto &bind_data = op.bind_data->Cast<TableScanBindData>();
-	bind_data.is_create_index = true;
-
-	auto table_scan =
-	    make_uniq<PhysicalTableScan>(op.info->scan_types, op.function, std::move(op.bind_data), op.info->column_ids,
-	                                 op.info->names, std::move(table_filters), op.estimated_cardinality);
+	// If we get here without the plan and the index type is not ART, we throw an exception
+	// because we don't support any other index type yet. However an operator extension could have
+	// replaced this part of the plan with a different index creation operator.
+	if (op.info->index_type != IndexType::ART) {
+		throw BinderException("Index type not supported");
+	}
 
+	// table scan operator for index key columns and row IDs
 	dependencies.AddDependency(op.table);
-	op.info->column_ids.pop_back();
 
 	D_ASSERT(op.info->scan_types.size() - 1 <= op.info->names.size());
 	D_ASSERT(op.info->scan_types.size() - 1 <= op.info->column_ids.size());
@@ -82,27 +76,44 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCreateInde
 	null_filter->types.emplace_back(LogicalType::ROW_TYPE);
 	null_filter->children.push_back(std::move(projection));
 
-	// order operator
-
-	vector<BoundOrderByNode> orders;
-	vector<idx_t> projections;
-	for (idx_t i = 0; i < new_column_types.size() - 1; i++) {
-		auto col_expr = make_uniq_base<Expression, BoundReferenceExpression>(new_column_types[i], i);
-		orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_FIRST, std::move(col_expr));
-		projections.emplace_back(i);
+	// determine if we sort the data prior to index creation
+	// we don't sort, if either VARCHAR or compound key
+	auto perform_sorting = true;
+	if (op.unbound_expressions.size() > 1) {
+		perform_sorting = false;
+	} else if (op.unbound_expressions[0]->return_type.InternalType() == PhysicalType::VARCHAR) {
+		perform_sorting = false;
 	}
-	projections.emplace_back(new_column_types.size() - 1);
-
-	auto physical_order =
-	    make_uniq<PhysicalOrder>(new_column_types, std::move(orders), std::move(projections), op.estimated_cardinality);
-	physical_order->children.push_back(std::move(null_filter));
 
 	// actual physical create index operator
 
 	auto physical_create_index =
-	    make_uniq<PhysicalCreateIndex>(op, op.table, op.info->column_ids, std::move(op.info),
-	                                   std::move(op.unbound_expressions), op.estimated_cardinality);
-	physical_create_index->children.push_back(std::move(physical_order));
+	    make_uniq<PhysicalCreateARTIndex>(op, op.table, op.info->column_ids, std::move(op.info),
+	                                      std::move(op.unbound_expressions), op.estimated_cardinality, perform_sorting);
+
+	if (perform_sorting) {
+
+		// optional order operator
+		vector<BoundOrderByNode> orders;
+		vector<idx_t> projections;
+		for (idx_t i = 0; i < new_column_types.size() - 1; i++) {
+			auto col_expr = make_uniq_base<Expression, BoundReferenceExpression>(new_column_types[i], i);
+			orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_FIRST, std::move(col_expr));
+			projections.emplace_back(i);
+		}
+		projections.emplace_back(new_column_types.size() - 1);
+
+		auto physical_order = make_uniq<PhysicalOrder>(new_column_types, std::move(orders), std::move(projections),
+		                                               op.estimated_cardinality);
+		physical_order->children.push_back(std::move(null_filter));
+
+		physical_create_index->children.push_back(std::move(physical_order));
+	} else {
+
+		// no ordering
+		physical_create_index->children.push_back(std::move(null_filter));
+	}
+
 	return std::move(physical_create_index);
 }
 

package/src/duckdb/src/execution/physical_plan/plan_cte.cpp

@@ -0,0 +1,33 @@
+#include "duckdb/common/types/column/column_data_collection.hpp"
+#include "duckdb/execution/operator/scan/physical_column_data_scan.hpp"
+#include "duckdb/execution/operator/set/physical_cte.hpp"
+#include "duckdb/execution/physical_plan_generator.hpp"
+#include "duckdb/planner/expression/bound_reference_expression.hpp"
+#include "duckdb/planner/operator/logical_cteref.hpp"
+#include "duckdb/planner/operator/logical_materialized_cte.hpp"
+
+namespace duckdb {
+
+unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalMaterializedCTE &op) {
+	D_ASSERT(op.children.size() == 2);
+
+	// Create the working_table that the PhysicalCTE will use for evaluation.
+	auto working_table = std::make_shared<ColumnDataCollection>(context, op.children[0]->types);
+
+	// Add the ColumnDataCollection to the context of this PhysicalPlanGenerator
+	recursive_cte_tables[op.table_index] = working_table;
+
+	// Create the plan for the left side. This is the materialization.
+	auto left = CreatePlan(*op.children[0]);
+	// Initialize an empty vector to collect the scan operators.
+	materialized_ctes.insert(op.table_index);
+	auto right = CreatePlan(*op.children[1]);
+
+	auto cte = make_uniq<PhysicalCTE>(op.ctename, op.table_index, op.children[1]->types, std::move(left),
+	                                  std::move(right), op.estimated_cardinality);
+	cte->working_table = working_table;
+
+	return std::move(cte);
+}
+
+} // namespace duckdb

package/src/duckdb/src/execution/physical_plan/plan_delim_join.cpp

@@ -1,10 +1,7 @@
-#include "duckdb/execution/aggregate_hashtable.hpp"
 #include "duckdb/execution/operator/join/physical_delim_join.hpp"
 #include "duckdb/execution/operator/join/physical_hash_join.hpp"
 #include "duckdb/execution/operator/projection/physical_projection.hpp"
 #include "duckdb/execution/physical_plan_generator.hpp"
-#include "duckdb/planner/operator/logical_delim_join.hpp"
-#include "duckdb/planner/expression/bound_aggregate_expression.hpp"
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp"
 
@@ -19,9 +16,9 @@ static void GatherDelimScans(const PhysicalOperator &op, vector<const_reference<
 	}
 }
 
-unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalDelimJoin &op) {
+unique_ptr<PhysicalOperator> PhysicalPlanGenerator::PlanDelimJoin(LogicalComparisonJoin &op) {
 	// first create the underlying join
-	auto plan = CreatePlan(op.Cast<LogicalComparisonJoin>());
+	auto plan = PlanComparisonJoin(op);
 	// this should create a join, not a cross product
 	D_ASSERT(plan && plan->type != PhysicalOperatorType::CROSS_PRODUCT);
 	// duplicate eliminated join

package/src/duckdb/src/execution/physical_plan/plan_get.cpp

@@ -55,7 +55,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalGet &op) {
 		// function does not support projection pushdown
 		auto node = make_uniq<PhysicalTableScan>(op.returned_types, op.function, std::move(op.bind_data),
 		                                         op.returned_types, op.column_ids, vector<column_t>(), op.names,
-		                                         std::move(table_filters), op.estimated_cardinality);
+		                                         std::move(table_filters), op.estimated_cardinality, op.extra_info);
 		// first check if an additional projection is necessary
 		if (op.column_ids.size() == op.returned_types.size()) {
 			bool projection_necessary = false;
@@ -93,7 +93,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalGet &op) {
 	} else {
 		return make_uniq<PhysicalTableScan>(op.types, op.function, std::move(op.bind_data), op.returned_types,
 		                                    op.column_ids, op.projection_ids, op.names, std::move(table_filters),
-		                                    op.estimated_cardinality);
+		                                    op.estimated_cardinality, op.extra_info);
 	}
 }
 

package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp

@@ -20,8 +20,8 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalRecursiveC
 	auto left = CreatePlan(*op.children[0]);
 	auto right = CreatePlan(*op.children[1]);
 
-	auto cte = make_uniq<PhysicalRecursiveCTE>(op.types, op.union_all, std::move(left), std::move(right),
-	                                           op.estimated_cardinality);
+	auto cte = make_uniq<PhysicalRecursiveCTE>(op.ctename, op.table_index, op.types, op.union_all, std::move(left),
+	                                           std::move(right), op.estimated_cardinality);
 	cte->working_table = working_table;
 
 	return std::move(cte);
@@ -30,14 +30,35 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalRecursiveC
 unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCTERef &op) {
 	D_ASSERT(op.children.empty());
 
-	auto chunk_scan =
-	    make_uniq<PhysicalColumnDataScan>(op.types, PhysicalOperatorType::RECURSIVE_CTE_SCAN, op.estimated_cardinality);
+	// Check if this LogicalCTERef is supposed to scan a materialized CTE.
+	if (op.materialized_cte == CTEMaterialize::CTE_MATERIALIZE_ALWAYS) {
+		// Lookup if there is a materialized CTE for the cte_index.
+		auto materialized_cte = materialized_ctes.find(op.cte_index);
+
+		// If this check fails, this is a reference to a materialized recursive CTE.
+		if (materialized_cte != materialized_ctes.end()) {
+			auto chunk_scan = make_uniq<PhysicalColumnDataScan>(op.types, PhysicalOperatorType::CTE_SCAN,
+			                                                    op.estimated_cardinality, op.cte_index);
+
+			auto cte = recursive_cte_tables.find(op.cte_index);
+			if (cte == recursive_cte_tables.end()) {
+				throw InvalidInputException("Referenced materialized CTE does not exist.");
+			}
+			chunk_scan->collection = cte->second.get();
+
+			return std::move(chunk_scan);
+		}
+	}
 
 	// CreatePlan of a LogicalRecursiveCTE must have happened before.
 	auto cte = recursive_cte_tables.find(op.cte_index);
 	if (cte == recursive_cte_tables.end()) {
 		throw InvalidInputException("Referenced recursive CTE does not exist.");
 	}
+
+	auto chunk_scan = make_uniq<PhysicalColumnDataScan>(
+	    cte->second.get()->Types(), PhysicalOperatorType::RECURSIVE_CTE_SCAN, op.estimated_cardinality, op.cte_index);
+
 	chunk_scan->collection = cte->second.get();
 	return std::move(chunk_scan);
 }

package/src/duckdb/src/execution/physical_plan_generator.cpp

@@ -113,12 +113,8 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalOperator &
 	case LogicalOperatorType::LOGICAL_ANY_JOIN:
 		plan = CreatePlan(op.Cast<LogicalAnyJoin>());
 		break;
-	case LogicalOperatorType::LOGICAL_DELIM_JOIN:
-		plan = CreatePlan(op.Cast<LogicalDelimJoin>());
-		break;
 	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
-		plan = CreatePlan(op.Cast<LogicalAsOfJoin>());
-		break;
+	case LogicalOperatorType::LOGICAL_DELIM_JOIN:
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
 		plan = CreatePlan(op.Cast<LogicalComparisonJoin>());
 		break;
@@ -194,6 +190,9 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalOperator &
 	case LogicalOperatorType::LOGICAL_RECURSIVE_CTE:
 		plan = CreatePlan(op.Cast<LogicalRecursiveCTE>());
 		break;
+	case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE:
+		plan = CreatePlan(op.Cast<LogicalMaterializedCTE>());
+		break;
 	case LogicalOperatorType::LOGICAL_CTE_REF:
 		plan = CreatePlan(op.Cast<LogicalCTERef>());
 		break;
@@ -217,6 +216,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalOperator &
 		}
 		break;
 	case LogicalOperatorType::LOGICAL_JOIN:
+	case LogicalOperatorType::LOGICAL_DEPENDENT_JOIN:
 	case LogicalOperatorType::LOGICAL_INVALID: {
 		throw NotImplementedException("Unimplemented logical operator type!");
 	}
@@ -225,12 +225,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalOperator &
 		throw InternalException("Physical plan generator - no plan generated");
 	}
 
-	if (op.estimated_props) {
-		plan->estimated_cardinality = op.estimated_props->GetCardinality<idx_t>();
-		plan->estimated_props = op.estimated_props->Copy();
-	} else {
-		plan->estimated_props = make_uniq<EstimatedProperties>();
-	}
+	plan->estimated_cardinality = op.estimated_cardinality;
 
 	return plan;
 }