duckdb 0.8.2-dev150.0 → 0.8.2-dev1559.0

package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp

@@ -44,6 +44,16 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalAsOfJoin &
 		return make_uniq<PhysicalAsOfJoin>(op, std::move(left), std::move(right));
 	}
 
+	//	Strip extra column from rhs projections
+	auto &right_projection_map = op.right_projection_map;
+	if (right_projection_map.empty()) {
+		const auto right_count = right->types.size();
+		right_projection_map.reserve(right_count);
+		for (column_t i = 0; i < right_count; ++i) {
+			right_projection_map.emplace_back(i);
+		}
+	}
+
 	//	Debug implementation: IEJoin of Window
 	//	LEAD(asof_column, 1, infinity) OVER (PARTITION BY equi_column... ORDER BY asof_column) AS asof_temp
 	auto &asof_comp = op.conditions[asof_idx];
@@ -64,7 +74,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalAsOfJoin &
 	vector<unique_ptr<Expression>> window_select;
 	window_select.emplace_back(std::move(asof_temp));
 
-	auto window_types = right->types;
+	auto &window_types = op.children[1]->types;
 	window_types.emplace_back(asof_type);
 
 	auto window = make_uniq<PhysicalWindow>(window_types, std::move(window_select), rhs_cardinality);
@@ -75,29 +85,10 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalAsOfJoin &
 	asof_upper.left = asof_comp.left->Copy();
 	asof_upper.right = make_uniq<BoundReferenceExpression>(asof_type, window_types.size() - 1);
 	asof_upper.comparison = ExpressionType::COMPARE_LESSTHAN;
-
-	// We have an equality condition, so we may have to deal with projection maps.
-	// IEJoin does not (currently) support them, so we have to do it manually
-	auto proj_types = op.types;
-	op.types.clear();
-
-	auto lhs_types = op.children[0]->types;
-	op.types = lhs_types;
-
-	auto rhs_types = op.children[1]->types;
-	op.types.insert(op.types.end(), rhs_types.begin(), rhs_types.end());
-
-	op.types.emplace_back(asof_type);
 	op.conditions.emplace_back(std::move(asof_upper));
-	auto iejoin = make_uniq<PhysicalIEJoin>(op, std::move(left), std::move(window), std::move(op.conditions),
-	                                        op.join_type, op.estimated_cardinality);
-
-	//	Project away asof_temp and anything from the projection maps
-	auto proj = PhysicalProjection::CreateJoinProjection(proj_types, lhs_types, rhs_types, op.left_projection_map,
-	                                                     op.right_projection_map, lhs_cardinality);
-	proj->children.push_back(std::move(iejoin));
 
-	return proj;
+	return make_uniq<PhysicalIEJoin>(op, std::move(left), std::move(window), std::move(op.conditions), op.join_type,
+	                                 lhs_cardinality);
 }
 
 } // namespace duckdb

package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp

@@ -278,8 +278,24 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparison
 		}
 	}
 
+	bool can_merge = has_range > 0;
+	bool can_iejoin = has_range >= 2 && recursive_cte_tables.empty();
+	switch (op.join_type) {
+	case JoinType::SEMI:
+	case JoinType::ANTI:
+	case JoinType::MARK:
+		can_merge = can_merge && op.conditions.size() == 1;
+		can_iejoin = false;
+		break;
+	default:
+		break;
+	}
+
+	//	TODO: Extend PWMJ to handle all comparisons and projection maps
+	const auto prefer_range_joins = (ClientConfig::GetConfig(context).prefer_range_joins && can_iejoin);
+
 	unique_ptr<PhysicalOperator> plan;
-	if (has_equality) {
+	if (has_equality && !prefer_range_joins) {
 		// check if we can use an index join
 		if (PlanIndexJoin(context, op, plan, left, right)) {
 			return plan;
@@ -293,18 +309,6 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparison
 
 	} else {
 		static constexpr const idx_t NESTED_LOOP_JOIN_THRESHOLD = 5;
-		bool can_merge = has_range > 0;
-		bool can_iejoin = has_range >= 2 && recursive_cte_tables.empty();
-		switch (op.join_type) {
-		case JoinType::SEMI:
-		case JoinType::ANTI:
-		case JoinType::MARK:
-			can_merge = can_merge && op.conditions.size() == 1;
-			can_iejoin = false;
-			break;
-		default:
-			break;
-		}
 		if (left->estimated_cardinality <= NESTED_LOOP_JOIN_THRESHOLD ||
 		    right->estimated_cardinality <= NESTED_LOOP_JOIN_THRESHOLD) {
 			can_iejoin = false;

package/src/duckdb/src/execution/physical_plan/plan_cte.cpp

@@ -0,0 +1,33 @@
+#include "duckdb/common/types/column/column_data_collection.hpp"
+#include "duckdb/execution/operator/scan/physical_column_data_scan.hpp"
+#include "duckdb/execution/operator/set/physical_cte.hpp"
+#include "duckdb/execution/physical_plan_generator.hpp"
+#include "duckdb/planner/expression/bound_reference_expression.hpp"
+#include "duckdb/planner/operator/logical_cteref.hpp"
+#include "duckdb/planner/operator/logical_materialized_cte.hpp"
+
+namespace duckdb {
+
+unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalMaterializedCTE &op) {
+	D_ASSERT(op.children.size() == 2);
+
+	// Create the working_table that the PhysicalCTE will use for evaluation.
+	auto working_table = std::make_shared<ColumnDataCollection>(context, op.children[0]->types);
+
+	// Add the ColumnDataCollection to the context of this PhysicalPlanGenerator
+	recursive_cte_tables[op.table_index] = working_table;
+
+	// Create the plan for the left side. This is the materialization.
+	auto left = CreatePlan(*op.children[0]);
+	// Initialize an empty vector to collect the scan operators.
+	materialized_ctes.insert(op.table_index);
+	auto right = CreatePlan(*op.children[1]);
+
+	auto cte = make_uniq<PhysicalCTE>(op.ctename, op.table_index, op.children[1]->types, std::move(left),
+	                                  std::move(right), op.estimated_cardinality);
+	cte->working_table = working_table;
+
+	return std::move(cte);
+}
+
+} // namespace duckdb

package/src/duckdb/src/execution/physical_plan/plan_get.cpp

@@ -55,7 +55,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalGet &op) {
 		// function does not support projection pushdown
 		auto node = make_uniq<PhysicalTableScan>(op.returned_types, op.function, std::move(op.bind_data),
 		                                         op.returned_types, op.column_ids, vector<column_t>(), op.names,
-		                                         std::move(table_filters), op.estimated_cardinality);
+		                                         std::move(table_filters), op.estimated_cardinality, op.extra_info);
 		// first check if an additional projection is necessary
 		if (op.column_ids.size() == op.returned_types.size()) {
 			bool projection_necessary = false;
@@ -93,7 +93,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalGet &op) {
 	} else {
 		return make_uniq<PhysicalTableScan>(op.types, op.function, std::move(op.bind_data), op.returned_types,
 		                                    op.column_ids, op.projection_ids, op.names, std::move(table_filters),
-		                                    op.estimated_cardinality);
+		                                    op.estimated_cardinality, op.extra_info);
 	}
 }
 

package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp

@@ -20,8 +20,8 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalRecursiveC
 	auto left = CreatePlan(*op.children[0]);
 	auto right = CreatePlan(*op.children[1]);
 
-	auto cte = make_uniq<PhysicalRecursiveCTE>(op.types, op.union_all, std::move(left), std::move(right),
-	                                           op.estimated_cardinality);
+	auto cte = make_uniq<PhysicalRecursiveCTE>(op.ctename, op.table_index, op.types, op.union_all, std::move(left),
+	                                           std::move(right), op.estimated_cardinality);
 	cte->working_table = working_table;
 
 	return std::move(cte);
@@ -30,14 +30,35 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalRecursiveC
 unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCTERef &op) {
 	D_ASSERT(op.children.empty());
 
-	auto chunk_scan =
-	    make_uniq<PhysicalColumnDataScan>(op.types, PhysicalOperatorType::RECURSIVE_CTE_SCAN, op.estimated_cardinality);
+	// Check if this LogicalCTERef is supposed to scan a materialized CTE.
+	if (op.materialized_cte == CTEMaterialize::CTE_MATERIALIZE_ALWAYS) {
+		// Lookup if there is a materialized CTE for the cte_index.
+		auto materialized_cte = materialized_ctes.find(op.cte_index);
+
+		// If this check fails, this is a reference to a materialized recursive CTE.
+		if (materialized_cte != materialized_ctes.end()) {
+			auto chunk_scan = make_uniq<PhysicalColumnDataScan>(op.types, PhysicalOperatorType::CTE_SCAN,
+			                                                    op.estimated_cardinality, op.cte_index);
+
+			auto cte = recursive_cte_tables.find(op.cte_index);
+			if (cte == recursive_cte_tables.end()) {
+				throw InvalidInputException("Referenced materialized CTE does not exist.");
+			}
+			chunk_scan->collection = cte->second.get();
+
+			return std::move(chunk_scan);
+		}
+	}
 
 	// CreatePlan of a LogicalRecursiveCTE must have happened before.
 	auto cte = recursive_cte_tables.find(op.cte_index);
 	if (cte == recursive_cte_tables.end()) {
 		throw InvalidInputException("Referenced recursive CTE does not exist.");
 	}
+
+	auto chunk_scan = make_uniq<PhysicalColumnDataScan>(
+	    cte->second.get()->Types(), PhysicalOperatorType::RECURSIVE_CTE_SCAN, op.estimated_cardinality, op.cte_index);
+
 	chunk_scan->collection = cte->second.get();
 	return std::move(chunk_scan);
 }

package/src/duckdb/src/execution/physical_plan_generator.cpp

@@ -194,6 +194,9 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalOperator &
 	case LogicalOperatorType::LOGICAL_RECURSIVE_CTE:
 		plan = CreatePlan(op.Cast<LogicalRecursiveCTE>());
 		break;
+	case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE:
+		plan = CreatePlan(op.Cast<LogicalMaterializedCTE>());
+		break;
 	case LogicalOperatorType::LOGICAL_CTE_REF:
 		plan = CreatePlan(op.Cast<LogicalCTERef>());
 		break;
@@ -217,6 +220,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalOperator &
 		}
 		break;
 	case LogicalOperatorType::LOGICAL_JOIN:
+	case LogicalOperatorType::LOGICAL_DEPENDENT_JOIN:
 	case LogicalOperatorType::LOGICAL_INVALID: {
 		throw NotImplementedException("Unimplemented logical operator type!");
 	}