duckdb 0.7.2-dev12.0 → 0.7.2-dev1238.0

package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp

@@ -9,7 +9,9 @@
 #include "duckdb/parser/expression/comparison_expression.hpp"
 #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
 #include "duckdb/planner/operator/logical_aggregate.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+#include "duckdb/function/function_binder.hpp"
+#include "duckdb/planner/expression/bound_reference_expression.hpp"
+
 namespace duckdb {
 
 static uint32_t RequiredBitsForValue(uint32_t n) {
@@ -50,23 +52,20 @@ static bool CanUsePerfectHashAggregate(ClientContext &context, LogicalAggregate
 			// for small types we can just set the stats to [type_min, type_max]
 			switch (group_type.InternalType()) {
 			case PhysicalType::INT8:
-				stats = make_unique<NumericStatistics>(group_type, Value::MinimumValue(group_type),
-				                                       Value::MaximumValue(group_type), StatisticsType::LOCAL_STATS);
-				break;
 			case PhysicalType::INT16:
-				stats = make_unique<NumericStatistics>(group_type, Value::MinimumValue(group_type),
-				                                       Value::MaximumValue(group_type), StatisticsType::LOCAL_STATS);
 				break;
 			default:
 				// type is too large and there are no stats: skip perfect hashing
 				return false;
 			}
-			// we had no stats before, so we have no clue if there are null values or not
-			stats->validity_stats = make_unique<ValidityStatistics>(true);
+			// construct stats with the min and max value of the type
+			stats = NumericStats::CreateUnknown(group_type).ToUnique();
+			NumericStats::SetMin(*stats, Value::MinimumValue(group_type));
+			NumericStats::SetMax(*stats, Value::MaximumValue(group_type));
 		}
-		auto &nstats = (NumericStatistics &)*stats;
+		auto &nstats = *stats;
 
-		if (nstats.min.IsNull() || nstats.max.IsNull()) {
+		if (!NumericStats::HasMinMax(nstats)) {
 			return false;
 		}
 		// we have a min and a max value for the stats: use that to figure out how many bits we have
@@ -75,17 +74,17 @@ static bool CanUsePerfectHashAggregate(ClientContext &context, LogicalAggregate
 		int64_t range;
 		switch (group_type.InternalType()) {
 		case PhysicalType::INT8:
-			range = int64_t(nstats.max.GetValueUnsafe<int8_t>()) - int64_t(nstats.min.GetValueUnsafe<int8_t>());
+			range = int64_t(NumericStats::GetMax<int8_t>(nstats)) - int64_t(NumericStats::GetMin<int8_t>(nstats));
 			break;
 		case PhysicalType::INT16:
-			range = int64_t(nstats.max.GetValueUnsafe<int16_t>()) - int64_t(nstats.min.GetValueUnsafe<int16_t>());
+			range = int64_t(NumericStats::GetMax<int16_t>(nstats)) - int64_t(NumericStats::GetMin<int16_t>(nstats));
 			break;
 		case PhysicalType::INT32:
-			range = int64_t(nstats.max.GetValueUnsafe<int32_t>()) - int64_t(nstats.min.GetValueUnsafe<int32_t>());
+			range = int64_t(NumericStats::GetMax<int32_t>(nstats)) - int64_t(NumericStats::GetMin<int32_t>(nstats));
 			break;
 		case PhysicalType::INT64:
-			if (!TrySubtractOperator::Operation(nstats.max.GetValueUnsafe<int64_t>(),
-			                                    nstats.min.GetValueUnsafe<int64_t>(), range)) {
+			if (!TrySubtractOperator::Operation(NumericStats::GetMax<int64_t>(nstats),
+			                                    NumericStats::GetMin<int64_t>(nstats), range)) {
 				return false;
 			}
 			break;
@@ -169,13 +168,20 @@ PhysicalPlanGenerator::ExtractAggregateExpressions(unique_ptr<PhysicalOperator>
 	vector<unique_ptr<Expression>> expressions;
 	vector<LogicalType> types;
 
+	// bind sorted aggregates
+	for (auto &aggr : aggregates) {
+		auto &bound_aggr = (BoundAggregateExpression &)*aggr;
+		if (bound_aggr.order_bys) {
+			// sorted aggregate!
+			FunctionBinder::BindSortedAggregate(context, bound_aggr, groups);
+		}
+	}
 	for (auto &group : groups) {
 		auto ref = make_unique<BoundReferenceExpression>(group->return_type, expressions.size());
 		types.push_back(group->return_type);
 		expressions.push_back(std::move(group));
 		group = std::move(ref);
 	}
-
 	for (auto &aggr : aggregates) {
 		auto &bound_aggr = (BoundAggregateExpression &)*aggr;
 		for (auto &child : bound_aggr.children) {

package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp

@@ -0,0 +1,97 @@
+#include "duckdb/execution/operator/aggregate/physical_window.hpp"
+#include "duckdb/execution/operator/join/physical_iejoin.hpp"
+#include "duckdb/execution/operator/projection/physical_projection.hpp"
+#include "duckdb/execution/physical_plan_generator.hpp"
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/planner/expression/bound_constant_expression.hpp"
+#include "duckdb/planner/expression/bound_reference_expression.hpp"
+#include "duckdb/planner/expression/bound_window_expression.hpp"
+#include "duckdb/planner/operator/logical_asof_join.hpp"
+
+namespace duckdb {
+
+unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalAsOfJoin &op) {
+	// now visit the children
+	D_ASSERT(op.children.size() == 2);
+	idx_t lhs_cardinality = op.children[0]->EstimateCardinality(context);
+	idx_t rhs_cardinality = op.children[1]->EstimateCardinality(context);
+	auto left = CreatePlan(*op.children[0]);
+	auto right = CreatePlan(*op.children[1]);
+	D_ASSERT(left && right);
+
+	//	Validate
+	vector<idx_t> equi_indexes;
+	auto asof_idx = op.conditions.size();
+	for (size_t c = 0; c < op.conditions.size(); ++c) {
+		auto &cond = op.conditions[c];
+		switch (cond.comparison) {
+		case ExpressionType::COMPARE_EQUAL:
+		case ExpressionType::COMPARE_NOT_DISTINCT_FROM:
+			equi_indexes.emplace_back(c);
+			break;
+		case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
+			D_ASSERT(asof_idx == op.conditions.size());
+			asof_idx = c;
+			break;
+		default:
+			throw InternalException("Invalid ASOF JOIN comparison");
+		}
+	}
+	D_ASSERT(asof_idx < op.conditions.size());
+
+	//	Temporary implementation: IEJoin of Window
+	//	LEAD(asof_column, 1, infinity) OVER (PARTITION BY equi_column... ORDER BY asof_column) AS asof_temp
+	auto &asof_comp = op.conditions[asof_idx];
+	auto &asof_column = asof_comp.right;
+	auto asof_type = asof_column->return_type;
+	auto asof_temp = make_unique<BoundWindowExpression>(ExpressionType::WINDOW_LEAD, asof_type, nullptr, nullptr);
+	asof_temp->children.emplace_back(asof_column->Copy());
+	asof_temp->offset_expr = make_unique<BoundConstantExpression>(Value::BIGINT(1));
+	asof_temp->default_expr = make_unique<BoundConstantExpression>(Value::Infinity(asof_type));
+	for (auto equi_idx : equi_indexes) {
+		asof_temp->partitions.emplace_back(op.conditions[equi_idx].right->Copy());
+	}
+	asof_temp->orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_FIRST, asof_column->Copy());
+	asof_temp->start = WindowBoundary::UNBOUNDED_PRECEDING;
+	asof_temp->end = WindowBoundary::CURRENT_ROW_ROWS;
+
+	vector<unique_ptr<Expression>> window_select;
+	window_select.emplace_back(std::move(asof_temp));
+
+	auto window_types = right->types;
+	window_types.emplace_back(asof_type);
+
+	auto window = make_unique<PhysicalWindow>(window_types, std::move(window_select), rhs_cardinality);
+	window->children.emplace_back(std::move(right));
+
+	// IEJoin(left, window, conditions || asof_column < asof_temp)
+	JoinCondition asof_upper;
+	asof_upper.left = asof_comp.left->Copy();
+	asof_upper.right = make_unique<BoundReferenceExpression>(asof_type, window_types.size() - 1);
+	asof_upper.comparison = ExpressionType::COMPARE_LESSTHAN;
+
+	// We have an equality condition, so we may have to deal with projection maps.
+	// IEJoin does not (currently) support them, so we have to do it manually
+	auto proj_types = op.types;
+	op.types.clear();
+
+	auto lhs_types = op.children[0]->types;
+	op.types = lhs_types;
+
+	auto rhs_types = op.children[1]->types;
+	op.types.insert(op.types.end(), rhs_types.begin(), rhs_types.end());
+
+	op.types.emplace_back(asof_type);
+	op.conditions.emplace_back(std::move(asof_upper));
+	auto iejoin = make_unique<PhysicalIEJoin>(op, std::move(left), std::move(window), std::move(op.conditions),
+	                                          op.join_type, op.estimated_cardinality);
+
+	//	Project away asof_temp and anything from the projection maps
+	auto proj = PhysicalProjection::CreateJoinProjection(proj_types, lhs_types, rhs_types, op.left_projection_map,
+	                                                     op.right_projection_map, lhs_cardinality);
+	proj->children.push_back(std::move(iejoin));
+
+	return proj;
+}
+
+} // namespace duckdb

package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp

@@ -10,10 +10,10 @@
 #include "duckdb/function/table/table_scan.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/planner/operator/logical_comparison_join.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
 #include "duckdb/transaction/duck_transaction.hpp"
 #include "duckdb/common/operator/subtract.hpp"
 #include "duckdb/execution/operator/join/physical_blockwise_nl_join.hpp"
+#include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
 #include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
 
@@ -92,19 +92,21 @@ void CheckForPerfectJoinOpt(LogicalComparisonJoin &op, PerfectHashJoinStats &joi
 	}
 	// with integral internal types
 	for (auto &&join_stat : op.join_stats) {
-		if (!TypeIsInteger(join_stat->type.InternalType()) || join_stat->type.InternalType() == PhysicalType::INT128) {
+		if (!TypeIsInteger(join_stat->GetType().InternalType()) ||
+		    join_stat->GetType().InternalType() == PhysicalType::INT128) {
 			// perfect join not possible for non-integral types or hugeint
 			return;
 		}
 	}
 
 	// and when the build range is smaller than the threshold
-	auto stats_build = reinterpret_cast<NumericStatistics *>(op.join_stats[0].get()); // lhs stats
-	if (stats_build->min.IsNull() || stats_build->max.IsNull()) {
+	auto &stats_build = *op.join_stats[0].get(); // lhs stats
+	if (!NumericStats::HasMinMax(stats_build)) {
 		return;
 	}
 	int64_t min_value, max_value;
-	if (!ExtractNumericValue(stats_build->min, min_value) || !ExtractNumericValue(stats_build->max, max_value)) {
+	if (!ExtractNumericValue(NumericStats::Min(stats_build), min_value) ||
+	    !ExtractNumericValue(NumericStats::Max(stats_build), max_value)) {
 		return;
 	}
 	int64_t build_range;
@@ -113,20 +115,24 @@ void CheckForPerfectJoinOpt(LogicalComparisonJoin &op, PerfectHashJoinStats &joi
 	}
 
 	// Fill join_stats for invisible join
-	auto stats_probe = reinterpret_cast<NumericStatistics *>(op.join_stats[1].get()); // rhs stats
+	auto &stats_probe = *op.join_stats[1].get(); // rhs stats
+	if (!NumericStats::HasMinMax(stats_probe)) {
+		return;
+	}
 
 	// The max size our build must have to run the perfect HJ
 	const idx_t MAX_BUILD_SIZE = 1000000;
-	join_state.probe_min = stats_probe->min;
-	join_state.probe_max = stats_probe->max;
-	join_state.build_min = stats_build->min;
-	join_state.build_max = stats_build->max;
+	join_state.probe_min = NumericStats::Min(stats_probe);
+	join_state.probe_max = NumericStats::Max(stats_probe);
+	join_state.build_min = NumericStats::Min(stats_build);
+	join_state.build_max = NumericStats::Max(stats_build);
 	join_state.estimated_cardinality = op.estimated_cardinality;
 	join_state.build_range = build_range;
-	if (join_state.build_range > MAX_BUILD_SIZE || stats_probe->max.IsNull() || stats_probe->min.IsNull()) {
+	if (join_state.build_range > MAX_BUILD_SIZE) {
 		return;
 	}
-	if (stats_build->min <= stats_probe->min && stats_probe->max <= stats_build->max) {
+	if (NumericStats::Min(stats_build) <= NumericStats::Min(stats_probe) &&
+	    NumericStats::Max(stats_probe) <= NumericStats::Max(stats_build)) {
 		join_state.is_probe_in_domain = true;
 	}
 	join_state.is_build_small = true;
@@ -146,27 +152,80 @@ static void CanUseIndexJoin(TableScanBindData *tbl, Expression &expr, Index **re
 	});
 }
 
-void TransformIndexJoin(ClientContext &context, LogicalComparisonJoin &op, Index **left_index, Index **right_index,
-                        PhysicalOperator *left, PhysicalOperator *right) {
+Index *CheckIndexJoin(ClientContext &context, LogicalComparisonJoin &op, PhysicalOperator &plan,
+                      Expression &condition) {
+	if (op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN) {
+		return nullptr;
+	}
 	// check if one of the tables has an index on column
-	if (op.join_type == JoinType::INNER && op.conditions.size() == 1) {
-		// check if one of the children are table scans and if they have an index in the join attribute
-		// (op.condition)
-		if (left->type == PhysicalOperatorType::TABLE_SCAN) {
-			auto &tbl_scan = (PhysicalTableScan &)*left;
-			auto tbl = dynamic_cast<TableScanBindData *>(tbl_scan.bind_data.get());
-			if (CanPlanIndexJoin(context, tbl, tbl_scan)) {
-				CanUseIndexJoin(tbl, *op.conditions[0].left, left_index);
-			}
-		}
-		if (right->type == PhysicalOperatorType::TABLE_SCAN) {
-			auto &tbl_scan = (PhysicalTableScan &)*right;
-			auto tbl = dynamic_cast<TableScanBindData *>(tbl_scan.bind_data.get());
-			if (CanPlanIndexJoin(context, tbl, tbl_scan)) {
-				CanUseIndexJoin(tbl, *op.conditions[0].right, right_index);
-			}
-		}
+	if (op.join_type != JoinType::INNER) {
+		return nullptr;
 	}
+	if (op.conditions.size() != 1) {
+		return nullptr;
+	}
+	// check if the child is (1) a table scan, and (2) has an index on the join condition
+	if (plan.type != PhysicalOperatorType::TABLE_SCAN) {
+		return nullptr;
+	}
+	auto &tbl_scan = (PhysicalTableScan &)plan;
+	auto tbl = dynamic_cast<TableScanBindData *>(tbl_scan.bind_data.get());
+	Index *result = nullptr;
+	if (CanPlanIndexJoin(context, tbl, tbl_scan)) {
+		CanUseIndexJoin(tbl, condition, &result);
+	}
+	return result;
+}
+
+static bool PlanIndexJoin(ClientContext &context, LogicalComparisonJoin &op, unique_ptr<PhysicalOperator> &plan,
+                          unique_ptr<PhysicalOperator> &left, unique_ptr<PhysicalOperator> &right, Index *index,
+                          bool swap_condition = false) {
+	if (!index) {
+		return false;
+	}
+	// index joins are not supported if there are pushed down table filters
+	D_ASSERT(right->type == PhysicalOperatorType::TABLE_SCAN);
+	auto &tbl_scan = (PhysicalTableScan &)*right;
+	//	if (tbl_scan.table_filters && !tbl_scan.table_filters->filters.empty()) {
+	//		return false;
+	//	}
+	// index joins are disabled if enable_optimizer is false
+	if (!ClientConfig::GetConfig(context).enable_optimizer) {
+		return false;
+	}
+	// check if the cardinality difference justifies an index join
+	if (!((ClientConfig::GetConfig(context).force_index_join ||
+	       left->estimated_cardinality < 0.01 * right->estimated_cardinality))) {
+		return false;
+	}
+
+	// plan the index join
+	if (swap_condition) {
+		swap(op.conditions[0].left, op.conditions[0].right);
+		swap(op.left_projection_map, op.right_projection_map);
+	}
+	plan = make_unique<PhysicalIndexJoin>(op, std::move(left), std::move(right), std::move(op.conditions), op.join_type,
+	                                      op.left_projection_map, op.right_projection_map, tbl_scan.column_ids, index,
+	                                      !swap_condition, op.estimated_cardinality);
+	return true;
+}
+
+static bool PlanIndexJoin(ClientContext &context, LogicalComparisonJoin &op, unique_ptr<PhysicalOperator> &plan,
+                          unique_ptr<PhysicalOperator> &left, unique_ptr<PhysicalOperator> &right) {
+	if (op.conditions.empty()) {
+		return false;
+	}
+	// check if we can plan an index join on the RHS
+	auto right_index = CheckIndexJoin(context, op, *right, *op.conditions[0].right);
+	if (PlanIndexJoin(context, op, plan, left, right, right_index)) {
+		return true;
+	}
+	// else check if we can plan an index join on the left side
+	auto left_index = CheckIndexJoin(context, op, *left, *op.conditions[0].left);
+	if (PlanIndexJoin(context, op, plan, right, left, left_index, true)) {
+		return true;
+	}
+	return false;
 }
 
 static void RewriteJoinCondition(Expression &expr, idx_t offset) {
@@ -184,6 +243,8 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparison
 	idx_t rhs_cardinality = op.children[1]->EstimateCardinality(context);
 	auto left = CreatePlan(*op.children[0]);
 	auto right = CreatePlan(*op.children[1]);
+	left->estimated_cardinality = lhs_cardinality;
+	right->estimated_cardinality = rhs_cardinality;
 	D_ASSERT(left && right);
 
 	if (op.conditions.empty()) {
@@ -218,22 +279,9 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparison
 
 	unique_ptr<PhysicalOperator> plan;
 	if (has_equality) {
-		Index *left_index {}, *right_index {};
-		TransformIndexJoin(context, op, &left_index, &right_index, left.get(), right.get());
-		if (left_index &&
-		    (ClientConfig::GetConfig(context).force_index_join || rhs_cardinality < 0.01 * lhs_cardinality)) {
-			auto &tbl_scan = (PhysicalTableScan &)*left;
-			swap(op.conditions[0].left, op.conditions[0].right);
-			return make_unique<PhysicalIndexJoin>(op, std::move(right), std::move(left), std::move(op.conditions),
-			                                      op.join_type, op.right_projection_map, op.left_projection_map,
-			                                      tbl_scan.column_ids, left_index, false, op.estimated_cardinality);
-		}
-		if (right_index &&
-		    (ClientConfig::GetConfig(context).force_index_join || lhs_cardinality < 0.01 * rhs_cardinality)) {
-			auto &tbl_scan = (PhysicalTableScan &)*right;
-			return make_unique<PhysicalIndexJoin>(op, std::move(left), std::move(right), std::move(op.conditions),
-			                                      op.join_type, op.left_projection_map, op.right_projection_map,
-			                                      tbl_scan.column_ids, right_index, true, op.estimated_cardinality);
+		// check if we can use an index join
+		if (PlanIndexJoin(context, op, plan, left, right)) {
+			return plan;
 		}
 		// Equality join with small number of keys : possible perfect join optimization
 		PerfectHashJoinStats perfect_join_stats;

package/src/duckdb/src/execution/physical_plan/plan_create_index.cpp

@@ -6,8 +6,9 @@
 #include "duckdb/execution/operator/order/physical_order.hpp"
 #include "duckdb/execution/physical_plan_generator.hpp"
 #include "duckdb/function/table/table_scan.hpp"
-#include "duckdb/planner/filter/null_filter.hpp"
 #include "duckdb/planner/operator/logical_create_index.hpp"
+#include "duckdb/planner/expression/bound_operator_expression.hpp"
+#include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/planner/table_filter.hpp"
 
 namespace duckdb {

package/src/duckdb/src/execution/physical_plan/plan_distinct.cpp

@@ -9,8 +9,10 @@
 
 namespace duckdb {
 
-unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreateDistinctOn(unique_ptr<PhysicalOperator> child,
-                                                                     vector<unique_ptr<Expression>> distinct_targets) {
+unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalDistinct &op) {
+	D_ASSERT(op.children.size() == 1);
+	auto child = CreatePlan(*op.children[0]);
+	auto &distinct_targets = op.distinct_targets;
 	D_ASSERT(child);
 	D_ASSERT(!distinct_targets.empty());
 
@@ -55,6 +57,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreateDistinctOn(unique_ptr<
 			FunctionBinder function_binder(context);
 			auto first_aggregate = function_binder.BindAggregateFunction(
 			    FirstFun::GetFunction(logical_type), std::move(first_children), nullptr, AggregateType::NON_DISTINCT);
+			first_aggregate->order_bys = op.order_by ? op.order_by->Copy() : nullptr;
 			// add the projection
 			projections.push_back(make_unique<BoundReferenceExpression>(logical_type, group_count + aggregates.size()));
 			// push it to the list of aggregates
@@ -81,10 +84,4 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreateDistinctOn(unique_ptr<
 	return std::move(aggr_projection);
 }
 
-unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalDistinct &op) {
-	D_ASSERT(op.children.size() == 1);
-	auto plan = CreatePlan(*op.children[0]);
-	return CreateDistinctOn(std::move(plan), std::move(op.distinct_targets));
-}
-
 } // namespace duckdb

package/src/duckdb/src/execution/physical_plan/plan_positional_join.cpp

@@ -10,12 +10,21 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalPositional
 
 	auto left = CreatePlan(*op.children[0]);
 	auto right = CreatePlan(*op.children[1]);
-	if (left->type == PhysicalOperatorType::TABLE_SCAN && right->type == PhysicalOperatorType::TABLE_SCAN) {
-		return make_unique<PhysicalPositionalScan>(op.types, std::move(left), std::move(right));
-	} else {
-		return make_unique<PhysicalPositionalJoin>(op.types, std::move(left), std::move(right),
-		                                           op.estimated_cardinality);
+	switch (left->type) {
+	case PhysicalOperatorType::TABLE_SCAN:
+	case PhysicalOperatorType::POSITIONAL_SCAN:
+		switch (right->type) {
+		case PhysicalOperatorType::TABLE_SCAN:
+		case PhysicalOperatorType::POSITIONAL_SCAN:
+			return make_unique<PhysicalPositionalScan>(op.types, std::move(left), std::move(right));
+		default:
+			break;
+		}
+	default:
+		break;
 	}
+
+	return make_unique<PhysicalPositionalJoin>(op.types, std::move(left), std::move(right), op.estimated_cardinality);
 }
 
 } // namespace duckdb

package/src/duckdb/src/execution/physical_plan_generator.cpp

@@ -115,6 +115,9 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalOperator &
 	case LogicalOperatorType::LOGICAL_DELIM_JOIN:
 		plan = CreatePlan((LogicalDelimJoin &)op);
 		break;
+	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
+		plan = CreatePlan((LogicalAsOfJoin &)op);
+		break;
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
 		plan = CreatePlan((LogicalComparisonJoin &)op);
 		break;

package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/execution/radix_partitioned_hashtable.hpp"
 #include "duckdb/parallel/task_scheduler.hpp"
 #include "duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp"
+#include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/parallel/event.hpp"
 
 namespace duckdb {
@@ -52,10 +53,13 @@ RadixPartitionedHashTable::RadixPartitionedHashTable(GroupingSet &grouping_set_p
 // Sink
 //===--------------------------------------------------------------------===//
 class RadixHTGlobalState : public GlobalSinkState {
+	constexpr const static idx_t MAX_RADIX_PARTITIONS = 32;
+
 public:
 	explicit RadixHTGlobalState(ClientContext &context)
-	    : is_empty(true), multi_scan(true), total_groups(0),
-	      partition_info((idx_t)TaskScheduler::GetScheduler(context).NumberOfThreads()) {
+	    : is_empty(true), multi_scan(true), partitioned(false),
+	      partition_info(
+	          MinValue<idx_t>(MAX_RADIX_PARTITIONS, TaskScheduler::GetScheduler(context).NumberOfThreads())) {
 	}
 
 	vector<unique_ptr<PartitionableHashTable>> intermediate_hts;
@@ -67,18 +71,19 @@ public:
 	bool multi_scan;
 	//! The lock for updating the global aggregate state
 	mutex lock;
-	//! a counter to determine if we should switch over to partitioning
-	atomic<idx_t> total_groups;
+	//! Whether or not any thread has crossed the partitioning threshold
+	atomic<bool> partitioned;
 
 	bool is_finalized = false;
 	bool is_partitioned = false;
 
 	RadixPartitionInfo partition_info;
+	AggregateHTAppendState append_state;
 };
 
 class RadixHTLocalState : public LocalSinkState {
 public:
-	explicit RadixHTLocalState(const RadixPartitionedHashTable &ht) : is_empty(true) {
+	explicit RadixHTLocalState(const RadixPartitionedHashTable &ht) : total_groups(0), is_empty(true) {
 		// if there are no groups we create a fake group so everything has the same group
 		group_chunk.InitializeEmpty(ht.group_types);
 		if (ht.grouping_set.empty()) {
@@ -89,6 +94,8 @@ public:
 	DataChunk group_chunk;
 	//! The aggregate HT
 	unique_ptr<PartitionableHashTable> ht;
+	//! The total number of groups found by this thread
+	idx_t total_groups;
 
 	//! Whether or not any tuples were added to the HT
 	bool is_empty;
@@ -145,7 +152,8 @@ void RadixPartitionedHashTable::Sink(ExecutionContext &context, GlobalSinkState
 		}
 		D_ASSERT(gstate.finalized_hts.size() == 1);
 		D_ASSERT(gstate.finalized_hts[0]);
-		gstate.total_groups += gstate.finalized_hts[0]->AddChunk(group_chunk, payload_input, filter);
+		llstate.total_groups +=
+		    gstate.finalized_hts[0]->AddChunk(gstate.append_state, group_chunk, payload_input, filter);
 		return;
 	}
 
@@ -159,9 +167,11 @@ void RadixPartitionedHashTable::Sink(ExecutionContext &context, GlobalSinkState
 		                                        group_types, op.payload_types, op.bindings);
 	}
 
-	gstate.total_groups +=
-	    llstate.ht->AddChunk(group_chunk, payload_input,
-	                         gstate.total_groups > radix_limit && gstate.partition_info.n_partitions > 1, filter);
+	llstate.total_groups += llstate.ht->AddChunk(group_chunk, payload_input,
+	                                             gstate.partitioned && gstate.partition_info.n_partitions > 1, filter);
+	if (llstate.total_groups >= radix_limit) {
+		gstate.partitioned = true;
+	}
 }
 
 void RadixPartitionedHashTable::Combine(ExecutionContext &context, GlobalSinkState &state,
@@ -182,19 +192,17 @@ void RadixPartitionedHashTable::Combine(ExecutionContext &context, GlobalSinkSta
 		return; // no data
 	}
 
-	if (!llstate.ht->IsPartitioned() && gstate.partition_info.n_partitions > 1 && gstate.total_groups > radix_limit) {
+	if (!llstate.ht->IsPartitioned() && gstate.partition_info.n_partitions > 1 && gstate.partitioned) {
 		llstate.ht->Partition();
 	}
 
-	lock_guard<mutex> glock(gstate.lock);
+	// we will never add new values to these HTs so we can drop the first part of the HT
+	llstate.ht->Finalize();
 
+	lock_guard<mutex> glock(gstate.lock);
 	if (!llstate.is_empty) {
 		gstate.is_empty = false;
 	}
-
-	// we will never add new values to these HTs so we can drop the first part of the HT
-	llstate.ht->Finalize();
-
 	// at this point we just collect them the PhysicalHashAggregateFinalizeTask (below) will merge them in parallel
 	gstate.intermediate_hts.push_back(std::move(llstate.ht));
 }