duckdb 0.7.2-dev0.0 → 0.7.2-dev1034.0

package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp

@@ -10,7 +10,7 @@
 #include "duckdb/function/table/table_scan.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/planner/operator/logical_comparison_join.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/transaction/duck_transaction.hpp"
 #include "duckdb/common/operator/subtract.hpp"
 #include "duckdb/execution/operator/join/physical_blockwise_nl_join.hpp"
@@ -92,19 +92,21 @@ void CheckForPerfectJoinOpt(LogicalComparisonJoin &op, PerfectHashJoinStats &joi
 	}
 	// with integral internal types
 	for (auto &&join_stat : op.join_stats) {
-		if (!TypeIsInteger(join_stat->type.InternalType()) || join_stat->type.InternalType() == PhysicalType::INT128) {
+		if (!TypeIsInteger(join_stat->GetType().InternalType()) ||
+		    join_stat->GetType().InternalType() == PhysicalType::INT128) {
 			// perfect join not possible for non-integral types or hugeint
 			return;
 		}
 	}
 
 	// and when the build range is smaller than the threshold
-	auto stats_build = reinterpret_cast<NumericStatistics *>(op.join_stats[0].get()); // lhs stats
-	if (stats_build->min.IsNull() || stats_build->max.IsNull()) {
+	auto &stats_build = *op.join_stats[0].get(); // lhs stats
+	if (!NumericStats::HasMinMax(stats_build)) {
 		return;
 	}
 	int64_t min_value, max_value;
-	if (!ExtractNumericValue(stats_build->min, min_value) || !ExtractNumericValue(stats_build->max, max_value)) {
+	if (!ExtractNumericValue(NumericStats::Min(stats_build), min_value) ||
+	    !ExtractNumericValue(NumericStats::Max(stats_build), max_value)) {
 		return;
 	}
 	int64_t build_range;
@@ -113,20 +115,24 @@ void CheckForPerfectJoinOpt(LogicalComparisonJoin &op, PerfectHashJoinStats &joi
 	}
 
 	// Fill join_stats for invisible join
-	auto stats_probe = reinterpret_cast<NumericStatistics *>(op.join_stats[1].get()); // rhs stats
+	auto &stats_probe = *op.join_stats[1].get(); // rhs stats
+	if (!NumericStats::HasMinMax(stats_probe)) {
+		return;
+	}
 
 	// The max size our build must have to run the perfect HJ
 	const idx_t MAX_BUILD_SIZE = 1000000;
-	join_state.probe_min = stats_probe->min;
-	join_state.probe_max = stats_probe->max;
-	join_state.build_min = stats_build->min;
-	join_state.build_max = stats_build->max;
+	join_state.probe_min = NumericStats::Min(stats_probe);
+	join_state.probe_max = NumericStats::Max(stats_probe);
+	join_state.build_min = NumericStats::Min(stats_build);
+	join_state.build_max = NumericStats::Max(stats_build);
 	join_state.estimated_cardinality = op.estimated_cardinality;
 	join_state.build_range = build_range;
-	if (join_state.build_range > MAX_BUILD_SIZE || stats_probe->max.IsNull() || stats_probe->min.IsNull()) {
+	if (join_state.build_range > MAX_BUILD_SIZE) {
 		return;
 	}
-	if (stats_build->min <= stats_probe->min && stats_probe->max <= stats_build->max) {
+	if (NumericStats::Min(stats_build) <= NumericStats::Min(stats_probe) &&
+	    NumericStats::Max(stats_probe) <= NumericStats::Max(stats_build)) {
 		join_state.is_probe_in_domain = true;
 	}
 	join_state.is_build_small = true;
@@ -146,27 +152,80 @@ static void CanUseIndexJoin(TableScanBindData *tbl, Expression &expr, Index **re
 	});
 }
 
-void TransformIndexJoin(ClientContext &context, LogicalComparisonJoin &op, Index **left_index, Index **right_index,
-                        PhysicalOperator *left, PhysicalOperator *right) {
+Index *CheckIndexJoin(ClientContext &context, LogicalComparisonJoin &op, PhysicalOperator &plan,
+                      Expression &condition) {
+	if (op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN) {
+		return nullptr;
+	}
 	// check if one of the tables has an index on column
-	if (op.join_type == JoinType::INNER && op.conditions.size() == 1) {
-		// check if one of the children are table scans and if they have an index in the join attribute
-		// (op.condition)
-		if (left->type == PhysicalOperatorType::TABLE_SCAN) {
-			auto &tbl_scan = (PhysicalTableScan &)*left;
-			auto tbl = dynamic_cast<TableScanBindData *>(tbl_scan.bind_data.get());
-			if (CanPlanIndexJoin(context, tbl, tbl_scan)) {
-				CanUseIndexJoin(tbl, *op.conditions[0].left, left_index);
-			}
-		}
-		if (right->type == PhysicalOperatorType::TABLE_SCAN) {
-			auto &tbl_scan = (PhysicalTableScan &)*right;
-			auto tbl = dynamic_cast<TableScanBindData *>(tbl_scan.bind_data.get());
-			if (CanPlanIndexJoin(context, tbl, tbl_scan)) {
-				CanUseIndexJoin(tbl, *op.conditions[0].right, right_index);
-			}
-		}
+	if (op.join_type != JoinType::INNER) {
+		return nullptr;
+	}
+	if (op.conditions.size() != 1) {
+		return nullptr;
+	}
+	// check if the child is (1) a table scan, and (2) has an index on the join condition
+	if (plan.type != PhysicalOperatorType::TABLE_SCAN) {
+		return nullptr;
+	}
+	auto &tbl_scan = (PhysicalTableScan &)plan;
+	auto tbl = dynamic_cast<TableScanBindData *>(tbl_scan.bind_data.get());
+	Index *result = nullptr;
+	if (CanPlanIndexJoin(context, tbl, tbl_scan)) {
+		CanUseIndexJoin(tbl, condition, &result);
+	}
+	return result;
+}
+
+static bool PlanIndexJoin(ClientContext &context, LogicalComparisonJoin &op, unique_ptr<PhysicalOperator> &plan,
+                          unique_ptr<PhysicalOperator> &left, unique_ptr<PhysicalOperator> &right, Index *index,
+                          bool swap_condition = false) {
+	if (!index) {
+		return false;
+	}
+	// index joins are not supported if there are pushed down table filters
+	D_ASSERT(right->type == PhysicalOperatorType::TABLE_SCAN);
+	auto &tbl_scan = (PhysicalTableScan &)*right;
+	//	if (tbl_scan.table_filters && !tbl_scan.table_filters->filters.empty()) {
+	//		return false;
+	//	}
+	// index joins are disabled if enable_optimizer is false
+	if (!ClientConfig::GetConfig(context).enable_optimizer) {
+		return false;
+	}
+	// check if the cardinality difference justifies an index join
+	if (!((ClientConfig::GetConfig(context).force_index_join ||
+	       left->estimated_cardinality < 0.01 * right->estimated_cardinality))) {
+		return false;
+	}
+
+	// plan the index join
+	if (swap_condition) {
+		swap(op.conditions[0].left, op.conditions[0].right);
+		swap(op.left_projection_map, op.right_projection_map);
+	}
+	plan = make_unique<PhysicalIndexJoin>(op, std::move(left), std::move(right), std::move(op.conditions), op.join_type,
+	                                      op.left_projection_map, op.right_projection_map, tbl_scan.column_ids, index,
+	                                      !swap_condition, op.estimated_cardinality);
+	return true;
+}
+
+static bool PlanIndexJoin(ClientContext &context, LogicalComparisonJoin &op, unique_ptr<PhysicalOperator> &plan,
+                          unique_ptr<PhysicalOperator> &left, unique_ptr<PhysicalOperator> &right) {
+	if (op.conditions.empty()) {
+		return false;
+	}
+	// check if we can plan an index join on the RHS
+	auto right_index = CheckIndexJoin(context, op, *right, *op.conditions[0].right);
+	if (PlanIndexJoin(context, op, plan, left, right, right_index)) {
+		return true;
 	}
+	// else check if we can plan an index join on the left side
+	auto left_index = CheckIndexJoin(context, op, *left, *op.conditions[0].left);
+	if (PlanIndexJoin(context, op, plan, right, left, left_index, true)) {
+		return true;
+	}
+	return false;
 }
 
 static void RewriteJoinCondition(Expression &expr, idx_t offset) {
@@ -184,6 +243,8 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparison
 	idx_t rhs_cardinality = op.children[1]->EstimateCardinality(context);
 	auto left = CreatePlan(*op.children[0]);
 	auto right = CreatePlan(*op.children[1]);
+	left->estimated_cardinality = lhs_cardinality;
+	right->estimated_cardinality = rhs_cardinality;
 	D_ASSERT(left && right);
 
 	if (op.conditions.empty()) {
@@ -218,22 +279,9 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparison
 
 	unique_ptr<PhysicalOperator> plan;
 	if (has_equality) {
-		Index *left_index {}, *right_index {};
-		TransformIndexJoin(context, op, &left_index, &right_index, left.get(), right.get());
-		if (left_index &&
-		    (ClientConfig::GetConfig(context).force_index_join || rhs_cardinality < 0.01 * lhs_cardinality)) {
-			auto &tbl_scan = (PhysicalTableScan &)*left;
-			swap(op.conditions[0].left, op.conditions[0].right);
-			return make_unique<PhysicalIndexJoin>(op, std::move(right), std::move(left), std::move(op.conditions),
-			                                      op.join_type, op.right_projection_map, op.left_projection_map,
-			                                      tbl_scan.column_ids, left_index, false, op.estimated_cardinality);
-		}
-		if (right_index &&
-		    (ClientConfig::GetConfig(context).force_index_join || lhs_cardinality < 0.01 * rhs_cardinality)) {
-			auto &tbl_scan = (PhysicalTableScan &)*right;
-			return make_unique<PhysicalIndexJoin>(op, std::move(left), std::move(right), std::move(op.conditions),
-			                                      op.join_type, op.left_projection_map, op.right_projection_map,
-			                                      tbl_scan.column_ids, right_index, true, op.estimated_cardinality);
+		// check if we can use an index join
+		if (PlanIndexJoin(context, op, plan, left, right)) {
+			return plan;
 		}
 		// Equality join with small number of keys : possible perfect join optimization
 		PerfectHashJoinStats perfect_join_stats;

package/src/duckdb/src/execution/physical_plan/plan_distinct.cpp

@@ -9,8 +9,10 @@
 
 namespace duckdb {
 
-unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreateDistinctOn(unique_ptr<PhysicalOperator> child,
-                                                                     vector<unique_ptr<Expression>> distinct_targets) {
+unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalDistinct &op) {
+	D_ASSERT(op.children.size() == 1);
+	auto child = CreatePlan(*op.children[0]);
+	auto &distinct_targets = op.distinct_targets;
 	D_ASSERT(child);
 	D_ASSERT(!distinct_targets.empty());
 
@@ -55,6 +57,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreateDistinctOn(unique_ptr<
 			FunctionBinder function_binder(context);
 			auto first_aggregate = function_binder.BindAggregateFunction(
 			    FirstFun::GetFunction(logical_type), std::move(first_children), nullptr, AggregateType::NON_DISTINCT);
+			first_aggregate->order_bys = op.order_by ? op.order_by->Copy() : nullptr;
 			// add the projection
 			projections.push_back(make_unique<BoundReferenceExpression>(logical_type, group_count + aggregates.size()));
 			// push it to the list of aggregates
@@ -81,10 +84,4 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreateDistinctOn(unique_ptr<
 	return std::move(aggr_projection);
 }
 
-unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalDistinct &op) {
-	D_ASSERT(op.children.size() == 1);
-	auto plan = CreatePlan(*op.children[0]);
-	return CreateDistinctOn(std::move(plan), std::move(op.distinct_targets));
-}
-
 } // namespace duckdb

package/src/duckdb/src/execution/physical_plan/plan_positional_join.cpp

@@ -10,12 +10,21 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalPositional
 
 	auto left = CreatePlan(*op.children[0]);
 	auto right = CreatePlan(*op.children[1]);
-	if (left->type == PhysicalOperatorType::TABLE_SCAN && right->type == PhysicalOperatorType::TABLE_SCAN) {
-		return make_unique<PhysicalPositionalScan>(op.types, std::move(left), std::move(right));
-	} else {
-		return make_unique<PhysicalPositionalJoin>(op.types, std::move(left), std::move(right),
-		                                           op.estimated_cardinality);
+	switch (left->type) {
+	case PhysicalOperatorType::TABLE_SCAN:
+	case PhysicalOperatorType::POSITIONAL_SCAN:
+		switch (right->type) {
+		case PhysicalOperatorType::TABLE_SCAN:
+		case PhysicalOperatorType::POSITIONAL_SCAN:
+			return make_unique<PhysicalPositionalScan>(op.types, std::move(left), std::move(right));
+		default:
+			break;
+		}
+	default:
+		break;
 	}
+
+	return make_unique<PhysicalPositionalJoin>(op.types, std::move(left), std::move(right), op.estimated_cardinality);
 }
 
 } // namespace duckdb

package/src/duckdb/src/execution/physical_plan_generator.cpp

@@ -115,6 +115,9 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalOperator &
 	case LogicalOperatorType::LOGICAL_DELIM_JOIN:
 		plan = CreatePlan((LogicalDelimJoin &)op);
 		break;
+	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
+		plan = CreatePlan((LogicalAsOfJoin &)op);
+		break;
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
 		plan = CreatePlan((LogicalComparisonJoin &)op);
 		break;

package/src/duckdb/src/execution/window_segment_tree.cpp

@@ -6,6 +6,178 @@
 
 namespace duckdb {
 
+//===--------------------------------------------------------------------===//
+// WindowAggregateState
+//===--------------------------------------------------------------------===//
+
+WindowAggregateState::WindowAggregateState(AggregateFunction &aggregate, FunctionData *bind_info,
+                                           const LogicalType &result_type_p)
+    : aggregate(aggregate), bind_info(bind_info), result_type(result_type_p), state(aggregate.state_size()),
+      statev(Value::POINTER((idx_t)state.data())), statep(Value::POINTER((idx_t)state.data())) {
+	statev.SetVectorType(VectorType::FLAT_VECTOR); // Prevent conversion of results to constants
+}
+
+WindowAggregateState::~WindowAggregateState() {
+}
+
+void WindowAggregateState::AggregateInit() {
+	aggregate.initialize(state.data());
+}
+
+void WindowAggregateState::AggegateFinal(Vector &result, idx_t rid) {
+	AggregateInputData aggr_input_data(bind_info, Allocator::DefaultAllocator());
+	aggregate.finalize(statev, aggr_input_data, result, 1, rid);
+
+	if (aggregate.destructor) {
+		aggregate.destructor(statev, 1);
+	}
+}
+
+void WindowAggregateState::Sink(DataChunk &payload_chunk, SelectionVector *filter_sel, idx_t filtered) {
+}
+
+void WindowAggregateState::Finalize() {
+}
+
+void WindowAggregateState::Compute(Vector &result, idx_t rid, idx_t start, idx_t end) {
+}
+
+//===--------------------------------------------------------------------===//
+// WindowConstantAggregate
+//===--------------------------------------------------------------------===//
+
+WindowConstantAggregate::WindowConstantAggregate(AggregateFunction &aggregate, FunctionData *bind_info,
+                                                 const LogicalType &result_type, const ValidityMask &partition_mask,
+                                                 const idx_t count)
+    : WindowAggregateState(aggregate, bind_info, result_type), partition(0), row(0) {
+
+	// Locate the partition boundaries
+	idx_t start = 0;
+	if (partition_mask.AllValid()) {
+		partition_offsets.emplace_back(0);
+	} else {
+		idx_t entry_idx;
+		idx_t shift;
+		while (start < count) {
+			partition_mask.GetEntryIndex(start, entry_idx, shift);
+
+			//	If start is aligned with the start of a block,
+			//	and the block is blank, then skip forward one block.
+			const auto block = partition_mask.GetValidityEntry(entry_idx);
+			if (partition_mask.NoneValid(block) && !shift) {
+				start += ValidityMask::BITS_PER_VALUE;
+				continue;
+			}
+
+			// Loop over the block
+			for (; shift < ValidityMask::BITS_PER_VALUE && start < count; ++shift, ++start) {
+				if (partition_mask.RowIsValid(block, shift)) {
+					partition_offsets.emplace_back(start);
+				}
+			}
+		}
+	}
+
+	//	Initialise the vector for caching the results
+	results = make_unique<Vector>(result_type, partition_offsets.size());
+	partition_offsets.emplace_back(count);
+
+	//	Start the first aggregate
+	AggregateInit();
+}
+
+void WindowConstantAggregate::Sink(DataChunk &payload_chunk, SelectionVector *filter_sel, idx_t filtered) {
+	const auto chunk_begin = row;
+	const auto chunk_end = chunk_begin + payload_chunk.size();
+
+	if (!inputs.ColumnCount() && payload_chunk.ColumnCount()) {
+		inputs.Initialize(Allocator::DefaultAllocator(), payload_chunk.GetTypes());
+	}
+
+	AggregateInputData aggr_input_data(bind_info, Allocator::DefaultAllocator());
+	idx_t begin = 0;
+	idx_t filter_idx = 0;
+	auto partition_end = partition_offsets[partition + 1];
+	while (row < chunk_end) {
+		if (row == partition_end) {
+			AggegateFinal(*results, partition++);
+			AggregateInit();
+			partition_end = partition_offsets[partition + 1];
+		}
+		partition_end = MinValue(partition_end, chunk_end);
+		auto end = partition_end - chunk_begin;
+
+		inputs.Reset();
+		if (begin) {
+			for (idx_t c = 0; c < payload_chunk.ColumnCount(); ++c) {
+				inputs.data[c].Slice(payload_chunk.data[c], begin, end);
+			}
+		} else {
+			inputs.Reference(payload_chunk);
+		}
+		inputs.SetCardinality(end - begin);
+
+		// Slice to any filtered rows
+		SelectionVector sel;
+		if (filter_sel) {
+			//	Find the first value in [begin, end)
+			for (; filter_idx < filtered; ++filter_idx) {
+				auto idx = filter_sel->get_index(filter_idx);
+				if (idx >= begin) {
+					break;
+				}
+			}
+			sel.Initialize(filter_sel->data());
+			idx_t nsel = 0;
+			for (; filter_idx < filtered; ++filter_idx, ++nsel) {
+				auto idx = filter_sel->get_index(filter_idx);
+				if (idx >= end) {
+					break;
+				}
+			}
+
+			if (nsel != inputs.size()) {
+				inputs.Slice(sel, nsel);
+			}
+		}
+
+		//	Aggregate the filtered rows into a single state
+		const auto count = inputs.size();
+		if (aggregate.simple_update) {
+			aggregate.simple_update(inputs.data.data(), aggr_input_data, inputs.ColumnCount(), state.data(), count);
+		} else {
+			aggregate.update(inputs.data.data(), aggr_input_data, inputs.ColumnCount(), statep, count);
+		}
+
+		//	Skip filtered rows too!
+		row += end - begin;
+		begin = end;
+	}
+}
+
+void WindowConstantAggregate::Finalize() {
+	AggegateFinal(*results, partition++);
+
+	partition = 0;
+	row = 0;
+}
+
+void WindowConstantAggregate::Compute(Vector &target, idx_t rid, idx_t start, idx_t end) {
+	//	Find the partition containing [start, end)
+	while (start < partition_offsets[partition] || partition_offsets[partition + 1] <= start) {
+		++partition;
+	}
+	D_ASSERT(partition_offsets[partition] <= start);
+	D_ASSERT(partition + 1 < partition_offsets.size());
+	D_ASSERT(end <= partition_offsets[partition + 1]);
+
+	// Copy the value
+	VectorOperations::Copy(*results, target, partition + 1, partition, rid);
+}
+
+//===--------------------------------------------------------------------===//
+// WindowSegmentTree
+//===--------------------------------------------------------------------===//
 WindowSegmentTree::WindowSegmentTree(AggregateFunction &aggregate, FunctionData *bind_info,
                                      const LogicalType &result_type_p, DataChunk *input,
                                      const ValidityMask &filter_mask_p, WindowAggregationMode mode_p)
@@ -107,7 +279,7 @@ void WindowSegmentTree::WindowSegmentValue(idx_t l_idx, idx_t begin, idx_t end)
 	if (l_idx == 0) {
 		ExtractFrame(begin, end);
 		AggregateInputData aggr_input_data(bind_info, Allocator::DefaultAllocator());
-		D_ASSERT(inputs.data.size() > 0);
+		D_ASSERT(!inputs.data.empty());
 		aggregate.update(&inputs.data[0], aggr_input_data, input_ref->ColumnCount(), s, inputs.size());
 	} else {
 		// find out where the states begin

package/src/duckdb/src/function/aggregate/algebraic/avg.cpp

@@ -127,9 +127,6 @@ struct NumericAverageOperation : public BaseSumOperation<AverageSetOperation, Re
 		if (state->count == 0) {
 			mask.SetInvalid(idx);
 		} else {
-			if (!Value::DoubleIsFinite(state->value)) {
-				throw OutOfRangeException("AVG is out of range!");
-			}
 			target[idx] = (state->value / state->count);
 		}
 	}
@@ -141,9 +138,6 @@ struct KahanAverageOperation : public BaseSumOperation<AverageSetOperation, Kaha
 		if (state->count == 0) {
 			mask.SetInvalid(idx);
 		} else {
-			if (!Value::DoubleIsFinite(state->value)) {
-				throw OutOfRangeException("AVG is out of range!");
-			}
 			target[idx] = (state->value / state->count) + (state->err / state->count);
 		}
 	}