duckdb 0.7.2-dev12.0 → 0.7.2-dev1244.0

package/src/duckdb/src/execution/column_binding_resolver.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/execution/column_binding_resolver.hpp"
 
 #include "duckdb/planner/operator/logical_comparison_join.hpp"
+#include "duckdb/planner/operator/logical_any_join.hpp"
 #include "duckdb/planner/operator/logical_create_index.hpp"
 #include "duckdb/planner/operator/logical_delim_join.hpp"
 #include "duckdb/planner/operator/logical_insert.hpp"
@@ -17,7 +18,10 @@ ColumnBindingResolver::ColumnBindingResolver() {
 }
 
 void ColumnBindingResolver::VisitOperator(LogicalOperator &op) {
-	if (op.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN || op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN) {
+	switch (op.type) {
+	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
+	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
+	case LogicalOperatorType::LOGICAL_DELIM_JOIN: {
 		// special case: comparison join
 		auto &comp_join = (LogicalComparisonJoin &)op;
 		// first get the bindings of the LHS and resolve the LHS expressions
@@ -40,27 +44,36 @@ void ColumnBindingResolver::VisitOperator(LogicalOperator &op) {
 		// finally update the bindings with the result bindings of the join
 		bindings = op.GetColumnBindings();
 		return;
-	} else if (op.type == LogicalOperatorType::LOGICAL_ANY_JOIN) {
+	}
+	case LogicalOperatorType::LOGICAL_ANY_JOIN: {
 		// ANY join, this join is different because we evaluate the expression on the bindings of BOTH join sides at
 		// once i.e. we set the bindings first to the bindings of the entire join, and then resolve the expressions of
 		// this operator
 		VisitOperatorChildren(op);
 		bindings = op.GetColumnBindings();
+		auto &any_join = (LogicalAnyJoin &)op;
+		if (any_join.join_type == JoinType::SEMI || any_join.join_type == JoinType::ANTI) {
+			auto right_bindings = op.children[1]->GetColumnBindings();
+			bindings.insert(bindings.end(), right_bindings.begin(), right_bindings.end());
+		}
 		VisitOperatorExpressions(op);
 		return;
-	} else if (op.type == LogicalOperatorType::LOGICAL_CREATE_INDEX) {
+	}
+	case LogicalOperatorType::LOGICAL_CREATE_INDEX: {
 		// CREATE INDEX statement, add the columns of the table with table index 0 to the binding set
 		// afterwards bind the expressions of the CREATE INDEX statement
 		auto &create_index = (LogicalCreateIndex &)op;
 		bindings = LogicalOperator::GenerateColumnBindings(0, create_index.table.GetColumns().LogicalColumnCount());
 		VisitOperatorExpressions(op);
 		return;
-	} else if (op.type == LogicalOperatorType::LOGICAL_GET) {
+	}
+	case LogicalOperatorType::LOGICAL_GET: {
 		//! We first need to update the current set of bindings and then visit operator expressions
 		bindings = op.GetColumnBindings();
 		VisitOperatorExpressions(op);
 		return;
-	} else if (op.type == LogicalOperatorType::LOGICAL_INSERT) {
+	}
+	case LogicalOperatorType::LOGICAL_INSERT: {
 		//! We want to execute the normal path, but also add a dummy 'excluded' binding if there is a
 		// ON CONFLICT DO UPDATE clause
 		auto &insert_op = (LogicalInsert &)op;
@@ -83,6 +96,9 @@ void ColumnBindingResolver::VisitOperator(LogicalOperator &op) {
 			return;
 		}
 	}
+	default:
+		break;
+	}
 	// general case
 	// first visit the children of this operator
 	VisitOperatorChildren(op);

package/src/duckdb/src/execution/expression_executor/execute_cast.cpp

@@ -11,7 +11,8 @@ unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const BoundCastE
 	result->AddChild(expr.child.get());
 	result->Finalize();
 	if (expr.bound_cast.init_local_state) {
-		result->local_state = expr.bound_cast.init_local_state(root.executor->GetContext());
+		CastLocalStateParameters parameters(root.executor->GetContext(), expr.bound_cast.cast_data);
+		result->local_state = expr.bound_cast.init_local_state(parameters);
 	}
 	return std::move(result);
 }

package/src/duckdb/src/execution/expression_executor/execute_comparison.cpp

@@ -266,12 +266,12 @@ idx_t VectorOperations::GreaterThanEquals(Vector &left, Vector &right, const Sel
 
 idx_t VectorOperations::LessThan(Vector &left, Vector &right, const SelectionVector *sel, idx_t count,
                                  SelectionVector *true_sel, SelectionVector *false_sel) {
-	return TemplatedSelectOperation<duckdb::LessThan>(left, right, sel, count, true_sel, false_sel);
+	return TemplatedSelectOperation<duckdb::GreaterThan>(right, left, sel, count, true_sel, false_sel);
 }
 
 idx_t VectorOperations::LessThanEquals(Vector &left, Vector &right, const SelectionVector *sel, idx_t count,
                                        SelectionVector *true_sel, SelectionVector *false_sel) {
-	return TemplatedSelectOperation<duckdb::LessThanEquals>(left, right, sel, count, true_sel, false_sel);
+	return TemplatedSelectOperation<duckdb::GreaterThanEquals>(right, left, sel, count, true_sel, false_sel);
 }
 
 idx_t ExpressionExecutor::Select(const BoundComparisonExpression &expr, ExpressionState *state,

package/src/duckdb/src/execution/index/art/art.cpp

@@ -6,6 +6,7 @@
 #include "duckdb/storage/arena_allocator.hpp"
 #include "duckdb/execution/index/art/art_key.hpp"
 #include "duckdb/common/types/conflict_manager.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 #include <algorithm>
 #include <cstring>
@@ -13,6 +14,18 @@
 
 namespace duckdb {
 
+struct ARTIndexScanState : public IndexScanState {
+
+	//! Scan predicates (single predicate scan or range scan)
+	Value values[2];
+	//! Expressions of the scan predicates
+	ExpressionType expressions[2];
+	bool checked = false;
+	//! All scanned row IDs
+	vector<row_t> result_ids;
+	Iterator iterator;
+};
+
 ART::ART(const vector<column_t> &column_ids, TableIOManager &table_io_manager,
          const vector<unique_ptr<Expression>> &unbound_expressions, IndexConstraintType constraint_type,
          AttachedDatabase &db, bool track_memory, idx_t block_id, idx_t block_offset)
@@ -330,8 +343,7 @@ bool ART::ConstructFromSorted(idx_t count, vector<Key> &keys, Vector &row_identi
 //===--------------------------------------------------------------------===//
 // Insert / Verification / Constraint Checking
 //===--------------------------------------------------------------------===//
-
-bool ART::Insert(IndexLock &lock, DataChunk &input, Vector &row_ids) {
+PreservedError ART::Insert(IndexLock &lock, DataChunk &input, Vector &row_ids) {
 
 	D_ASSERT(row_ids.GetType().InternalType() == ROW_TYPE);
 	D_ASSERT(logical_types[0] == input.data[0].GetType());
@@ -375,12 +387,13 @@ bool ART::Insert(IndexLock &lock, DataChunk &input, Vector &row_ids) {
 
 	IncreaseAndVerifyMemorySize(old_memory_size);
 	if (failed_index != DConstants::INVALID_INDEX) {
-		return false;
+		return PreservedError(ConstraintException("PRIMARY KEY or UNIQUE constraint violated: duplicate key \"%s\"",
+		                                          AppendRowError(input, failed_index)));
 	}
-	return true;
+	return PreservedError();
 }
 
-bool ART::Append(IndexLock &lock, DataChunk &appended_data, Vector &row_identifiers) {
+PreservedError ART::Append(IndexLock &lock, DataChunk &appended_data, Vector &row_identifiers) {
 	DataChunk expression_result;
 	expression_result.Initialize(Allocator::DefaultAllocator(), logical_types);
 
@@ -654,6 +667,7 @@ bool ART::SearchEqual(Key &key, idx_t max_count, vector<row_t> &result_ids) {
 }
 
 void ART::SearchEqualJoinNoFetch(Key &key, idx_t &result_size) {
+	result_size = 0;
 
 	// we need to look for a leaf
 	auto old_memory_size = memory_size;

package/src/duckdb/src/execution/join_hashtable.cpp

@@ -219,7 +219,9 @@ void JoinHashTable::Build(DataChunk &keys, DataChunk &payload) {
 		}
 		info.correlated_payload.SetCardinality(keys);
 		info.correlated_payload.data[0].Reference(keys.data[info.correlated_types.size()]);
-		info.correlated_counts->AddChunk(info.group_chunk, info.correlated_payload, AggregateType::NON_DISTINCT);
+		AggregateHTAppendState append_state;
+		info.correlated_counts->AddChunk(append_state, info.group_chunk, info.correlated_payload,
+		                                 AggregateType::NON_DISTINCT);
 	}
 
 	// prepare the keys for processing

package/src/duckdb/src/execution/operator/aggregate/physical_hash_aggregate.cpp

@@ -3,13 +3,13 @@
 #include "duckdb/catalog/catalog_entry/aggregate_function_catalog_entry.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/execution/aggregate_hashtable.hpp"
-#include "duckdb/execution/partitionable_hashtable.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/parallel/pipeline.hpp"
 #include "duckdb/parallel/task_scheduler.hpp"
 #include "duckdb/parallel/thread_context.hpp"
 #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
+#include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/parallel/base_pipeline_event.hpp"
 #include "duckdb/common/atomic.hpp"
 #include "duckdb/execution/operator/aggregate/distinct_aggregate_data.hpp"

package/src/duckdb/src/execution/operator/aggregate/physical_perfecthash_aggregate.cpp

@@ -4,14 +4,13 @@
 #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
 
 namespace duckdb {
 
 PhysicalPerfectHashAggregate::PhysicalPerfectHashAggregate(ClientContext &context, vector<LogicalType> types_p,
                                                            vector<unique_ptr<Expression>> aggregates_p,
                                                            vector<unique_ptr<Expression>> groups_p,
-                                                           vector<unique_ptr<BaseStatistics>> group_stats,
+                                                           const vector<unique_ptr<BaseStatistics>> &group_stats,
                                                            vector<idx_t> required_bits_p, idx_t estimated_cardinality)
     : PhysicalOperator(PhysicalOperatorType::PERFECT_HASH_GROUP_BY, std::move(types_p), estimated_cardinality),
       groups(std::move(groups_p)), aggregates(std::move(aggregates_p)), required_bits(std::move(required_bits_p)) {
@@ -19,9 +18,9 @@ PhysicalPerfectHashAggregate::PhysicalPerfectHashAggregate(ClientContext &contex
 	group_minima.reserve(group_stats.size());
 	for (auto &stats : group_stats) {
 		D_ASSERT(stats);
-		auto &nstats = (NumericStatistics &)*stats;
-		D_ASSERT(!nstats.min.IsNull());
-		group_minima.push_back(std::move(nstats.min));
+		auto &nstats = *stats;
+		D_ASSERT(NumericStats::HasMin(nstats));
+		group_minima.push_back(NumericStats::Min(nstats));
 	}
 	for (auto &expr : groups) {
 		group_types.push_back(expr->return_type);

package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp

@@ -111,7 +111,7 @@ public:
 				orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_FIRST, pexpr->Copy(), nullptr);
 			} else {
 				orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_FIRST, pexpr->Copy(),
-				                    wexpr->partitions_stats[prt_idx]->Copy());
+				                    wexpr->partitions_stats[prt_idx]->ToUnique());
 			}
 			partitions.emplace_back(orders.back().Copy());
 		}
@@ -994,7 +994,10 @@ void WindowBoundariesState::Update(const idx_t row_idx, WindowInputColumn &range
 }
 
 struct WindowExecutor {
-	WindowExecutor(BoundWindowExpression *wexpr, ClientContext &context, const idx_t count);
+	static bool IsConstantAggregate(const BoundWindowExpression &wexpr);
+
+	WindowExecutor(BoundWindowExpression *wexpr, ClientContext &context, const ValidityMask &partition_mask,
+	               const idx_t count);
 
 	void Sink(DataChunk &input_chunk, const idx_t input_idx, const idx_t total_count);
 	void Finalize(WindowAggregationMode mode);
@@ -1038,9 +1041,67 @@ struct WindowExecutor {
 	// build a segment tree for frame-adhering aggregates
 	// see http://www.vldb.org/pvldb/vol8/p1058-leis.pdf
 	unique_ptr<WindowSegmentTree> segment_tree = nullptr;
+
+	// all aggregate values are the same for each partition
+	unique_ptr<WindowConstantAggregate> constant_aggregate = nullptr;
 };
 
-WindowExecutor::WindowExecutor(BoundWindowExpression *wexpr, ClientContext &context, const idx_t count)
+bool WindowExecutor::IsConstantAggregate(const BoundWindowExpression &wexpr) {
+	if (!wexpr.aggregate) {
+		return false;
+	}
+
+	//	COUNT(*) is already handled efficiently by segment trees.
+	if (wexpr.children.empty()) {
+		return false;
+	}
+
+	/*
+	    The default framing option is RANGE UNBOUNDED PRECEDING, which
+	    is the same as RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT
+	    ROW; it sets the frame to be all rows from the partition start
+	    up through the current row's last peer (a row that the window's
+	    ORDER BY clause considers equivalent to the current row; all
+	    rows are peers if there is no ORDER BY). In general, UNBOUNDED
+	    PRECEDING means that the frame starts with the first row of the
+	    partition, and similarly UNBOUNDED FOLLOWING means that the
+	    frame ends with the last row of the partition, regardless of
+	    RANGE, ROWS or GROUPS mode. In ROWS mode, CURRENT ROW means that
+	    the frame starts or ends with the current row; but in RANGE or
+	    GROUPS mode it means that the frame starts or ends with the
+	    current row's first or last peer in the ORDER BY ordering. The
+	    offset PRECEDING and offset FOLLOWING options vary in meaning
+	    depending on the frame mode.
+	*/
+	switch (wexpr.start) {
+	case WindowBoundary::UNBOUNDED_PRECEDING:
+		break;
+	case WindowBoundary::CURRENT_ROW_RANGE:
+		if (!wexpr.orders.empty()) {
+			return false;
+		}
+		break;
+	default:
+		return false;
+	}
+
+	switch (wexpr.end) {
+	case WindowBoundary::UNBOUNDED_FOLLOWING:
+		break;
+	case WindowBoundary::CURRENT_ROW_RANGE:
+		if (!wexpr.orders.empty()) {
+			return false;
+		}
+		break;
+	default:
+		return false;
+	}
+
+	return true;
+}
+
+WindowExecutor::WindowExecutor(BoundWindowExpression *wexpr, ClientContext &context, const ValidityMask &partition_mask,
+                               const idx_t count)
     : wexpr(wexpr), bounds(wexpr, count), payload_collection(), payload_executor(context), filter_executor(context),
       leadlag_offset(wexpr->offset_expr.get(), context), leadlag_default(wexpr->default_expr.get(), context),
       boundary_start(wexpr->start_expr.get(), context), boundary_end(wexpr->end_expr.get(), context),
@@ -1050,6 +1111,12 @@ WindowExecutor::WindowExecutor(BoundWindowExpression *wexpr, ClientContext &cont
 {
 	// TODO we could evaluate those expressions in parallel
 
+	//	Check for constant aggregate
+	if (IsConstantAggregate(*wexpr)) {
+		constant_aggregate = make_unique<WindowConstantAggregate>(*(wexpr->aggregate), wexpr->bind_info.get(),
+		                                                          wexpr->return_type, partition_mask, count);
+	}
+
 	// evaluate the FILTER clause and stuff it into a large mask for compactness and reuse
 	if (wexpr->filter_expr) {
 		// 	Start with all invalid and set the ones that pass
@@ -1097,11 +1164,25 @@ void WindowExecutor::Sink(DataChunk &input_chunk, const idx_t input_idx, const i
 
 	const auto count = input_chunk.size();
 
+	idx_t filtered = 0;
+	SelectionVector *filtering = nullptr;
+	if (wexpr->filter_expr) {
+		filtering = &filter_sel;
+		filtered = filter_executor.SelectExpression(input_chunk, filter_sel);
+		for (idx_t f = 0; f < filtered; ++f) {
+			filter_mask.SetValid(input_idx + filter_sel[f]);
+		}
+	}
+
 	if (!wexpr->children.empty()) {
 		payload_chunk.Reset();
 		payload_executor.Execute(input_chunk, payload_chunk);
 		payload_chunk.Verify();
-		payload_collection.Append(payload_chunk, true);
+		if (constant_aggregate) {
+			constant_aggregate->Sink(payload_chunk, filtering, filtered);
+		} else {
+			payload_collection.Append(payload_chunk, true);
+		}
 
 		// process payload chunks while they are still piping hot
 		if (check_nulls) {
@@ -1130,21 +1211,15 @@ void WindowExecutor::Sink(DataChunk &input_chunk, const idx_t input_idx, const i
 		}
 	}
 
-	if (wexpr->filter_expr) {
-		const auto filtered = filter_executor.SelectExpression(input_chunk, filter_sel);
-		for (idx_t f = 0; f < filtered; ++f) {
-			filter_mask.SetValid(input_idx + filter_sel[f]);
-		}
-	}
-
 	range.Append(input_chunk);
 }
 
 void WindowExecutor::Finalize(WindowAggregationMode mode) {
 	// build a segment tree for frame-adhering aggregates
 	// see http://www.vldb.org/pvldb/vol8/p1058-leis.pdf
-
-	if (wexpr->aggregate) {
+	if (constant_aggregate) {
+		constant_aggregate->Finalize();
+	} else if (wexpr->aggregate) {
 		segment_tree = make_unique<WindowSegmentTree>(*(wexpr->aggregate), wexpr->bind_info.get(), wexpr->return_type,
 		                                              &payload_collection, filter_mask, mode);
 	}
@@ -1184,7 +1259,11 @@ void WindowExecutor::Evaluate(idx_t row_idx, DataChunk &input_chunk, Vector &res
 
 		switch (wexpr->type) {
 		case ExpressionType::WINDOW_AGGREGATE: {
-			segment_tree->Compute(result, output_offset, bounds.window_start, bounds.window_end);
+			if (constant_aggregate) {
+				constant_aggregate->Compute(result, output_offset, bounds.window_start, bounds.window_end);
+			} else {
+				segment_tree->Compute(result, output_offset, bounds.window_start, bounds.window_end);
+			}
 			break;
 		}
 		case ExpressionType::WINDOW_ROW_NUMBER: {
@@ -1289,15 +1368,24 @@ void WindowExecutor::Evaluate(idx_t row_idx, DataChunk &input_chunk, Vector &res
 			break;
 		}
 		case ExpressionType::WINDOW_FIRST_VALUE: {
+			//	Same as NTH_VALUE(..., 1)
 			idx_t n = 1;
 			const auto first_idx = FindNextStart(ignore_nulls, bounds.window_start, bounds.window_end, n);
-			CopyCell(payload_collection, 0, first_idx, result, output_offset);
+			if (!n) {
+				CopyCell(payload_collection, 0, first_idx, result, output_offset);
+			} else {
+				FlatVector::SetNull(result, output_offset, true);
+			}
 			break;
 		}
 		case ExpressionType::WINDOW_LAST_VALUE: {
 			idx_t n = 1;
-			CopyCell(payload_collection, 0, FindPrevStart(ignore_nulls, bounds.window_start, bounds.window_end, n),
-			         result, output_offset);
+			const auto last_idx = FindPrevStart(ignore_nulls, bounds.window_start, bounds.window_end, n);
+			if (!n) {
+				CopyCell(payload_collection, 0, last_idx, result, output_offset);
+			} else {
+				FlatVector::SetNull(result, output_offset, true);
+			}
 			break;
 		}
 		case ExpressionType::WINDOW_NTH_VALUE: {
@@ -1550,6 +1638,9 @@ TaskExecutionResult WindowMergeTask::ExecuteTask(TaskExecutionMode mode) {
 	size_t sorted = 0;
 	while (sorted < hash_groups.states.size()) {
 		// First check if there is an unfinished task for this thread
+		if (executor.HasError()) {
+			return TaskExecutionResult::TASK_ERROR;
+		}
 		if (!local_state.TaskFinished()) {
 			local_state.ExecuteTask();
 			continue;
@@ -1799,15 +1890,6 @@ void WindowLocalSourceState::GeneratePartition(WindowGlobalSinkState &gstate, co
 		return;
 	}
 
-	// Create the executors for each function
-	window_execs.clear();
-	for (idx_t expr_idx = 0; expr_idx < op.select_list.size(); ++expr_idx) {
-		D_ASSERT(op.select_list[expr_idx]->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
-		auto wexpr = reinterpret_cast<BoundWindowExpression *>(op.select_list[expr_idx].get());
-		auto wexec = make_unique<WindowExecutor>(wexpr, context, count);
-		window_execs.emplace_back(std::move(wexec));
-	}
-
 	//	Initialise masks to false
 	const auto bit_count = ValidityMask::ValidityMaskSize(count);
 	partition_bits.clear();
@@ -1838,6 +1920,15 @@ void WindowLocalSourceState::GeneratePartition(WindowGlobalSinkState &gstate, co
 		return;
 	}
 
+	// Create the executors for each function
+	window_execs.clear();
+	for (idx_t expr_idx = 0; expr_idx < op.select_list.size(); ++expr_idx) {
+		D_ASSERT(op.select_list[expr_idx]->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
+		auto wexpr = reinterpret_cast<BoundWindowExpression *>(op.select_list[expr_idx].get());
+		auto wexec = make_unique<WindowExecutor>(wexpr, context, partition_mask, count);
+		window_execs.emplace_back(std::move(wexec));
+	}
+
 	//	First pass over the input without flushing
 	//	TODO: Factor out the constructor data as global state
 	scanner = make_unique<RowDataCollectionScanner>(*rows, *heap, layout, external, false);

package/src/duckdb/src/execution/operator/helper/physical_limit.cpp

@@ -111,6 +111,9 @@ SinkResultType PhysicalLimit::Sink(ExecutionContext &context, GlobalSinkState &g
 	}
 	state.data.Append(input, lstate.batch_index);
 	state.current_offset += input.size();
+	if (state.current_offset == max_element) {
+		return SinkResultType::FINISHED;
+	}
 	return SinkResultType::NEED_MORE_INPUT;
 }
 

package/src/duckdb/src/execution/operator/helper/physical_vacuum.cpp

@@ -49,6 +49,9 @@ SinkResultType PhysicalVacuum::Sink(ExecutionContext &context, GlobalSinkState &
 	D_ASSERT(lstate.column_distinct_stats.size() == info->column_id_map.size());
 
 	for (idx_t col_idx = 0; col_idx < input.data.size(); col_idx++) {
+		if (!DistinctStatistics::TypeIsSupported(input.data[col_idx].GetType())) {
+			continue;
+		}
 		lstate.column_distinct_stats[col_idx]->Update(input.data[col_idx], input.size(), false);
 	}
 
@@ -72,9 +75,8 @@ SinkFinalizeType PhysicalVacuum::Finalize(Pipeline &pipeline, Event &event, Clie
 
 	auto table = info->table;
 	for (idx_t col_idx = 0; col_idx < sink.column_distinct_stats.size(); col_idx++) {
-		table->GetStorage().SetStatistics(info->column_id_map.at(col_idx), [&](BaseStatistics &stats) {
-			stats.distinct_stats = std::move(sink.column_distinct_stats[col_idx]);
-		});
+		table->GetStorage().SetDistinct(info->column_id_map.at(col_idx),
+		                                std::move(sink.column_distinct_stats[col_idx]));
 	}
 
 	return SinkFinalizeType::READY;

package/src/duckdb/src/execution/operator/join/physical_blockwise_nl_join.cpp

@@ -87,11 +87,23 @@ public:
 	OuterJoinMarker left_outer;
 	SelectionVector match_sel;
 	ExpressionExecutor executor;
+	DataChunk intermediate_chunk;
 };
 
 unique_ptr<OperatorState> PhysicalBlockwiseNLJoin::GetOperatorState(ExecutionContext &context) const {
 	auto &gstate = (BlockwiseNLJoinGlobalState &)*sink_state;
-	return make_unique<BlockwiseNLJoinState>(context, gstate.right_chunks, *this);
+	auto result = make_unique<BlockwiseNLJoinState>(context, gstate.right_chunks, *this);
+	if (join_type == JoinType::SEMI || join_type == JoinType::ANTI) {
+		vector<LogicalType> intermediate_types;
+		for (auto &type : children[0]->types) {
+			intermediate_types.emplace_back(type);
+		}
+		for (auto &type : children[1]->types) {
+			intermediate_types.emplace_back(type);
+		}
+		result->intermediate_chunk.Initialize(Allocator::DefaultAllocator(), intermediate_types);
+	}
+	return std::move(result);
 }
 
 OperatorResultType PhysicalBlockwiseNLJoin::ExecuteInternal(ExecutionContext &context, DataChunk &input,
@@ -111,44 +123,79 @@ OperatorResultType PhysicalBlockwiseNLJoin::ExecuteInternal(ExecutionContext &co
 		}
 	}
 
+	DataChunk *intermediate_chunk = &chunk;
+	if (join_type == JoinType::SEMI || join_type == JoinType::ANTI) {
+		intermediate_chunk = &state.intermediate_chunk;
+		intermediate_chunk->Reset();
+	}
+
 	// now perform the actual join
-	// we perform a cross product, then execute the expression directly on the cross product' result
+	// we perform a cross product, then execute the expression directly on the cross product result
 	idx_t result_count = 0;
+	bool found_match[STANDARD_VECTOR_SIZE] = {false};
+
 	do {
-		auto result = state.cross_product.Execute(input, chunk);
+		auto result = state.cross_product.Execute(input, *intermediate_chunk);
 		if (result == OperatorResultType::NEED_MORE_INPUT) {
 			// exhausted input, have to pull new LHS chunk
 			if (state.left_outer.Enabled()) {
 				// left join: before we move to the next chunk, see if we need to output any vectors that didn't
 				// have a match found
-				state.left_outer.ConstructLeftJoinResult(input, chunk);
+				state.left_outer.ConstructLeftJoinResult(input, *intermediate_chunk);
 				state.left_outer.Reset();
 			}
+
+			if (join_type == JoinType::SEMI) {
+				PhysicalJoin::ConstructSemiJoinResult(input, chunk, found_match);
+			}
+			if (join_type == JoinType::ANTI) {
+				PhysicalJoin::ConstructAntiJoinResult(input, chunk, found_match);
+			}
+
 			return OperatorResultType::NEED_MORE_INPUT;
 		}
 
 		// now perform the computation
-		result_count = state.executor.SelectExpression(chunk, state.match_sel);
+		result_count = state.executor.SelectExpression(*intermediate_chunk, state.match_sel);
+
+		// handle anti and semi joins with different logic
 		if (result_count > 0) {
 			// found a match!
-			// check if the cross product is scanning the LHS or the RHS in its entirety
-			if (!state.cross_product.ScanLHS()) {
-				// set the match flags in the LHS
-				state.left_outer.SetMatches(state.match_sel, result_count);
-				// set the match flag in the RHS
-				gstate.right_outer.SetMatch(state.cross_product.ScanPosition() + state.cross_product.PositionInChunk());
+			// handle anti semi join conditions first
+			if (join_type == JoinType::ANTI || join_type == JoinType::SEMI) {
+				if (state.cross_product.ScanLHS()) {
+					found_match[state.cross_product.PositionInChunk()] = true;
+				} else {
+					for (idx_t i = 0; i < result_count; i++) {
+						found_match[state.match_sel.get_index(i)] = true;
+					}
+				}
+				intermediate_chunk->Reset();
+				// trick the loop to continue as semi and anti joins will never produce more output than
+				// the LHS cardinality
+				result_count = 0;
 			} else {
-				// set the match flag in the LHS
-				state.left_outer.SetMatch(state.cross_product.PositionInChunk());
-				// set the match flags in the RHS
-				gstate.right_outer.SetMatches(state.match_sel, result_count, state.cross_product.ScanPosition());
+				// check if the cross product is scanning the LHS or the RHS in its entirety
+				if (!state.cross_product.ScanLHS()) {
+					// set the match flags in the LHS
+					state.left_outer.SetMatches(state.match_sel, result_count);
+					// set the match flag in the RHS
+					gstate.right_outer.SetMatch(state.cross_product.ScanPosition() +
+					                            state.cross_product.PositionInChunk());
+				} else {
+					// set the match flag in the LHS
+					state.left_outer.SetMatch(state.cross_product.PositionInChunk());
+					// set the match flags in the RHS
+					gstate.right_outer.SetMatches(state.match_sel, result_count, state.cross_product.ScanPosition());
+				}
+				intermediate_chunk->Slice(state.match_sel, result_count);
 			}
-			chunk.Slice(state.match_sel, result_count);
 		} else {
 			// no result: reset the chunk
-			chunk.Reset();
+			intermediate_chunk->Reset();
 		}
 	} while (result_count == 0);
+
 	return OperatorResultType::HAVE_MORE_OUTPUT;
 }
 

package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp

@@ -1,5 +1,7 @@
 #include "duckdb/execution/operator/join/physical_hash_join.hpp"
 
+#include "duckdb/planner/expression/bound_aggregate_expression.hpp"
+#include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/common/types/column_data_collection.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/execution/expression_executor.hpp"

package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp

@@ -386,7 +386,7 @@ IEJoinUnion::IEJoinUnion(ClientContext &context, const PhysicalIEJoin &op, Sorte
 	// Sort on the first expression
 	auto ref = make_unique<BoundReferenceExpression>(order1.expression->return_type, 0);
 	vector<BoundOrderByNode> orders;
-	orders.emplace_back(BoundOrderByNode(order1.type, order1.null_order, std::move(ref)));
+	orders.emplace_back(order1.type, order1.null_order, std::move(ref));
 
 	l1 = make_unique<SortedTable>(context, orders, payload_layout);
 
@@ -422,7 +422,7 @@ IEJoinUnion::IEJoinUnion(ClientContext &context, const PhysicalIEJoin &op, Sorte
 	// Sort on the first expression
 	orders.clear();
 	ref = make_unique<BoundReferenceExpression>(order2.expression->return_type, 0);
-	orders.emplace_back(BoundOrderByNode(order2.type, order2.null_order, std::move(ref)));
+	orders.emplace_back(order2.type, order2.null_order, std::move(ref));
 
 	ExpressionExecutor executor(context);
 	executor.AddExpression(*orders[0].expression);