duckdb 0.3.2-dev1000.0 → 0.3.2-dev1028.0

package/src/duckdb.cpp

@@ -24867,8 +24867,9 @@ public:
 	void PrepareMergePhase();
 	//! Initializes the global sort state for another round of merging
 	void InitializeMergeRound();
-	//! Completes the cascaded merge sort round
-	void CompleteMergeRound();
+	//! Completes the cascaded merge sort round.
+	//! Pass true if you wish to use the radix data for further comparisons.
+	void CompleteMergeRound(bool keep_radix_data = false);
 
 public:
 	//! The lock for updating the order global state
@@ -26764,7 +26765,7 @@ void GlobalSortState::InitializeMergeRound() {
 	}
 }
 
-void GlobalSortState::CompleteMergeRound() {
+void GlobalSortState::CompleteMergeRound(bool keep_radix_data) {
 	sorted_blocks.clear();
 	for (auto &sorted_block_vector : sorted_blocks_temp) {
 		sorted_blocks.push_back(make_unique<SortedBlock>(buffer_manager, *this));
@@ -26776,7 +26777,7 @@ void GlobalSortState::CompleteMergeRound() {
 		odd_one_out = nullptr;
 	}
 	// Only one block left: Done!
-	if (sorted_blocks.size() == 1) {
+	if (sorted_blocks.size() == 1 && !keep_radix_data) {
 		sorted_blocks[0]->radix_sorting_data.clear();
 		sorted_blocks[0]->blob_sorting_data = nullptr;
 	}
@@ -27075,6 +27076,7 @@ void PayloadScanner::Scan(DataChunk &chunk) {
 	auto count = MinValue((idx_t)STANDARD_VECTOR_SIZE, total_count - total_scanned);
 	if (count == 0) {
 		D_ASSERT(read_state.block_idx == sorted_data.data_blocks.size());
+		chunk.SetCardinality(count);
 		return;
 	}
 	// Eagerly delete references to blocks that we've passed
@@ -47663,366 +47665,6 @@ idx_t JoinHashTable::FillWithHTOffsets(data_ptr_t *key_locations, JoinHTScanStat
 	}
 	return key_count;
 }
-} // namespace duckdb
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/execution/merge_join.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-
-
-
-
-
-
-
-namespace duckdb {
-
-struct MergeOrder {
-	SelectionVector order;
-	idx_t count;
-	VectorData vdata;
-};
-
-enum MergeInfoType : uint8_t { SCALAR_MERGE_INFO = 1, CHUNK_MERGE_INFO = 2 };
-
-struct MergeInfo {
-	MergeInfo(MergeInfoType info_type, LogicalType type) : info_type(info_type), type(type) {
-	}
-	MergeInfoType info_type;
-	LogicalType type;
-};
-
-struct ScalarMergeInfo : public MergeInfo {
-	MergeOrder &order;
-	idx_t &pos;
-	SelectionVector result;
-
-	ScalarMergeInfo(MergeOrder &order, LogicalType type, idx_t &pos)
-	    : MergeInfo(MergeInfoType::SCALAR_MERGE_INFO, type), order(order), pos(pos), result(STANDARD_VECTOR_SIZE) {
-	}
-};
-
-struct ChunkMergeInfo : public MergeInfo {
-	ChunkCollection &data_chunks;
-	vector<MergeOrder> &order_info;
-	bool found_match[STANDARD_VECTOR_SIZE];
-
-	ChunkMergeInfo(ChunkCollection &data_chunks, vector<MergeOrder> &order_info)
-	    : MergeInfo(MergeInfoType::CHUNK_MERGE_INFO, data_chunks.Types()[0]), data_chunks(data_chunks),
-	      order_info(order_info) {
-		memset(found_match, 0, sizeof(found_match));
-	}
-};
-
-struct MergeJoinComplex {
-	struct LessThan {
-		template <class T>
-		static idx_t Operation(ScalarMergeInfo &l, ScalarMergeInfo &r);
-	};
-	struct LessThanEquals {
-		template <class T>
-		static idx_t Operation(ScalarMergeInfo &l, ScalarMergeInfo &r);
-	};
-	struct GreaterThan {
-		template <class T>
-		static idx_t Operation(ScalarMergeInfo &l, ScalarMergeInfo &r) {
-			return LessThan::Operation<T>(r, l);
-		}
-	};
-	struct GreaterThanEquals {
-		template <class T>
-		static idx_t Operation(ScalarMergeInfo &l, ScalarMergeInfo &r) {
-			return LessThanEquals::Operation<T>(r, l);
-		}
-	};
-
-	static idx_t Perform(MergeInfo &l, MergeInfo &r, ExpressionType comparison_type);
-};
-
-struct MergeJoinSimple {
-	struct LessThan {
-		template <class T>
-		static idx_t Operation(ScalarMergeInfo &l, ChunkMergeInfo &r);
-	};
-	struct LessThanEquals {
-		template <class T>
-		static idx_t Operation(ScalarMergeInfo &l, ChunkMergeInfo &r);
-	};
-	struct GreaterThan {
-		template <class T>
-		static idx_t Operation(ScalarMergeInfo &l, ChunkMergeInfo &r);
-	};
-	struct GreaterThanEquals {
-		template <class T>
-		static idx_t Operation(ScalarMergeInfo &l, ChunkMergeInfo &r);
-	};
-
-	static idx_t Perform(MergeInfo &l, MergeInfo &r, ExpressionType comparison);
-};
-
-#define INSTANTIATE_MERGEJOIN_TEMPLATES(MJCLASS, OPNAME, L, R)                                                         \
-	template idx_t MJCLASS::OPNAME::Operation<int8_t>(L & l, R & r);                                                   \
-	template idx_t MJCLASS::OPNAME::Operation<int16_t>(L & l, R & r);                                                  \
-	template idx_t MJCLASS::OPNAME::Operation<int32_t>(L & l, R & r);                                                  \
-	template idx_t MJCLASS::OPNAME::Operation<int64_t>(L & l, R & r);                                                  \
-	template idx_t MJCLASS::OPNAME::Operation<uint8_t>(L & l, R & r);                                                  \
-	template idx_t MJCLASS::OPNAME::Operation<uint16_t>(L & l, R & r);                                                 \
-	template idx_t MJCLASS::OPNAME::Operation<uint32_t>(L & l, R & r);                                                 \
-	template idx_t MJCLASS::OPNAME::Operation<uint64_t>(L & l, R & r);                                                 \
-	template idx_t MJCLASS::OPNAME::Operation<hugeint_t>(L & l, R & r);                                                \
-	template idx_t MJCLASS::OPNAME::Operation<float>(L & l, R & r);                                                    \
-	template idx_t MJCLASS::OPNAME::Operation<double>(L & l, R & r);                                                   \
-	template idx_t MJCLASS::OPNAME::Operation<interval_t>(L & l, R & r);                                               \
-	template idx_t MJCLASS::OPNAME::Operation<string_t>(L & l, R & r);
-
-} // namespace duckdb
-
-
-
-
-namespace duckdb {
-
-template <class MJ, class L_ARG, class R_ARG>
-static idx_t MergeJoinSwitch(L_ARG &l, R_ARG &r) {
-	switch (l.type.InternalType()) {
-	case PhysicalType::BOOL:
-	case PhysicalType::INT8:
-		return MJ::template Operation<int8_t>(l, r);
-	case PhysicalType::INT16:
-		return MJ::template Operation<int16_t>(l, r);
-	case PhysicalType::INT32:
-		return MJ::template Operation<int32_t>(l, r);
-	case PhysicalType::INT64:
-		return MJ::template Operation<int64_t>(l, r);
-	case PhysicalType::UINT8:
-		return MJ::template Operation<uint8_t>(l, r);
-	case PhysicalType::UINT16:
-		return MJ::template Operation<uint16_t>(l, r);
-	case PhysicalType::UINT32:
-		return MJ::template Operation<uint32_t>(l, r);
-	case PhysicalType::UINT64:
-		return MJ::template Operation<uint64_t>(l, r);
-	case PhysicalType::INT128:
-		return MJ::template Operation<hugeint_t>(l, r);
-	case PhysicalType::FLOAT:
-		return MJ::template Operation<float>(l, r);
-	case PhysicalType::DOUBLE:
-		return MJ::template Operation<double>(l, r);
-	case PhysicalType::INTERVAL:
-		return MJ::template Operation<interval_t>(l, r);
-	case PhysicalType::VARCHAR:
-		return MJ::template Operation<string_t>(l, r);
-	default:
-		throw InternalException("Type not implemented for merge join!");
-	}
-}
-
-template <class T, class L_ARG, class R_ARG>
-static idx_t MergeJoinComparisonSwitch(L_ARG &l, R_ARG &r, ExpressionType comparison_type) {
-	switch (comparison_type) {
-	case ExpressionType::COMPARE_LESSTHAN:
-		return MergeJoinSwitch<typename T::LessThan, L_ARG, R_ARG>(l, r);
-	case ExpressionType::COMPARE_LESSTHANOREQUALTO:
-		return MergeJoinSwitch<typename T::LessThanEquals, L_ARG, R_ARG>(l, r);
-	case ExpressionType::COMPARE_GREATERTHAN:
-		return MergeJoinSwitch<typename T::GreaterThan, L_ARG, R_ARG>(l, r);
-	case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
-		return MergeJoinSwitch<typename T::GreaterThanEquals, L_ARG, R_ARG>(l, r);
-	default:
-		throw InternalException("Unimplemented comparison type for merge join!");
-	}
-}
-
-idx_t MergeJoinComplex::Perform(MergeInfo &l, MergeInfo &r, ExpressionType comparison_type) {
-	D_ASSERT(l.info_type == MergeInfoType::SCALAR_MERGE_INFO && r.info_type == MergeInfoType::SCALAR_MERGE_INFO);
-	auto &left = (ScalarMergeInfo &)l;
-	auto &right = (ScalarMergeInfo &)r;
-	D_ASSERT(left.type == right.type);
-	if (left.order.count == 0 || right.order.count == 0) {
-		return 0;
-	}
-	return MergeJoinComparisonSwitch<MergeJoinComplex, ScalarMergeInfo, ScalarMergeInfo>(left, right, comparison_type);
-}
-
-idx_t MergeJoinSimple::Perform(MergeInfo &l, MergeInfo &r, ExpressionType comparison_type) {
-	D_ASSERT(l.info_type == MergeInfoType::SCALAR_MERGE_INFO && r.info_type == MergeInfoType::CHUNK_MERGE_INFO);
-	auto &left = (ScalarMergeInfo &)l;
-	auto &right = (ChunkMergeInfo &)r;
-	D_ASSERT(left.type == right.type);
-	if (left.order.count == 0 || right.data_chunks.Count() == 0) {
-		return 0;
-	}
-	return MergeJoinComparisonSwitch<MergeJoinSimple, ScalarMergeInfo, ChunkMergeInfo>(left, right, comparison_type);
-}
-
-} // namespace duckdb
-
-
-
-
-
-namespace duckdb {
-
-template <class T, class OP>
-idx_t MergeJoinComplexLessThan(ScalarMergeInfo &l, ScalarMergeInfo &r) {
-	if (r.pos >= r.order.count) {
-		return 0;
-	}
-	auto ldata = (T *)l.order.vdata.data;
-	auto rdata = (T *)r.order.vdata.data;
-	auto &lorder = l.order.order;
-	auto &rorder = r.order.order;
-	idx_t result_count = 0;
-	while (true) {
-		if (l.pos < l.order.count) {
-			auto lidx = lorder.get_index(l.pos);
-			auto ridx = rorder.get_index(r.pos);
-			auto dlidx = l.order.vdata.sel->get_index(lidx);
-			auto dridx = r.order.vdata.sel->get_index(ridx);
-			if (OP::Operation(ldata[dlidx], rdata[dridx])) {
-				// left side smaller: found match
-				l.result.set_index(result_count, lidx);
-				r.result.set_index(result_count, ridx);
-				result_count++;
-				// move left side forward
-				l.pos++;
-				if (result_count == STANDARD_VECTOR_SIZE) {
-					// out of space!
-					break;
-				}
-				continue;
-			}
-		}
-		// right side smaller or equal, or left side exhausted: move
-		// right pointer forward reset left side to start
-		l.pos = 0;
-		r.pos++;
-		if (r.pos == r.order.count) {
-			break;
-		}
-	}
-	return result_count;
-}
-
-template <class T>
-idx_t MergeJoinComplex::LessThan::Operation(ScalarMergeInfo &l, ScalarMergeInfo &r) {
-	return MergeJoinComplexLessThan<T, duckdb::LessThan>(l, r);
-}
-
-template <class T>
-idx_t MergeJoinComplex::LessThanEquals::Operation(ScalarMergeInfo &l, ScalarMergeInfo &r) {
-	return MergeJoinComplexLessThan<T, duckdb::LessThanEquals>(l, r);
-}
-
-INSTANTIATE_MERGEJOIN_TEMPLATES(MergeJoinComplex, LessThan, ScalarMergeInfo, ScalarMergeInfo)
-INSTANTIATE_MERGEJOIN_TEMPLATES(MergeJoinComplex, LessThanEquals, ScalarMergeInfo, ScalarMergeInfo)
-
-} // namespace duckdb
-
-
-
-
-
-namespace duckdb {
-
-template <class T, class OP>
-static idx_t MergeJoinSimpleGreaterThan(ScalarMergeInfo &l, ChunkMergeInfo &r) {
-	auto ldata = (T *)l.order.vdata.data;
-	auto &lorder = l.order.order;
-	l.pos = l.order.count;
-	for (idx_t chunk_idx = 0; chunk_idx < r.order_info.size(); chunk_idx++) {
-		// we only care about the SMALLEST value in each of the RHS
-		// because we want to figure out if they are greater than [or equal] to ANY value
-		// get the smallest value from the RHS
-		auto &rorder = r.order_info[chunk_idx];
-		auto rdata = (T *)rorder.vdata.data;
-		auto min_r_value = rdata[rorder.vdata.sel->get_index(rorder.order.get_index(0))];
-		// now we start from the current lpos value and check if we found a new value that is [>= OR >] the min RHS
-		// value
-		while (true) {
-			auto lidx = lorder.get_index(l.pos - 1);
-			auto dlidx = l.order.vdata.sel->get_index(lidx);
-			if (OP::Operation(ldata[dlidx], min_r_value)) {
-				// found a match for lpos, set it in the found_match vector
-				r.found_match[lidx] = true;
-				l.pos--;
-				if (l.pos == 0) {
-					// early out: we exhausted the entire LHS and they all match
-					return 0;
-				}
-			} else {
-				// we found no match: any subsequent value from the LHS we scan now will be smaller and thus also not
-				// match move to the next RHS chunk
-				break;
-			}
-		}
-	}
-	return 0;
-}
-template <class T>
-idx_t MergeJoinSimple::GreaterThan::Operation(ScalarMergeInfo &l, ChunkMergeInfo &r) {
-	return MergeJoinSimpleGreaterThan<T, duckdb::GreaterThan>(l, r);
-}
-
-template <class T>
-idx_t MergeJoinSimple::GreaterThanEquals::Operation(ScalarMergeInfo &l, ChunkMergeInfo &r) {
-	return MergeJoinSimpleGreaterThan<T, duckdb::GreaterThanEquals>(l, r);
-}
-
-template <class T, class OP>
-static idx_t MergeJoinSimpleLessThan(ScalarMergeInfo &l, ChunkMergeInfo &r) {
-	auto ldata = (T *)l.order.vdata.data;
-	auto &lorder = l.order.order;
-	l.pos = 0;
-	for (idx_t chunk_idx = 0; chunk_idx < r.order_info.size(); chunk_idx++) {
-		// we only care about the BIGGEST value in each of the RHS
-		// because we want to figure out if they are less than [or equal] to ANY value
-		// get the biggest value from the RHS
-		auto &rorder = r.order_info[chunk_idx];
-		auto rdata = (T *)rorder.vdata.data;
-		auto max_r_value = rdata[rorder.vdata.sel->get_index(rorder.order.get_index(rorder.count - 1))];
-		// now we start from the current lpos value and check if we found a new value that is [<= OR <] the max RHS
-		// value
-		while (true) {
-			auto lidx = lorder.get_index(l.pos);
-			auto dlidx = l.order.vdata.sel->get_index(lidx);
-			if (OP::Operation(ldata[dlidx], max_r_value)) {
-				// found a match for lpos, set it in the found_match vector
-				r.found_match[lidx] = true;
-				l.pos++;
-				if (l.pos == l.order.count) {
-					// early out: we exhausted the entire LHS and they all match
-					return 0;
-				}
-			} else {
-				// we found no match: any subsequent value from the LHS we scan now will be bigger and thus also not
-				// match move to the next RHS chunk
-				break;
-			}
-		}
-	}
-	return 0;
-}
-
-template <class T>
-idx_t MergeJoinSimple::LessThan::Operation(ScalarMergeInfo &l, ChunkMergeInfo &r) {
-	return MergeJoinSimpleLessThan<T, duckdb::LessThan>(l, r);
-}
-
-template <class T>
-idx_t MergeJoinSimple::LessThanEquals::Operation(ScalarMergeInfo &l, ChunkMergeInfo &r) {
-	return MergeJoinSimpleLessThan<T, duckdb::LessThanEquals>(l, r);
-}
-
-INSTANTIATE_MERGEJOIN_TEMPLATES(MergeJoinSimple, LessThan, ScalarMergeInfo, ChunkMergeInfo)
-INSTANTIATE_MERGEJOIN_TEMPLATES(MergeJoinSimple, LessThanEquals, ScalarMergeInfo, ChunkMergeInfo)
-INSTANTIATE_MERGEJOIN_TEMPLATES(MergeJoinSimple, GreaterThan, ScalarMergeInfo, ChunkMergeInfo)
-INSTANTIATE_MERGEJOIN_TEMPLATES(MergeJoinSimple, GreaterThanEquals, ScalarMergeInfo, ChunkMergeInfo)
-
 } // namespace duckdb
 
 //===----------------------------------------------------------------------===//
@@ -55010,6 +54652,8 @@ void PhysicalNestedLoopJoin::GetData(ExecutionContext &context, DataChunk &chunk
 
 namespace duckdb {
 
+class MergeJoinGlobalState;
+
 //! PhysicalPiecewiseMergeJoin represents a piecewise merge loop join between
 //! two tables
 class PhysicalPiecewiseMergeJoin : public PhysicalComparisonJoin {
@@ -55019,6 +54663,8 @@ public:
 	                           idx_t estimated_cardinality);
 
 	vector<LogicalType> join_key_types;
+	vector<BoundOrderByNode> lhs_orders;
+	vector<BoundOrderByNode> rhs_orders;
 
 public:
 	// Operator Interface
@@ -55057,6 +54703,9 @@ public:
 	SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
 	                          GlobalSinkState &gstate) const override;
 
+	//! Schedules tasks to merge sort the RHS data during the Finalize phase
+	static void ScheduleMergeTasks(Pipeline &pipeline, Event &event, MergeJoinGlobalState &state);
+
 	bool IsSink() const override {
 		return true;
 	}
@@ -55081,6 +54730,11 @@ private:
 
 
 
+
+
+
+
+
 namespace duckdb {
 
 PhysicalPiecewiseMergeJoin::PhysicalPiecewiseMergeJoin(LogicalOperator &op, unique_ptr<PhysicalOperator> left,
@@ -55091,10 +54745,27 @@ PhysicalPiecewiseMergeJoin::PhysicalPiecewiseMergeJoin(LogicalOperator &op, uniq
 	// for now we only support one condition!
 	D_ASSERT(conditions.size() == 1);
 	for (auto &cond : conditions) {
-		// COMPARE NOT EQUAL not supported with merge join
-		D_ASSERT(cond.comparison != ExpressionType::COMPARE_NOTEQUAL);
 		D_ASSERT(cond.left->return_type == cond.right->return_type);
 		join_key_types.push_back(cond.left->return_type);
+
+		// Convert the conditions to sort orders
+		auto left = cond.left->Copy();
+		auto right = cond.right->Copy();
+		switch (cond.comparison) {
+		case ExpressionType::COMPARE_LESSTHAN:
+		case ExpressionType::COMPARE_LESSTHANOREQUALTO:
+			lhs_orders.emplace_back(BoundOrderByNode(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, move(left)));
+			rhs_orders.emplace_back(BoundOrderByNode(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, move(right)));
+			break;
+		case ExpressionType::COMPARE_GREATERTHAN:
+		case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
+			lhs_orders.emplace_back(BoundOrderByNode(OrderType::DESCENDING, OrderByNullType::NULLS_LAST, move(left)));
+			rhs_orders.emplace_back(BoundOrderByNode(OrderType::DESCENDING, OrderByNullType::NULLS_LAST, move(right)));
+			break;
+		default:
+			// COMPARE NOT EQUAL not supported with merge join
+			throw NotImplementedException("Unimplemented join type for merge join");
+		}
 	}
 	children.push_back(move(left));
 	children.push_back(move(right));
@@ -55103,114 +54774,242 @@ PhysicalPiecewiseMergeJoin::PhysicalPiecewiseMergeJoin(LogicalOperator &op, uniq
 //===--------------------------------------------------------------------===//
 // Sink
 //===--------------------------------------------------------------------===//
-class MergeJoinLocalState : public LocalSinkState {
+class MergeJoinGlobalState : public GlobalSinkState {
 public:
-	explicit MergeJoinLocalState(const vector<JoinCondition> &conditions) {
-		vector<LogicalType> condition_types;
-		for (auto &cond : conditions) {
-			rhs_executor.AddExpression(*cond.right);
-			condition_types.push_back(cond.right->return_type);
-		}
-		join_keys.Initialize(condition_types);
+	MergeJoinGlobalState(BufferManager &buffer_manager, const vector<BoundOrderByNode> &orders, RowLayout &rhs_layout)
+	    : rhs_global_sort_state(buffer_manager, orders, rhs_layout), rhs_has_null(0), rhs_count(0),
+	      memory_per_thread(0) {
 	}
 
-	//! The chunk holding the right condition
-	DataChunk join_keys;
-	//! The executor of the RHS condition
-	ExpressionExecutor rhs_executor;
-};
-
-class MergeJoinGlobalState : public GlobalSinkState {
-public:
-	MergeJoinGlobalState() : has_null(false), right_outer_position(0) {
+	inline idx_t Count() const {
+		return rhs_count;
 	}
 
-	mutex mj_lock;
-	//! The materialized data of the RHS
-	ChunkCollection right_chunks;
-	//! The materialized join keys of the RHS
-	ChunkCollection right_conditions;
-	//! The join orders of the RHS
-	vector<MergeOrder> right_orders;
-	//! Whether or not the RHS of the nested loop join has NULL values
-	bool has_null;
+	//! The lock for updating the global state
+	mutex lock;
+	//! Global sort state
+	GlobalSortState rhs_global_sort_state;
+	//! Whether or not the RHS has NULL values
+	idx_t rhs_has_null;
+	//! The total number of rows in the RHS
+	idx_t rhs_count;
 	//! A bool indicating for each tuple in the RHS if they found a match (only used in FULL OUTER JOIN)
-	unique_ptr<bool[]> right_found_match;
-	//! The position in the RHS in the final scan of the FULL OUTER JOIN
-	idx_t right_outer_position;
+	unique_ptr<bool[]> rhs_found_match;
+	//! Memory usage per thread
+	idx_t memory_per_thread;
 };
 
 unique_ptr<GlobalSinkState> PhysicalPiecewiseMergeJoin::GetGlobalSinkState(ClientContext &context) const {
-	return make_unique<MergeJoinGlobalState>();
-}
-
-unique_ptr<LocalSinkState> PhysicalPiecewiseMergeJoin::GetLocalSinkState(ExecutionContext &context) const {
-	return make_unique<MergeJoinLocalState>(conditions);
+	// Get the payload layout from the rhs types
+	RowLayout rhs_layout;
+	auto types = children[1]->types;
+	rhs_layout.Initialize(types);
+	auto state = make_unique<MergeJoinGlobalState>(BufferManager::GetBufferManager(context), rhs_orders, rhs_layout);
+	// Set external (can be force with the PRAGMA)
+	auto &config = ClientConfig::GetConfig(context);
+	state->rhs_global_sort_state.external = config.force_external;
+	// Memory usage per thread should scale with max mem / num threads
+	// We take 1/4th of this, to be conservative
+	idx_t max_memory = BufferManager::GetBufferManager(context).GetMaxMemory();
+	idx_t num_threads = TaskScheduler::GetScheduler(context).NumberOfThreads();
+	state->memory_per_thread = (max_memory / num_threads) / 4;
+	return move(state);
 }
 
-SinkResultType PhysicalPiecewiseMergeJoin::Sink(ExecutionContext &context, GlobalSinkState &state,
-                                                LocalSinkState &lstate, DataChunk &input) const {
-	auto &gstate = (MergeJoinGlobalState &)state;
-	auto &mj_state = (MergeJoinLocalState &)lstate;
-
-	// resolve the join keys for this chunk
-	mj_state.rhs_executor.SetChunk(input);
+static idx_t CountValid(Vector &v, const idx_t count) {
+	idx_t valid = 0;
 
-	mj_state.join_keys.Reset();
-	mj_state.join_keys.SetCardinality(input);
-	for (idx_t k = 0; k < conditions.size(); k++) {
-		// resolve the join key
-		mj_state.rhs_executor.ExecuteExpression(k, mj_state.join_keys.data[k]);
+	VectorData vdata;
+	v.Orrify(count, vdata);
+	if (vdata.validity.AllValid()) {
+		return count;
 	}
-	// append the join keys and the chunk to the chunk collection
-	lock_guard<mutex> mj_guard(gstate.mj_lock);
-	gstate.right_chunks.Append(input);
-	gstate.right_conditions.Append(mj_state.join_keys);
-	return SinkResultType::NEED_MORE_INPUT;
-}
+	switch (v.GetVectorType()) {
+	case VectorType::FLAT_VECTOR:
+		valid += vdata.validity.CountValid(count);
+		break;
+	case VectorType::CONSTANT_VECTOR:
+		valid += vdata.validity.CountValid(1) * count;
+		break;
+	default:
+		for (idx_t i = 0; i < count; ++i) {
+			const auto row_idx = vdata.sel->get_index(i);
+			valid += int(vdata.validity.RowIsValid(row_idx));
+		}
+		break;
+	}
+
+	return valid;
+}
 
-void PhysicalPiecewiseMergeJoin::Combine(ExecutionContext &context, GlobalSinkState &gstate,
-                                         LocalSinkState &lstate) const {
-	auto &state = (MergeJoinLocalState &)lstate;
+class MergeJoinLocalState : public LocalSinkState {
+public:
+	explicit MergeJoinLocalState() : rhs_has_null(0), rhs_count(0) {
+	}
+
+	//! The local sort state
+	LocalSortState rhs_local_sort_state;
+	//! Local copy of the sorting expression executor
+	ExpressionExecutor rhs_executor;
+	//! Holds a vector of incoming sorting columns
+	DataChunk rhs_keys;
+	//! Whether or not the RHS has NULL values
+	idx_t rhs_has_null;
+	//! The total number of rows in the RHS
+	idx_t rhs_count;
+};
+
+unique_ptr<LocalSinkState> PhysicalPiecewiseMergeJoin::GetLocalSinkState(ExecutionContext &context) const {
+	auto result = make_unique<MergeJoinLocalState>();
+	// Initialize order clause expression executor and DataChunk
+	vector<LogicalType> types;
+	for (auto &order : rhs_orders) {
+		types.push_back(order.expression->return_type);
+		result->rhs_executor.AddExpression(*order.expression);
+	}
+	result->rhs_keys.Initialize(types);
+	return move(result);
+}
+
+SinkResultType PhysicalPiecewiseMergeJoin::Sink(ExecutionContext &context, GlobalSinkState &gstate_p,
+                                                LocalSinkState &lstate_p, DataChunk &input) const {
+	auto &gstate = (MergeJoinGlobalState &)gstate_p;
+	auto &lstate = (MergeJoinLocalState &)lstate_p;
+
+	auto &global_sort_state = gstate.rhs_global_sort_state;
+	auto &local_sort_state = lstate.rhs_local_sort_state;
+
+	// Initialize local state (if necessary)
+	if (!local_sort_state.initialized) {
+		local_sort_state.Initialize(global_sort_state, BufferManager::GetBufferManager(context.client));
+	}
+
+	// Obtain sorting columns
+	auto &join_keys = lstate.rhs_keys;
+	join_keys.Reset();
+	lstate.rhs_executor.Execute(input, join_keys);
+
+	// Count the NULLs so we can exclude them later
+	// TODO: Sort any comparison NULLs to the end using an initial sort column
+	const auto count = join_keys.size();
+	for (auto &key : join_keys.data) {
+		lstate.rhs_has_null += (count - CountValid(key, count));
+	}
+	lstate.rhs_count += count;
+
+	// Sink the data into the local sort state
+	local_sort_state.SinkChunk(join_keys, input);
+
+	// When sorting data reaches a certain size, we sort it
+	if (local_sort_state.SizeInBytes() >= gstate.memory_per_thread) {
+		local_sort_state.Sort(global_sort_state, true);
+	}
+	return SinkResultType::NEED_MORE_INPUT;
+}
+
+void PhysicalPiecewiseMergeJoin::Combine(ExecutionContext &context, GlobalSinkState &gstate_p,
+                                         LocalSinkState &lstate_p) const {
+	auto &gstate = (MergeJoinGlobalState &)gstate_p;
+	auto &lstate = (MergeJoinLocalState &)lstate_p;
+	gstate.rhs_global_sort_state.AddLocalState(lstate.rhs_local_sort_state);
+	lock_guard<mutex> locked(gstate.lock);
+	gstate.rhs_has_null += lstate.rhs_has_null;
+	gstate.rhs_count += lstate.rhs_count;
 	auto &client_profiler = QueryProfiler::Get(context.client);
 
-	context.thread.profiler.Flush(this, &state.rhs_executor, "rhs_executor", 1);
+	context.thread.profiler.Flush(this, &lstate.rhs_executor, "rhs_executor", 1);
 	client_profiler.Flush(context.thread.profiler);
 }
 
 //===--------------------------------------------------------------------===//
 // Finalize
 //===--------------------------------------------------------------------===//
-static void OrderVector(Vector &vector, idx_t count, MergeOrder &order);
+class MergeJoinFinalizeTask : public ExecutorTask {
+public:
+	MergeJoinFinalizeTask(shared_ptr<Event> event_p, ClientContext &context, MergeJoinGlobalState &state)
+	    : ExecutorTask(context), event(move(event_p)), context(context), state(state) {
+	}
+
+	TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override {
+		// Initialize merge sorted and iterate until done
+		auto &global_sort_state = state.rhs_global_sort_state;
+		MergeSorter merge_sorter(global_sort_state, BufferManager::GetBufferManager(context));
+		merge_sorter.PerformInMergeRound();
+		event->FinishTask();
+
+		return TaskExecutionResult::TASK_FINISHED;
+	}
+
+private:
+	shared_ptr<Event> event;
+	ClientContext &context;
+	MergeJoinGlobalState &state;
+};
+
+class MergeJoinFinalizeEvent : public Event {
+public:
+	MergeJoinFinalizeEvent(MergeJoinGlobalState &gstate_p, Pipeline &pipeline_p)
+	    : Event(pipeline_p.executor), gstate(gstate_p), pipeline(pipeline_p) {
+	}
+
+	MergeJoinGlobalState &gstate;
+	Pipeline &pipeline;
+
+public:
+	void Schedule() override {
+		auto &context = pipeline.GetClientContext();
+
+		// Schedule tasks equal to the number of threads, which will each merge multiple partitions
+		auto &ts = TaskScheduler::GetScheduler(context);
+		idx_t num_threads = ts.NumberOfThreads();
+
+		vector<unique_ptr<Task>> merge_tasks;
+		for (idx_t tnum = 0; tnum < num_threads; tnum++) {
+			merge_tasks.push_back(make_unique<MergeJoinFinalizeTask>(shared_from_this(), context, gstate));
+		}
+		SetTasks(move(merge_tasks));
+	}
+
+	void FinishEvent() override {
+		auto &global_sort_state = gstate.rhs_global_sort_state;
+
+		global_sort_state.CompleteMergeRound(true);
+		if (global_sort_state.sorted_blocks.size() > 1) {
+			// Multiple blocks remaining: Schedule the next round
+			PhysicalPiecewiseMergeJoin::ScheduleMergeTasks(pipeline, *this, gstate);
+		}
+	}
+};
+
+void PhysicalPiecewiseMergeJoin::ScheduleMergeTasks(Pipeline &pipeline, Event &event, MergeJoinGlobalState &gstate) {
+	// Initialize global sort state for a round of merging
+	gstate.rhs_global_sort_state.InitializeMergeRound();
+	auto new_event = make_shared<MergeJoinFinalizeEvent>(gstate, pipeline);
+	event.InsertEvent(move(new_event));
+}
 
 SinkFinalizeType PhysicalPiecewiseMergeJoin::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
                                                       GlobalSinkState &gstate_p) const {
 	auto &gstate = (MergeJoinGlobalState &)gstate_p;
-	if (gstate.right_conditions.ChunkCount() > 0) {
-		// now order all the chunks
-		gstate.right_orders.resize(gstate.right_conditions.ChunkCount());
-		for (idx_t i = 0; i < gstate.right_conditions.ChunkCount(); i++) {
-			auto &chunk_to_order = gstate.right_conditions.GetChunk(i);
-			D_ASSERT(chunk_to_order.ColumnCount() == 1);
-			for (idx_t col_idx = 0; col_idx < chunk_to_order.ColumnCount(); col_idx++) {
-				OrderVector(chunk_to_order.data[col_idx], chunk_to_order.size(), gstate.right_orders[i]);
-				if (gstate.right_orders[i].count < chunk_to_order.size()) {
-					// the amount of entries in the order vector is smaller than the amount of entries in the vector
-					// this only happens if there are NULL values in the right-hand side
-					// hence we set the has_null to true (this is required for the MARK join)
-					gstate.has_null = true;
-				}
-			}
-		}
-	}
+	auto &global_sort_state = gstate.rhs_global_sort_state;
+
 	if (IsRightOuterJoin(join_type)) {
 		// for FULL/RIGHT OUTER JOIN, initialize found_match to false for every tuple
-		gstate.right_found_match = unique_ptr<bool[]>(new bool[gstate.right_chunks.Count()]);
-		memset(gstate.right_found_match.get(), 0, sizeof(bool) * gstate.right_chunks.Count());
+		gstate.rhs_found_match = unique_ptr<bool[]>(new bool[gstate.Count()]);
+		memset(gstate.rhs_found_match.get(), 0, sizeof(bool) * gstate.Count());
 	}
-	if (gstate.right_chunks.Count() == 0 && EmptyResultIfRHSIsEmpty()) {
+	if (global_sort_state.sorted_blocks.empty() && EmptyResultIfRHSIsEmpty()) {
+		// Empty input!
 		return SinkFinalizeType::NO_OUTPUT_POSSIBLE;
 	}
+
+	// Prepare for merge sort phase
+	global_sort_state.PrepareMergePhase();
+
+	// Start the merge phase or finish if a merge is not necessary
+	if (global_sort_state.sorted_blocks.size() > 1) {
+		PhysicalPiecewiseMergeJoin::ScheduleMergeTasks(pipeline, event, gstate);
+	}
 	return SinkFinalizeType::READY;
 }
 
@@ -55219,43 +55018,75 @@ SinkFinalizeType PhysicalPiecewiseMergeJoin::Finalize(Pipeline &pipeline, Event
 //===--------------------------------------------------------------------===//
 class PiecewiseMergeJoinState : public OperatorState {
 public:
-	explicit PiecewiseMergeJoinState(const PhysicalPiecewiseMergeJoin &op)
-	    : op(op), first_fetch(true), finished(true), left_position(0), right_position(0), right_chunk_index(0) {
+	explicit PiecewiseMergeJoinState(const PhysicalPiecewiseMergeJoin &op, BufferManager &buffer_manager,
+	                                 bool force_external)
+	    : op(op), buffer_manager(buffer_manager), force_external(force_external), left_position(0), first_fetch(true),
+	      finished(true), right_position(0), right_chunk_index(0) {
 		vector<LogicalType> condition_types;
 		for (auto &cond : op.conditions) {
 			lhs_executor.AddExpression(*cond.left);
 			condition_types.push_back(cond.left->return_type);
 		}
-		join_keys.Initialize(condition_types);
+		lhs_keys.Initialize(condition_types);
 		if (IsLeftOuterJoin(op.join_type)) {
-			left_found_match = unique_ptr<bool[]>(new bool[STANDARD_VECTOR_SIZE]);
-			memset(left_found_match.get(), 0, sizeof(bool) * STANDARD_VECTOR_SIZE);
+			lhs_found_match = unique_ptr<bool[]>(new bool[STANDARD_VECTOR_SIZE]);
+			memset(lhs_found_match.get(), 0, sizeof(bool) * STANDARD_VECTOR_SIZE);
 		}
+		lhs_layout.Initialize(op.children[0]->types);
 	}
 
 	const PhysicalPiecewiseMergeJoin &op;
+	BufferManager &buffer_manager;
+	bool force_external;
+
+	// Block sorting
+	DataChunk lhs_keys;
+	ExpressionExecutor lhs_executor;
+	unique_ptr<bool[]> lhs_found_match;
+	RowLayout lhs_layout;
+	unique_ptr<LocalSortState> lhs_local_state;
+	unique_ptr<GlobalSortState> lhs_global_state;
+	idx_t lhs_count;
+	idx_t lhs_has_null;
+
+	// Simple scans
+	idx_t left_position;
+
+	// Complex scans
 	bool first_fetch;
 	bool finished;
-	idx_t left_position;
 	idx_t right_position;
 	idx_t right_chunk_index;
-	DataChunk left_chunk;
-	DataChunk join_keys;
-	MergeOrder left_orders;
-	//! The executor of the RHS condition
-	ExpressionExecutor lhs_executor;
-	unique_ptr<bool[]> left_found_match;
+	idx_t right_base;
 
 public:
 	void ResolveJoinKeys(DataChunk &input) {
 		// resolve the join keys for the input
-		join_keys.Reset();
-		lhs_executor.SetChunk(input);
-		join_keys.SetCardinality(input);
-		for (idx_t k = 0; k < op.conditions.size(); k++) {
-			lhs_executor.ExecuteExpression(k, join_keys.data[k]);
-			// sort by join key
-			OrderVector(join_keys.data[k], join_keys.size(), left_orders);
+		lhs_keys.Reset();
+		lhs_executor.Execute(input, lhs_keys);
+
+		// Count the NULLs so we can exclude them later
+		// TODO: Sort any multi-comparison NULLs to the end using an initial sort column
+		lhs_count = lhs_keys.size();
+		for (auto &key : lhs_keys.data) {
+			lhs_has_null = lhs_count - CountValid(key, lhs_count);
+			break;
+		}
+
+		// sort by join key
+		lhs_global_state = make_unique<GlobalSortState>(buffer_manager, op.lhs_orders, lhs_layout);
+		lhs_local_state = make_unique<LocalSortState>();
+		lhs_local_state->Initialize(*lhs_global_state, buffer_manager);
+		lhs_local_state->SinkChunk(lhs_keys, input);
+
+		// Set external (can be force with the PRAGMA)
+		lhs_global_state->external = force_external;
+		lhs_global_state->AddLocalState(*lhs_local_state);
+		lhs_global_state->PrepareMergePhase();
+		while (lhs_global_state->sorted_blocks.size() > 1) {
+			MergeSorter merge_sorter(*lhs_global_state, buffer_manager);
+			merge_sorter.PerformInMergeRound();
+			lhs_global_state->CompleteMergeRound();
 		}
 	}
 
@@ -55265,7 +55096,176 @@ public:
 };
 
 unique_ptr<OperatorState> PhysicalPiecewiseMergeJoin::GetOperatorState(ClientContext &context) const {
-	return make_unique<PiecewiseMergeJoinState>(*this);
+	auto &buffer_manager = BufferManager::GetBufferManager(context);
+	auto &config = ClientConfig::GetConfig(context);
+	return make_unique<PiecewiseMergeJoinState>(*this, buffer_manager, config.force_external);
+}
+
+static inline idx_t SortedBlockNotNull(const idx_t base, const idx_t count, const idx_t not_null) {
+	return MinValue(base + count, MaxValue(base, not_null)) - base;
+}
+
+static int MergeJoinComparisonValue(ExpressionType comparison) {
+	switch (comparison) {
+	case ExpressionType::COMPARE_LESSTHAN:
+	case ExpressionType::COMPARE_GREATERTHAN:
+		return -1;
+	case ExpressionType::COMPARE_LESSTHANOREQUALTO:
+	case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
+		return 0;
+	default:
+		throw InternalException("Unimplemented comparison type for merge join!");
+	}
+}
+
+struct BlockMergeInfo {
+	GlobalSortState &state;
+	//! The block being scanned
+	const idx_t block_idx;
+	//! The start position being read from the block
+	const idx_t base_idx;
+	//! The number of not-NULL values in the block (they are at the end)
+	const idx_t not_null;
+	//! The current offset in the block
+	idx_t &entry_idx;
+	SelectionVector result;
+
+	BlockMergeInfo(GlobalSortState &state, idx_t block_idx, idx_t base_idx, idx_t &entry_idx, idx_t not_null)
+	    : state(state), block_idx(block_idx), base_idx(base_idx), not_null(not_null), entry_idx(entry_idx),
+	      result(STANDARD_VECTOR_SIZE) {
+	}
+};
+
+static void SliceSortedPayload(DataChunk &payload, BlockMergeInfo &info, const idx_t result_count, const idx_t vcount,
+                               const idx_t left_cols = 0) {
+	// There should only be one sorted block if they have been sorted
+	D_ASSERT(info.state.sorted_blocks.size() == 1);
+	SBScanState read_state(info.state.buffer_manager, info.state);
+	read_state.sb = info.state.sorted_blocks[0].get();
+	auto &sorted_data = *read_state.sb->payload_data;
+
+	// We have to create pointers for the entire block
+	// because unswizzle works on ranges not selections.
+	read_state.SetIndices(info.block_idx, info.base_idx);
+	read_state.PinData(sorted_data);
+	const auto data_ptr = read_state.DataPtr(sorted_data);
+	const auto next = vcount - info.base_idx;
+
+	// Set up a batch of pointers to scan data from
+	Vector addresses(LogicalType::POINTER, next);
+	auto data_pointers = FlatVector::GetData<data_ptr_t>(addresses);
+
+	// Set up the data pointers
+	data_ptr_t row_ptr = data_ptr;
+	const idx_t &row_width = sorted_data.layout.GetRowWidth();
+	for (idx_t i = 0; i < next; ++i) {
+		data_pointers[i] = row_ptr;
+		row_ptr += row_width;
+	}
+	// Unswizzle the offsets back to pointers (if needed)
+	if (!sorted_data.layout.AllConstant() && info.state.external) {
+		RowOperations::UnswizzlePointers(sorted_data.layout, data_ptr, read_state.payload_heap_handle->Ptr(), next);
+	}
+
+	// Deserialize the payload data
+	for (idx_t col_idx = 0; col_idx < sorted_data.layout.ColumnCount(); col_idx++) {
+		const auto col_offset = sorted_data.layout.GetOffsets()[col_idx];
+		RowOperations::Gather(addresses, info.result, payload.data[left_cols + col_idx],
+		                      *FlatVector::IncrementalSelectionVector(), result_count, col_offset, col_idx);
+	}
+}
+
+static void MergeJoinPinSortingBlock(SBScanState &scan, const idx_t block_idx) {
+	scan.SetIndices(block_idx, 0);
+	scan.PinRadix(block_idx);
+
+	auto &sd = *scan.sb->blob_sorting_data;
+	if (block_idx < sd.data_blocks.size()) {
+		scan.PinData(sd);
+	}
+}
+
+static data_ptr_t MergeJoinRadixPtr(SBScanState &scan, const idx_t entry_idx) {
+	scan.entry_idx = entry_idx;
+	return scan.RadixPtr();
+}
+
+static idx_t MergeJoinSimpleBlocks(PiecewiseMergeJoinState &lstate, MergeJoinGlobalState &rstate, bool *found_match,
+                                   const ExpressionType comparison) {
+	const auto cmp = MergeJoinComparisonValue(comparison);
+
+	// The sort parameters should all be the same
+	auto &lsort = *lstate.lhs_global_state;
+	auto &rsort = rstate.rhs_global_sort_state;
+	D_ASSERT(lsort.sort_layout.all_constant == rsort.sort_layout.all_constant);
+	const auto all_constant = lsort.sort_layout.all_constant;
+	D_ASSERT(lsort.external == rsort.external);
+	const auto external = lsort.external;
+
+	// There should only be one sorted block if they have been sorted
+	D_ASSERT(lsort.sorted_blocks.size() == 1);
+	SBScanState lread(lsort.buffer_manager, lsort);
+	lread.sb = lsort.sorted_blocks[0].get();
+
+	const idx_t l_block_idx = 0;
+	idx_t l_entry_idx = 0;
+	const auto lhs_not_null = lstate.lhs_count - lstate.lhs_has_null;
+	MergeJoinPinSortingBlock(lread, l_block_idx);
+	auto l_ptr = MergeJoinRadixPtr(lread, l_entry_idx);
+
+	D_ASSERT(rsort.sorted_blocks.size() == 1);
+	SBScanState rread(rsort.buffer_manager, rsort);
+	rread.sb = rsort.sorted_blocks[0].get();
+
+	const auto cmp_size = lsort.sort_layout.comparison_size;
+	const auto entry_size = lsort.sort_layout.entry_size;
+
+	idx_t right_base = 0;
+	for (idx_t r_block_idx = 0; r_block_idx < rread.sb->radix_sorting_data.size(); r_block_idx++) {
+		// we only care about the BIGGEST value in each of the RHS data blocks
+		// because we want to figure out if the LHS values are less than [or equal] to ANY value
+		// get the biggest value from the RHS chunk
+		MergeJoinPinSortingBlock(rread, r_block_idx);
+
+		auto &rblock = rread.sb->radix_sorting_data[r_block_idx];
+		const auto r_not_null = SortedBlockNotNull(right_base, rblock.count, rstate.rhs_count - rstate.rhs_has_null);
+		if (r_not_null == 0) {
+			break;
+		}
+		const auto r_entry_idx = r_not_null - 1;
+		right_base += rblock.count;
+
+		auto r_ptr = MergeJoinRadixPtr(rread, r_entry_idx);
+
+		// now we start from the current lpos value and check if we found a new value that is [<= OR <] the max RHS
+		// value
+		while (true) {
+			int comp_res;
+			if (all_constant) {
+				comp_res = FastMemcmp(l_ptr, r_ptr, cmp_size);
+			} else {
+				lread.entry_idx = l_entry_idx;
+				rread.entry_idx = r_entry_idx;
+				comp_res = Comparators::CompareTuple(lread, rread, l_ptr, r_ptr, lsort.sort_layout, external);
+			}
+
+			if (comp_res <= cmp) {
+				// found a match for lpos, set it in the found_match vector
+				found_match[l_entry_idx] = true;
+				l_entry_idx++;
+				l_ptr += entry_size;
+				if (l_entry_idx >= lhs_not_null) {
+					// early out: we exhausted the entire LHS and they all match
+					return 0;
+				}
+			} else {
+				// we found no match: any subsequent value from the LHS we scan now will be bigger and thus also not
+				// match move to the next RHS chunk
+				break;
+			}
+		}
+	}
+	return 0;
 }
 
 void PhysicalPiecewiseMergeJoin::ResolveSimpleJoin(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
@@ -55275,37 +55275,138 @@ void PhysicalPiecewiseMergeJoin::ResolveSimpleJoin(ExecutionContext &context, Da
 
 	state.ResolveJoinKeys(input);
 
-	ScalarMergeInfo left_info(state.left_orders, state.join_keys.data[0].GetType(), state.left_position);
-	ChunkMergeInfo right_info(gstate.right_conditions, gstate.right_orders);
-
 	// perform the actual join
-	MergeJoinSimple::Perform(left_info, right_info, conditions[0].comparison);
+	bool found_match[STANDARD_VECTOR_SIZE];
+	memset(found_match, 0, sizeof(found_match));
+	MergeJoinSimpleBlocks(state, gstate, found_match, conditions[0].comparison);
 
-	// now construct the result based ont he join result
+	// extract the sorted payload
+	idx_t left_position = 0;
+	const auto lhs_not_null = state.lhs_count - state.lhs_has_null;
+	BlockMergeInfo left_info(*state.lhs_global_state, 0, 0, left_position, lhs_not_null);
+	left_info.result.Initialize(nullptr);
+
+	DataChunk payload;
+	payload.Initialize(input.GetTypes());
+	SliceSortedPayload(payload, left_info, state.lhs_count, state.lhs_count);
+	payload.SetCardinality(state.lhs_count);
+
+	// now construct the result based on the join result
 	switch (join_type) {
-	case JoinType::MARK:
-		PhysicalJoin::ConstructMarkJoinResult(state.join_keys, input, chunk, right_info.found_match, gstate.has_null);
+	case JoinType::MARK: {
+		// The only part of the join keys that is actually used is the validity mask.
+		// Since the payload is sorted, we can just set the tail end of the validity masks to invalid.
+		for (auto &key : state.lhs_keys.data) {
+			key.Normalify(state.lhs_keys.size());
+			auto &mask = FlatVector::Validity(key);
+			if (mask.AllValid()) {
+				continue;
+			}
+			mask.SetAllValid(lhs_not_null);
+			for (idx_t i = lhs_not_null; i < state.lhs_count; ++i) {
+				mask.SetInvalid(i);
+			}
+		}
+		// So we make a set of keys that have the validity mask set for the
+		PhysicalJoin::ConstructMarkJoinResult(state.lhs_keys, payload, chunk, found_match, gstate.rhs_has_null);
 		break;
+	}
 	case JoinType::SEMI:
-		PhysicalJoin::ConstructSemiJoinResult(input, chunk, right_info.found_match);
+		PhysicalJoin::ConstructSemiJoinResult(payload, chunk, found_match);
 		break;
 	case JoinType::ANTI:
-		PhysicalJoin::ConstructAntiJoinResult(input, chunk, right_info.found_match);
+		PhysicalJoin::ConstructAntiJoinResult(payload, chunk, found_match);
 		break;
 	default:
 		throw NotImplementedException("Unimplemented join type for merge join");
 	}
 }
 
+static idx_t MergeJoinComplexBlocks(BlockMergeInfo &l, BlockMergeInfo &r, const ExpressionType comparison) {
+	const auto cmp = MergeJoinComparisonValue(comparison);
+
+	// The sort parameters should all be the same
+	D_ASSERT(l.state.sort_layout.all_constant == r.state.sort_layout.all_constant);
+	const auto all_constant = r.state.sort_layout.all_constant;
+	D_ASSERT(l.state.external == r.state.external);
+	const auto external = l.state.external;
+
+	// There should only be one sorted block if they have been sorted
+	D_ASSERT(l.state.sorted_blocks.size() == 1);
+	SBScanState lread(l.state.buffer_manager, l.state);
+	lread.sb = l.state.sorted_blocks[0].get();
+	D_ASSERT(lread.sb->radix_sorting_data.size() == 1);
+	MergeJoinPinSortingBlock(lread, l.block_idx);
+	auto l_start = MergeJoinRadixPtr(lread, 0);
+	auto l_ptr = MergeJoinRadixPtr(lread, l.entry_idx);
+
+	D_ASSERT(r.state.sorted_blocks.size() == 1);
+	SBScanState rread(r.state.buffer_manager, r.state);
+	rread.sb = r.state.sorted_blocks[0].get();
+
+	if (r.entry_idx >= r.not_null) {
+		return 0;
+	}
+
+	MergeJoinPinSortingBlock(rread, r.block_idx);
+	auto r_ptr = MergeJoinRadixPtr(rread, r.entry_idx);
+
+	const auto cmp_size = l.state.sort_layout.comparison_size;
+	const auto entry_size = l.state.sort_layout.entry_size;
+
+	idx_t result_count = 0;
+	while (true) {
+		if (l.entry_idx < l.not_null) {
+			int comp_res;
+			if (all_constant) {
+				comp_res = FastMemcmp(l_ptr, r_ptr, cmp_size);
+			} else {
+				lread.entry_idx = l.entry_idx;
+				rread.entry_idx = r.entry_idx;
+				comp_res = Comparators::CompareTuple(lread, rread, l_ptr, r_ptr, l.state.sort_layout, external);
+			}
+
+			if (comp_res <= cmp) {
+				// left side smaller: found match
+				l.result.set_index(result_count, sel_t(l.entry_idx - l.base_idx));
+				r.result.set_index(result_count, sel_t(r.entry_idx - r.base_idx));
+				result_count++;
+				// move left side forward
+				l.entry_idx++;
+				l_ptr += entry_size;
+				if (result_count == STANDARD_VECTOR_SIZE) {
+					// out of space!
+					break;
+				}
+				continue;
+			}
+		}
+		// right side smaller or equal, or left side exhausted: move
+		// right pointer forward reset left side to start
+		r.entry_idx++;
+		if (r.entry_idx >= r.not_null) {
+			break;
+		}
+		r_ptr += entry_size;
+
+		l_ptr = l_start;
+		l.entry_idx = 0;
+	}
+
+	return result_count;
+}
+
 OperatorResultType PhysicalPiecewiseMergeJoin::ResolveComplexJoin(ExecutionContext &context, DataChunk &input,
                                                                   DataChunk &chunk, OperatorState &state_p) const {
 	auto &state = (PiecewiseMergeJoinState &)state_p;
 	auto &gstate = (MergeJoinGlobalState &)*sink_state;
+	auto &rsorted = *gstate.rhs_global_sort_state.sorted_blocks[0];
 	do {
 		if (state.first_fetch) {
 			state.ResolveJoinKeys(input);
 
 			state.right_chunk_index = 0;
+			state.right_base = 0;
 			state.left_position = 0;
 			state.right_position = 0;
 			state.first_fetch = false;
@@ -55315,47 +55416,53 @@ OperatorResultType PhysicalPiecewiseMergeJoin::ResolveComplexJoin(ExecutionConte
 			if (IsLeftOuterJoin(join_type)) {
 				// left join: before we move to the next chunk, see if we need to output any vectors that didn't
 				// have a match found
-				PhysicalJoin::ConstructLeftJoinResult(input, chunk, state.left_found_match.get());
-				memset(state.left_found_match.get(), 0, sizeof(bool) * STANDARD_VECTOR_SIZE);
+				PhysicalJoin::ConstructLeftJoinResult(input, chunk, state.lhs_found_match.get());
+				memset(state.lhs_found_match.get(), 0, sizeof(bool) * STANDARD_VECTOR_SIZE);
 			}
 			state.first_fetch = true;
 			state.finished = false;
 			return OperatorResultType::NEED_MORE_INPUT;
 		}
 
-		auto &right_chunk = gstate.right_chunks.GetChunk(state.right_chunk_index);
-		auto &right_condition_chunk = gstate.right_conditions.GetChunk(state.right_chunk_index);
-		auto &right_orders = gstate.right_orders[state.right_chunk_index];
+		const auto lhs_not_null = state.lhs_count - state.lhs_has_null;
+		BlockMergeInfo left_info(*state.lhs_global_state, 0, 0, state.left_position, lhs_not_null);
 
-		ScalarMergeInfo left_info(state.left_orders, state.join_keys.data[0].GetType(), state.left_position);
-		ScalarMergeInfo right_info(right_orders, right_condition_chunk.data[0].GetType(), state.right_position);
+		const auto &rblock = rsorted.radix_sorting_data[state.right_chunk_index];
+		const auto rhs_not_null =
+		    SortedBlockNotNull(state.right_base, rblock.count, gstate.rhs_count - gstate.rhs_has_null);
+		BlockMergeInfo right_info(gstate.rhs_global_sort_state, state.right_chunk_index, state.right_position,
+		                          state.right_position, rhs_not_null);
 
-		idx_t result_count = MergeJoinComplex::Perform(left_info, right_info, conditions[0].comparison);
+		idx_t result_count = MergeJoinComplexBlocks(left_info, right_info, conditions[0].comparison);
 		if (result_count == 0) {
 			// exhausted this chunk on the right side
 			// move to the next right chunk
 			state.left_position = 0;
 			state.right_position = 0;
+			state.right_base += rsorted.radix_sorting_data[state.right_chunk_index].count;
 			state.right_chunk_index++;
-			if (state.right_chunk_index >= gstate.right_chunks.ChunkCount()) {
+			if (state.right_chunk_index >= rsorted.radix_sorting_data.size()) {
 				state.finished = true;
 			}
 		} else {
 			// found matches: mark the found matches if required
-			if (state.left_found_match) {
+			if (state.lhs_found_match) {
 				for (idx_t i = 0; i < result_count; i++) {
-					state.left_found_match[left_info.result.get_index(i)] = true;
+					state.lhs_found_match[left_info.result[i]] = true;
 				}
 			}
-			if (gstate.right_found_match) {
-				idx_t base_index = state.right_chunk_index * STANDARD_VECTOR_SIZE;
+			if (gstate.rhs_found_match) {
+				//	Absolute position of the block + start position inside that block
+				const idx_t base_index = state.right_base + right_info.base_idx;
 				for (idx_t i = 0; i < result_count; i++) {
-					gstate.right_found_match[base_index + right_info.result.get_index(i)] = true;
+					gstate.rhs_found_match[base_index + right_info.result[i]] = true;
 				}
 			}
 			// found matches: output them
-			chunk.Slice(input, left_info.result, result_count);
-			chunk.Slice(right_chunk, right_info.result, result_count, input.ColumnCount());
+			SliceSortedPayload(chunk, left_info, result_count, input.size());
+			SliceSortedPayload(chunk, right_info, result_count, right_info.entry_idx + 1, input.ColumnCount());
+			chunk.SetCardinality(result_count);
+			chunk.Verify();
 		}
 	} while (chunk.size() == 0);
 	return OperatorResultType::HAVE_MORE_OUTPUT;
@@ -55365,10 +55472,10 @@ OperatorResultType PhysicalPiecewiseMergeJoin::Execute(ExecutionContext &context
                                                        OperatorState &state) const {
 	auto &gstate = (MergeJoinGlobalState &)*sink_state;
 
-	if (gstate.right_chunks.Count() == 0) {
+	if (gstate.Count() == 0) {
 		// empty RHS
 		if (!EmptyResultIfRHSIsEmpty()) {
-			ConstructEmptyJoinResult(join_type, gstate.has_null, input, chunk);
+			ConstructEmptyJoinResult(join_type, gstate.rhs_has_null, input, chunk);
 			return OperatorResultType::NEED_MORE_INPUT;
 		} else {
 			return OperatorResultType::FINISHED;
@@ -55392,169 +55499,6 @@ OperatorResultType PhysicalPiecewiseMergeJoin::Execute(ExecutionContext &context
 	}
 }
 
-//===--------------------------------------------------------------------===//
-// OrderVector
-//===--------------------------------------------------------------------===//
-template <class T, class OP>
-static sel_t TemplatedQuicksortInitial(T *data, const SelectionVector &sel, const SelectionVector &not_null_sel,
-                                       idx_t count, SelectionVector &result) {
-	// select pivot
-	auto pivot_idx = not_null_sel.get_index(0);
-	auto dpivot_idx = sel.get_index(pivot_idx);
-	sel_t low = 0, high = count - 1;
-	// now insert elements
-	for (idx_t i = 1; i < count; i++) {
-		auto idx = not_null_sel.get_index(i);
-		auto didx = sel.get_index(idx);
-		if (OP::Operation(data[didx], data[dpivot_idx])) {
-			result.set_index(low++, idx);
-		} else {
-			result.set_index(high--, idx);
-		}
-	}
-	D_ASSERT(low == high);
-	result.set_index(low, pivot_idx);
-	return low;
-}
-
-struct QuickSortPivot {
-	QuickSortPivot(sel_t left_p, sel_t right_p) : left(left_p), right(right_p) {
-	}
-
-	sel_t left;
-	sel_t right;
-};
-
-template <class T, class OP>
-static void TemplatedQuicksortRefine(T *data, const SelectionVector &sel, idx_t count, SelectionVector &result,
-                                     sel_t left, sel_t right, vector<QuickSortPivot> &pivots) {
-	if (left >= right) {
-		return;
-	}
-
-	sel_t middle = left + (right - left) / 2;
-	sel_t dpivot_idx = sel.get_index(result.get_index(middle));
-
-	// move the mid point value to the front.
-	sel_t i = left + 1;
-	sel_t j = right;
-
-	result.swap(middle, left);
-	while (i <= j) {
-		while (i <= j && (OP::Operation(data[sel.get_index(result.get_index(i))], data[dpivot_idx]))) {
-			i++;
-		}
-
-		while (i <= j && !OP::Operation(data[sel.get_index(result.get_index(j))], data[dpivot_idx])) {
-			j--;
-		}
-
-		if (i < j) {
-			result.swap(i, j);
-		}
-	}
-	result.swap(i - 1, left);
-	sel_t part = i - 1;
-
-	if (part > 0) {
-		pivots.emplace_back(left, part - 1);
-	}
-	if (part + 1 < right) {
-		pivots.emplace_back(part + 1, right);
-	}
-}
-
-template <class T, class OP>
-void TemplatedQuicksort(T *__restrict data, const SelectionVector &sel, const SelectionVector &not_null_sel,
-                        idx_t count, SelectionVector &result) {
-	auto part = TemplatedQuicksortInitial<T, OP>(data, sel, not_null_sel, count, result);
-	if (part > count) {
-		return;
-	}
-	vector<QuickSortPivot> pivots;
-	pivots.emplace_back(0, part);
-	pivots.emplace_back(part + 1, count - 1);
-	for (idx_t i = 0; i < pivots.size(); i++) {
-		auto pivot = pivots[i];
-		TemplatedQuicksortRefine<T, OP>(data, sel, count, result, pivot.left, pivot.right, pivots);
-	}
-}
-
-template <class T>
-static void TemplatedQuicksort(VectorData &vdata, const SelectionVector &not_null_sel, idx_t not_null_count,
-                               SelectionVector &result) {
-	if (not_null_count == 0) {
-		return;
-	}
-	TemplatedQuicksort<T, duckdb::LessThanEquals>((T *)vdata.data, *vdata.sel, not_null_sel, not_null_count, result);
-}
-
-void OrderVector(Vector &vector, idx_t count, MergeOrder &order) {
-	if (count == 0) {
-		order.count = 0;
-		return;
-	}
-	vector.Orrify(count, order.vdata);
-	auto &vdata = order.vdata;
-
-	// first filter out all the non-null values
-	SelectionVector not_null(STANDARD_VECTOR_SIZE);
-	idx_t not_null_count = 0;
-	for (idx_t i = 0; i < count; i++) {
-		auto idx = vdata.sel->get_index(i);
-		if (vdata.validity.RowIsValid(idx)) {
-			not_null.set_index(not_null_count++, i);
-		}
-	}
-
-	order.count = not_null_count;
-	order.order.Initialize(STANDARD_VECTOR_SIZE);
-	switch (vector.GetType().InternalType()) {
-	case PhysicalType::BOOL:
-	case PhysicalType::INT8:
-		TemplatedQuicksort<int8_t>(vdata, not_null, not_null_count, order.order);
-		break;
-	case PhysicalType::INT16:
-		TemplatedQuicksort<int16_t>(vdata, not_null, not_null_count, order.order);
-		break;
-	case PhysicalType::INT32:
-		TemplatedQuicksort<int32_t>(vdata, not_null, not_null_count, order.order);
-		break;
-	case PhysicalType::INT64:
-		TemplatedQuicksort<int64_t>(vdata, not_null, not_null_count, order.order);
-		break;
-	case PhysicalType::UINT8:
-		TemplatedQuicksort<uint8_t>(vdata, not_null, not_null_count, order.order);
-		break;
-	case PhysicalType::UINT16:
-		TemplatedQuicksort<uint16_t>(vdata, not_null, not_null_count, order.order);
-		break;
-	case PhysicalType::UINT32:
-		TemplatedQuicksort<uint32_t>(vdata, not_null, not_null_count, order.order);
-		break;
-	case PhysicalType::UINT64:
-		TemplatedQuicksort<uint64_t>(vdata, not_null, not_null_count, order.order);
-		break;
-	case PhysicalType::INT128:
-		TemplatedQuicksort<hugeint_t>(vdata, not_null, not_null_count, order.order);
-		break;
-	case PhysicalType::FLOAT:
-		TemplatedQuicksort<float>(vdata, not_null, not_null_count, order.order);
-		break;
-	case PhysicalType::DOUBLE:
-		TemplatedQuicksort<double>(vdata, not_null, not_null_count, order.order);
-		break;
-	case PhysicalType::INTERVAL:
-		TemplatedQuicksort<interval_t>(vdata, not_null, not_null_count, order.order);
-		break;
-	case PhysicalType::VARCHAR:
-		TemplatedQuicksort<string_t>(vdata, not_null, not_null_count, order.order);
-		break;
-	default:
-		throw NotImplementedException("Unimplemented type for sort");
-	}
-}
-
 //===--------------------------------------------------------------------===//
 // Source
 //===--------------------------------------------------------------------===//
@@ -55565,12 +55509,13 @@ public:
 
 	mutex lock;
 	const PhysicalPiecewiseMergeJoin &op;
+	unique_ptr<PayloadScanner> scanner;
 	idx_t right_outer_position;
 
 public:
 	idx_t MaxThreads() override {
 		auto &sink = (MergeJoinGlobalState &)*op.sink_state;
-		return sink.right_chunks.Count() / (STANDARD_VECTOR_SIZE * 10);
+		return sink.Count() / (STANDARD_VECTOR_SIZE * idx_t(10));
 	}
 };
 
@@ -55578,17 +55523,63 @@ unique_ptr<GlobalSourceState> PhysicalPiecewiseMergeJoin::GetGlobalSourceState(C
 	return make_unique<PiecewiseJoinScanState>(*this);
 }
 
-void PhysicalPiecewiseMergeJoin::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
+void PhysicalPiecewiseMergeJoin::GetData(ExecutionContext &context, DataChunk &result, GlobalSourceState &gstate,
                                          LocalSourceState &lstate) const {
 	D_ASSERT(IsRightOuterJoin(join_type));
 	// check if we need to scan any unmatched tuples from the RHS for the full/right outer join
 	auto &sink = (MergeJoinGlobalState &)*sink_state;
 	auto &state = (PiecewiseJoinScanState &)gstate;
 
+	lock_guard<mutex> l(state.lock);
+	if (!state.scanner) {
+		// Initialize scanner (if not yet initialized)
+		auto &sort_state = sink.rhs_global_sort_state;
+		if (sort_state.sorted_blocks.empty()) {
+			return;
+		}
+		state.scanner = make_unique<PayloadScanner>(*sort_state.sorted_blocks[0]->payload_data, sort_state);
+	}
+
 	// if the LHS is exhausted in a FULL/RIGHT OUTER JOIN, we scan the found_match for any chunks we
 	// still need to output
-	lock_guard<mutex> l(state.lock);
-	ConstructFullOuterJoinResult(sink.right_found_match.get(), sink.right_chunks, chunk, state.right_outer_position);
+	const auto found_match = sink.rhs_found_match.get();
+
+	// ConstructFullOuterJoinResult(sink.rhs_found_match.get(), sink.right_chunks, chunk, state.right_outer_position);
+	DataChunk rhs_chunk;
+	rhs_chunk.Initialize(children[1]->types);
+	SelectionVector rsel(STANDARD_VECTOR_SIZE);
+	for (;;) {
+		// Read the next sorted chunk
+		state.scanner->Scan(rhs_chunk);
+
+		const auto count = rhs_chunk.size();
+		if (count == 0) {
+			return;
+		}
+
+		idx_t result_count = 0;
+		// figure out which tuples didn't find a match in the RHS
+		for (idx_t i = 0; i < count; i++) {
+			if (!found_match[state.right_outer_position + i]) {
+				rsel.set_index(result_count++, i);
+			}
+		}
+		state.right_outer_position += count;
+
+		if (result_count > 0) {
+			// if there were any tuples that didn't find a match, output them
+			const idx_t left_column_count = result.ColumnCount() - rhs_chunk.ColumnCount();
+			for (idx_t col_idx = 0; col_idx < left_column_count; ++col_idx) {
+				result.data[col_idx].SetVectorType(VectorType::CONSTANT_VECTOR);
+				ConstantVector::SetNull(result.data[col_idx], true);
+			}
+			for (idx_t col_idx = 0; col_idx < rhs_chunk.ColumnCount(); ++col_idx) {
+				result.data[left_column_count + col_idx].Slice(rhs_chunk.data[col_idx], rsel, result_count);
+			}
+			result.SetCardinality(result_count);
+			return;
+		}
+	}
 }
 
 } // namespace duckdb