duckdb 0.3.5-dev617.0 → 0.3.5-dev651.0

package/src/duckdb.cpp

@@ -7044,6 +7044,8 @@ string PhysicalOperatorToString(PhysicalOperatorType type) {
 		return "LIMIT";
 	case PhysicalOperatorType::LIMIT_PERCENT:
 		return "LIMIT_PERCENT";
+	case PhysicalOperatorType::STREAMING_LIMIT:
+		return "STREAMING_LIMIT";
 	case PhysicalOperatorType::RESERVOIR_SAMPLE:
 		return "RESERVOIR_SAMPLE";
 	case PhysicalOperatorType::STREAMING_SAMPLE:
@@ -7142,6 +7144,8 @@ string PhysicalOperatorToString(PhysicalOperatorType type) {
 		return "INOUT_FUNCTION";
 	case PhysicalOperatorType::CREATE_TYPE:
 		return "CREATE_TYPE";
+	case PhysicalOperatorType::RESULT_COLLECTOR:
+		return "RESULT_COLLECTOR";
 	case PhysicalOperatorType::INVALID:
 		break;
 	}
@@ -36623,6 +36627,9 @@ public:
 	unique_ptr<PhysicalHashAggregate> distinct;
 	vector<PhysicalOperator *> delim_scans;
 
+public:
+	vector<PhysicalOperator *> GetChildren() const override;
+
 public:
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
 	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
@@ -36640,6 +36647,9 @@ public:
 	}
 
 	string ParamsToString() const override;
+
+public:
+	void BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) override;
 };
 
 } // namespace duckdb
@@ -37666,10 +37676,127 @@ unique_ptr<RenderTree> TreeRenderer::CreateTree(const Pipeline &op) {
 	return CreateRenderTree<PipelineRenderNode>(*node);
 }
 
+} // namespace duckdb//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/types/batched_chunk_collection.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
+namespace duckdb {
+
+struct BatchedChunkScanState {
+	map<idx_t, unique_ptr<ChunkCollection>>::iterator iterator;
+	idx_t chunk_index;
+};
+
+//!  A BatchedChunkCollection holds a number of data entries that are partitioned by batch index
+//! Scans over a BatchedChunkCollection are ordered by batch index
+class BatchedChunkCollection {
+public:
+	DUCKDB_API BatchedChunkCollection();
+
+	//! Appends a datachunk with the given batch index to the batched collection
+	DUCKDB_API void Append(DataChunk &input, idx_t batch_index);
+
+	//! Merge the other batched chunk collection into this batched collection
+	DUCKDB_API void Merge(BatchedChunkCollection &other);
+
+	//! Initialize a scan over the batched chunk collection
+	DUCKDB_API void InitializeScan(BatchedChunkScanState &state);
+
+	//! Scan a chunk from the batched chunk collection, in-order of batch index
+	DUCKDB_API void Scan(BatchedChunkScanState &state, DataChunk &output);
+
+	DUCKDB_API string ToString() const;
+	DUCKDB_API void Print() const;
+
+private:
+	//! The data of the batched chunk collection - a set of batch_index -> ChunkCollection pointers
+	map<idx_t, unique_ptr<ChunkCollection>> data;
+};
 } // namespace duckdb
 
 
 
+namespace duckdb {
+
+BatchedChunkCollection::BatchedChunkCollection() {
+}
+
+void BatchedChunkCollection::Append(DataChunk &input, idx_t batch_index) {
+	D_ASSERT(batch_index != DConstants::INVALID_INDEX);
+	auto entry = data.find(batch_index);
+	ChunkCollection *collection;
+	if (entry == data.end()) {
+		auto new_collection = make_unique<ChunkCollection>();
+		collection = new_collection.get();
+		data.insert(make_pair(batch_index, move(new_collection)));
+	} else {
+		collection = entry->second.get();
+	}
+	collection->Append(input);
+}
+
+void BatchedChunkCollection::Merge(BatchedChunkCollection &other) {
+	for (auto &entry : other.data) {
+		if (data.find(entry.first) != data.end()) {
+			throw InternalException(
+			    "BatchChunkCollection::Merge error - batch index %d is present in both collections. This occurs when "
+			    "batch indexes are not uniquely distributed over threads",
+			    entry.first);
+		}
+		data[entry.first] = move(entry.second);
+	}
+	other.data.clear();
+}
+
+void BatchedChunkCollection::InitializeScan(BatchedChunkScanState &state) {
+	state.iterator = data.begin();
+	state.chunk_index = 0;
+}
+
+void BatchedChunkCollection::Scan(BatchedChunkScanState &state, DataChunk &output) {
+	while (state.iterator != data.end()) {
+		// check if there is a chunk remaining in this collection
+		auto collection = state.iterator->second.get();
+		if (state.chunk_index < collection->ChunkCount()) {
+			// there is! increment the chunk count
+			output.Reference(collection->GetChunk(state.chunk_index));
+			state.chunk_index++;
+			return;
+		}
+		// there isn't! move to the next collection
+		state.iterator++;
+		state.chunk_index = 0;
+	}
+}
+
+string BatchedChunkCollection::ToString() const {
+	string result;
+	result += "Batched Chunk Collection\n";
+	for (auto &entry : data) {
+		result += "Batch Index - " + to_string(entry.first) + "\n";
+		result += entry.second->ToString() + "\n\n";
+	}
+	return result;
+}
+
+void BatchedChunkCollection::Print() const {
+	Printer::Print(ToString());
+}
+
+} // namespace duckdb
+
+
+
+
 
 
 
@@ -38507,7 +38634,12 @@ void ChunkCollection::CopyCell(idx_t column, idx_t index, Vector &target, idx_t
 	VectorOperations::Copy(source, target, source_offset + 1, source_offset, target_offset);
 }
 
-void ChunkCollection::Print() {
+string ChunkCollection::ToString() const {
+	return chunks.empty() ? "ChunkCollection [ 0 ]"
+	                      : "ChunkCollection [ " + std::to_string(count) + " ]: \n" + chunks[0]->ToString();
+}
+
+void ChunkCollection::Print() const {
 	Printer::Print(ToString());
 }
 
@@ -48636,17 +48768,14 @@ inline idx_t ComparisonSelector::Select<duckdb::LessThanEquals>(Vector &left, Ve
 	return VectorOperations::LessThanEquals(left, right, sel, count, true_sel, false_sel);
 }
 
-static idx_t ComparesNotNull(ValidityMask &vleft, ValidityMask &vright, ValidityMask &vresult, idx_t count,
-                             SelectionVector &not_null) {
-	idx_t valid = 0;
+static void ComparesNotNull(VectorData &ldata, VectorData &rdata, ValidityMask &vresult, idx_t count) {
 	for (idx_t i = 0; i < count; ++i) {
-		if (vleft.RowIsValid(i) && vright.RowIsValid(i)) {
-			not_null.set_index(valid++, i);
-		} else {
+		auto lidx = ldata.sel->get_index(i);
+		auto ridx = rdata.sel->get_index(i);
+		if (!ldata.validity.RowIsValid(lidx) || !rdata.validity.RowIsValid(ridx)) {
 			vresult.SetInvalid(i);
 		}
 	}
-	return valid;
 }
 
 template <typename OP>
@@ -48673,23 +48802,17 @@ static void NestedComparisonExecutor(Vector &left, Vector &right, Vector &result
 
 	result.SetVectorType(VectorType::FLAT_VECTOR);
 	auto result_data = FlatVector::GetData<bool>(result);
-	auto &validity = FlatVector::Validity(result);
+	auto &result_validity = FlatVector::Validity(result);
 
 	VectorData leftv, rightv;
 	left.Orrify(count, leftv);
 	right.Orrify(count, rightv);
-
+	if (!leftv.validity.AllValid() || !rightv.validity.AllValid()) {
+		ComparesNotNull(leftv, rightv, result_validity, count);
+	}
 	SelectionVector true_sel(count);
 	SelectionVector false_sel(count);
-
-	idx_t match_count = 0;
-	if (leftv.validity.AllValid() && rightv.validity.AllValid()) {
-		match_count = ComparisonSelector::Select<OP>(left, right, nullptr, count, &true_sel, &false_sel);
-	} else {
-		SelectionVector not_null(count);
-		count = ComparesNotNull(leftv.validity, rightv.validity, validity, count, not_null);
-		match_count = ComparisonSelector::Select<OP>(left, right, &not_null, count, &true_sel, &false_sel);
-	}
+	idx_t match_count = ComparisonSelector::Select<OP>(left, right, nullptr, count, &true_sel, &false_sel);
 
 	for (idx_t i = 0; i < match_count; ++i) {
 		const auto idx = true_sel.get_index(i);
@@ -60510,6 +60633,171 @@ string PhysicalFilter::ParamsToString() const {
 	return expression->GetName();
 }
 
+} // namespace duckdb
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/helper/physical_batch_collector.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/helper/physical_result_collector.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
+namespace duckdb {
+class PreparedStatementData;
+
+//! PhysicalResultCollector is an abstract class that is used to generate the final result of a query
+class PhysicalResultCollector : public PhysicalOperator {
+public:
+	PhysicalResultCollector(PreparedStatementData &data);
+
+	StatementType statement_type;
+	StatementProperties properties;
+	PhysicalOperator *plan;
+	vector<string> names;
+
+public:
+	static unique_ptr<PhysicalResultCollector> GetResultCollector(ClientContext &context, PreparedStatementData &data);
+
+public:
+	//! The final method used to fetch the query result from this operator
+	virtual unique_ptr<QueryResult> GetResult(GlobalSinkState &state) = 0;
+
+	bool IsSink() const override {
+		return true;
+	}
+
+public:
+	vector<PhysicalOperator *> GetChildren() const override;
+
+	void BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) override;
+};
+
+} // namespace duckdb
+
+
+namespace duckdb {
+
+class PhysicalBatchCollector : public PhysicalResultCollector {
+public:
+	PhysicalBatchCollector(PreparedStatementData &data);
+
+public:
+	unique_ptr<QueryResult> GetResult(GlobalSinkState &state) override;
+
+public:
+	// Sink interface
+	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
+	                    DataChunk &input) const override;
+	void Combine(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate) const override;
+	SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
+	                          GlobalSinkState &gstate) const override;
+
+	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
+	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
+
+	bool RequiresBatchIndex() const override {
+		return true;
+	}
+
+	bool ParallelSink() const override {
+		return true;
+	}
+};
+
+} // namespace duckdb
+
+
+
+
+
+namespace duckdb {
+
+PhysicalBatchCollector::PhysicalBatchCollector(PreparedStatementData &data) : PhysicalResultCollector(data) {
+}
+
+//===--------------------------------------------------------------------===//
+// Sink
+//===--------------------------------------------------------------------===//
+class BatchCollectorGlobalState : public GlobalSinkState {
+public:
+	mutex glock;
+	BatchedChunkCollection data;
+	unique_ptr<MaterializedQueryResult> result;
+};
+
+class BatchCollectorLocalState : public LocalSinkState {
+public:
+	BatchedChunkCollection data;
+};
+
+SinkResultType PhysicalBatchCollector::Sink(ExecutionContext &context, GlobalSinkState &gstate,
+                                            LocalSinkState &lstate_p, DataChunk &input) const {
+	auto &state = (BatchCollectorLocalState &)lstate_p;
+	state.data.Append(input, state.batch_index);
+	return SinkResultType::NEED_MORE_INPUT;
+}
+
+void PhysicalBatchCollector::Combine(ExecutionContext &context, GlobalSinkState &gstate_p,
+                                     LocalSinkState &lstate_p) const {
+	auto &gstate = (BatchCollectorGlobalState &)gstate_p;
+	auto &state = (BatchCollectorLocalState &)lstate_p;
+
+	lock_guard<mutex> lock(gstate.glock);
+	gstate.data.Merge(state.data);
+}
+
+SinkFinalizeType PhysicalBatchCollector::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
+                                                  GlobalSinkState &gstate_p) const {
+	auto &gstate = (BatchCollectorGlobalState &)gstate_p;
+	auto result =
+	    make_unique<MaterializedQueryResult>(statement_type, properties, types, names, context.shared_from_this());
+	DataChunk output;
+	output.Initialize(types);
+
+	BatchedChunkScanState state;
+	gstate.data.InitializeScan(state);
+	while (true) {
+		output.Reset();
+		gstate.data.Scan(state, output);
+		if (output.size() == 0) {
+			break;
+		}
+		result->collection.Append(output);
+	}
+
+	gstate.result = move(result);
+	return SinkFinalizeType::READY;
+}
+
+unique_ptr<LocalSinkState> PhysicalBatchCollector::GetLocalSinkState(ExecutionContext &context) const {
+	return make_unique<BatchCollectorLocalState>();
+}
+
+unique_ptr<GlobalSinkState> PhysicalBatchCollector::GetGlobalSinkState(ClientContext &context) const {
+	return make_unique<BatchCollectorGlobalState>();
+}
+
+unique_ptr<QueryResult> PhysicalBatchCollector::GetResult(GlobalSinkState &state) {
+	auto &gstate = (BatchCollectorGlobalState &)state;
+	D_ASSERT(gstate.result);
+	return move(gstate.result);
+}
+
 } // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB
@@ -60586,6 +60874,12 @@ public:
 	PhysicalOperator *plan;
 	unique_ptr<PhysicalOperator> owned_plan;
 	shared_ptr<PreparedStatementData> prepared;
+
+public:
+	vector<PhysicalOperator *> GetChildren() const override;
+
+public:
+	void BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) override;
 };
 
 } // namespace duckdb
@@ -60597,6 +60891,15 @@ PhysicalExecute::PhysicalExecute(PhysicalOperator *plan)
     : PhysicalOperator(PhysicalOperatorType::EXECUTE, plan->types, -1), plan(plan) {
 }
 
+vector<PhysicalOperator *> PhysicalExecute::GetChildren() const {
+	return {plan};
+}
+
+void PhysicalExecute::BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) {
+	// EXECUTE statement: build pipeline on child
+	plan->BuildPipelines(executor, current, state);
+}
+
 } // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB
@@ -60724,16 +61027,18 @@ namespace duckdb {
 class PhysicalLimit : public PhysicalOperator {
 public:
 	PhysicalLimit(vector<LogicalType> types, idx_t limit, idx_t offset, unique_ptr<Expression> limit_expression,
-	              unique_ptr<Expression> offset_expression, idx_t estimated_cardinality)
-	    : PhysicalOperator(PhysicalOperatorType::LIMIT, move(types), estimated_cardinality), limit_value(limit),
-	      offset_value(offset), limit_expression(move(limit_expression)), offset_expression(move(offset_expression)) {
-	}
+	              unique_ptr<Expression> offset_expression, idx_t estimated_cardinality);
 
 	idx_t limit_value;
 	idx_t offset_value;
 	unique_ptr<Expression> limit_expression;
 	unique_ptr<Expression> offset_expression;
 
+public:
+	bool IsOrderDependent() const override {
+		return true;
+	}
+
 public:
 	// Source interface
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
@@ -60742,18 +61047,27 @@ public:
 
 public:
 	// Sink Interface
-	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
 	                    DataChunk &input) const override;
+	void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const override;
+	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
+	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
 
 	bool IsSink() const override {
 		return true;
 	}
 
-	bool SinkOrderMatters() const override {
+	bool ParallelSink() const override {
+		return true;
+	}
+
+	bool RequiresBatchIndex() const override {
 		return true;
 	}
 
+public:
+	static bool ComputeOffset(DataChunk &input, idx_t &limit, idx_t &offset, idx_t current_offset, idx_t &max_element,
+	                          Expression *limit_expression, Expression *offset_expression);
 	static bool HandleOffset(DataChunk &input, idx_t &current_offset, idx_t offset, idx_t limit);
 	static Value GetDelimiter(DataChunk &input, Expression *expr);
 };
@@ -60766,14 +61080,75 @@ public:
 
 
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/helper/physical_streaming_limit.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
+namespace duckdb {
+
+class PhysicalStreamingLimit : public PhysicalOperator {
+public:
+	PhysicalStreamingLimit(vector<LogicalType> types, idx_t limit, idx_t offset,
+	                       unique_ptr<Expression> limit_expression, unique_ptr<Expression> offset_expression,
+	                       idx_t estimated_cardinality, bool parallel);
+
+	idx_t limit_value;
+	idx_t offset_value;
+	unique_ptr<Expression> limit_expression;
+	unique_ptr<Expression> offset_expression;
+	bool parallel;
+
+public:
+	// Operator interface
+	unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const override;
+	unique_ptr<GlobalOperatorState> GetGlobalOperatorState(ClientContext &context) const override;
+	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
+
+	bool IsOrderDependent() const override;
+	bool ParallelOperator() const override;
+};
+
+} // namespace duckdb
+
+
 namespace duckdb {
 
+PhysicalLimit::PhysicalLimit(vector<LogicalType> types, idx_t limit, idx_t offset,
+                             unique_ptr<Expression> limit_expression, unique_ptr<Expression> offset_expression,
+                             idx_t estimated_cardinality)
+    : PhysicalOperator(PhysicalOperatorType::LIMIT, move(types), estimated_cardinality), limit_value(limit),
+      offset_value(offset), limit_expression(move(limit_expression)), offset_expression(move(offset_expression)) {
+}
+
 //===--------------------------------------------------------------------===//
 // Sink
 //===--------------------------------------------------------------------===//
 class LimitGlobalState : public GlobalSinkState {
 public:
-	explicit LimitGlobalState(const PhysicalLimit &op) : current_offset(0) {
+	explicit LimitGlobalState(const PhysicalLimit &op) {
+		limit = 0;
+		offset = 0;
+	}
+
+	mutex glock;
+	idx_t limit;
+	idx_t offset;
+	BatchedChunkCollection data;
+};
+
+class LimitLocalState : public LocalSinkState {
+public:
+	explicit LimitLocalState(const PhysicalLimit &op) : current_offset(0) {
 		this->limit = op.limit_expression ? DConstants::INVALID_INDEX : op.limit_value;
 		this->offset = op.offset_expression ? DConstants::INVALID_INDEX : op.offset_value;
 	}
@@ -60781,31 +61156,30 @@ public:
 	idx_t current_offset;
 	idx_t limit;
 	idx_t offset;
-	ChunkCollection data;
+	BatchedChunkCollection data;
 };
 
 unique_ptr<GlobalSinkState> PhysicalLimit::GetGlobalSinkState(ClientContext &context) const {
 	return make_unique<LimitGlobalState>(*this);
 }
 
-SinkResultType PhysicalLimit::Sink(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate,
-                                   DataChunk &input) const {
-	D_ASSERT(input.size() > 0);
-	auto &state = (LimitGlobalState &)gstate;
-	auto &limit = state.limit;
-	auto &offset = state.offset;
+unique_ptr<LocalSinkState> PhysicalLimit::GetLocalSinkState(ExecutionContext &context) const {
+	return make_unique<LimitLocalState>(*this);
+}
 
+bool PhysicalLimit::ComputeOffset(DataChunk &input, idx_t &limit, idx_t &offset, idx_t current_offset,
+                                  idx_t &max_element, Expression *limit_expression, Expression *offset_expression) {
 	if (limit != DConstants::INVALID_INDEX && offset != DConstants::INVALID_INDEX) {
-		idx_t max_element = limit + offset;
-		if ((limit == 0 || state.current_offset >= max_element) && !(limit_expression || offset_expression)) {
-			return SinkResultType::FINISHED;
+		max_element = limit + offset;
+		if ((limit == 0 || current_offset >= max_element) && !(limit_expression || offset_expression)) {
+			return false;
 		}
 	}
 
 	// get the next chunk from the child
 	if (limit == DConstants::INVALID_INDEX) {
 		limit = 1ULL << 62ULL;
-		Value val = GetDelimiter(input, limit_expression.get());
+		Value val = GetDelimiter(input, limit_expression);
 		if (!val.IsNull()) {
 			limit = val.GetValue<idx_t>();
 		}
@@ -60815,7 +61189,7 @@ SinkResultType PhysicalLimit::Sink(ExecutionContext &context, GlobalSinkState &g
 	}
 	if (offset == DConstants::INVALID_INDEX) {
 		offset = 0;
-		Value val = GetDelimiter(input, offset_expression.get());
+		Value val = GetDelimiter(input, offset_expression);
 		if (!val.IsNull()) {
 			offset = val.GetValue<idx_t>();
 		}
@@ -60823,42 +61197,77 @@ SinkResultType PhysicalLimit::Sink(ExecutionContext &context, GlobalSinkState &g
 			throw BinderException("Max value %lld for LIMIT/OFFSET is %lld", offset, 1ULL << 62ULL);
 		}
 	}
-	idx_t max_element = limit + offset;
-	if (limit == 0 || state.current_offset >= max_element) {
-		return SinkResultType::FINISHED;
-	}
-	if (!HandleOffset(input, state.current_offset, offset, limit)) {
-		return SinkResultType::NEED_MORE_INPUT;
+	max_element = limit + offset;
+	if (limit == 0 || current_offset >= max_element) {
+		return false;
 	}
+	return true;
+}
 
-	state.data.Append(input);
+SinkResultType PhysicalLimit::Sink(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate,
+                                   DataChunk &input) const {
+
+	D_ASSERT(input.size() > 0);
+	auto &state = (LimitLocalState &)lstate;
+	auto &limit = state.limit;
+	auto &offset = state.offset;
+
+	idx_t max_element;
+	if (!ComputeOffset(input, limit, offset, state.current_offset, max_element, limit_expression.get(),
+	                   offset_expression.get())) {
+		return SinkResultType::FINISHED;
+	}
+	state.data.Append(input, lstate.batch_index);
+	state.current_offset += input.size();
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
+void PhysicalLimit::Combine(ExecutionContext &context, GlobalSinkState &gstate_p, LocalSinkState &lstate_p) const {
+	auto &gstate = (LimitGlobalState &)gstate_p;
+	auto &state = (LimitLocalState &)lstate_p;
+
+	lock_guard<mutex> lock(gstate.glock);
+	gstate.limit = state.limit;
+	gstate.offset = state.offset;
+	gstate.data.Merge(state.data);
+}
+
 //===--------------------------------------------------------------------===//
 // Source
 //===--------------------------------------------------------------------===//
-class LimitOperatorState : public GlobalSourceState {
+class LimitSourceState : public GlobalSourceState {
 public:
-	LimitOperatorState() : chunk_idx(0) {
+	LimitSourceState() {
+		initialized = false;
+		current_offset = 0;
 	}
 
-	idx_t chunk_idx;
+	bool initialized;
+	idx_t current_offset;
+	BatchedChunkScanState scan_state;
 };
 
 unique_ptr<GlobalSourceState> PhysicalLimit::GetGlobalSourceState(ClientContext &context) const {
-	return make_unique<LimitOperatorState>();
+	return make_unique<LimitSourceState>();
 }
 
 void PhysicalLimit::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate_p,
                             LocalSourceState &lstate) const {
 	auto &gstate = (LimitGlobalState &)*sink_state;
-	auto &state = (LimitOperatorState &)gstate_p;
-	if (state.chunk_idx >= gstate.data.ChunkCount()) {
-		return;
+	auto &state = (LimitSourceState &)gstate_p;
+	while (state.current_offset < gstate.limit + gstate.offset) {
+		if (!state.initialized) {
+			gstate.data.InitializeScan(state.scan_state);
+			state.initialized = true;
+		}
+		gstate.data.Scan(state.scan_state, chunk);
+		if (chunk.size() == 0) {
+			break;
+		}
+		if (HandleOffset(chunk, state.current_offset, gstate.offset, gstate.limit)) {
+			break;
+		}
 	}
-	chunk.Reference(gstate.data.GetChunk(state.chunk_idx));
-	state.chunk_idx++;
 }
 
 bool PhysicalLimit::HandleOffset(DataChunk &input, idx_t &current_offset, idx_t offset, idx_t limit) {
@@ -60948,6 +61357,11 @@ public:
 	unique_ptr<Expression> limit_expression;
 	unique_ptr<Expression> offset_expression;
 
+public:
+	bool IsOrderDependent() const override {
+		return true;
+	}
+
 public:
 	// Source interface
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
@@ -60963,10 +61377,6 @@ public:
 	bool IsSink() const override {
 		return true;
 	}
-
-	bool SinkOrderMatters() const override {
-		return true;
-	}
 };
 
 } // namespace duckdb
@@ -61182,6 +61592,86 @@ void PhysicalLoad::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSo
 	}
 }
 
+} // namespace duckdb
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/helper/physical_materialized_collector.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+namespace duckdb {
+
+class PhysicalMaterializedCollector : public PhysicalResultCollector {
+public:
+	PhysicalMaterializedCollector(PreparedStatementData &data, bool parallel);
+
+	bool parallel;
+
+public:
+	unique_ptr<QueryResult> GetResult(GlobalSinkState &state) override;
+
+public:
+	// Sink interface
+	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
+	                    DataChunk &input) const override;
+
+	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
+
+	bool ParallelSink() const override;
+};
+
+} // namespace duckdb
+
+
+
+
+
+namespace duckdb {
+
+PhysicalMaterializedCollector::PhysicalMaterializedCollector(PreparedStatementData &data, bool parallel)
+    : PhysicalResultCollector(data), parallel(parallel) {
+}
+
+//===--------------------------------------------------------------------===//
+// Sink
+//===--------------------------------------------------------------------===//
+class MaterializedCollectorGlobalState : public GlobalSinkState {
+public:
+	mutex glock;
+	unique_ptr<MaterializedQueryResult> result;
+};
+
+SinkResultType PhysicalMaterializedCollector::Sink(ExecutionContext &context, GlobalSinkState &gstate_p,
+                                                   LocalSinkState &lstate, DataChunk &input) const {
+	auto &gstate = (MaterializedCollectorGlobalState &)gstate_p;
+	lock_guard<mutex> lock(gstate.glock);
+	gstate.result->collection.Append(input);
+	return SinkResultType::NEED_MORE_INPUT;
+}
+
+unique_ptr<GlobalSinkState> PhysicalMaterializedCollector::GetGlobalSinkState(ClientContext &context) const {
+	auto state = make_unique<MaterializedCollectorGlobalState>();
+	state->result =
+	    make_unique<MaterializedQueryResult>(statement_type, properties, types, names, context.shared_from_this());
+	return move(state);
+}
+
+unique_ptr<QueryResult> PhysicalMaterializedCollector::GetResult(GlobalSinkState &state) {
+	auto &gstate = (MaterializedCollectorGlobalState &)state;
+	D_ASSERT(gstate.result);
+	return move(gstate.result);
+}
+
+bool PhysicalMaterializedCollector::ParallelSink() const {
+	return parallel;
+}
+
 } // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB
@@ -61520,6 +62010,55 @@ string PhysicalReservoirSample::ParamsToString() const {
 }
 
 } // namespace duckdb
+
+
+
+
+
+
+
+namespace duckdb {
+
+PhysicalResultCollector::PhysicalResultCollector(PreparedStatementData &data)
+    : PhysicalOperator(PhysicalOperatorType::RESULT_COLLECTOR, {LogicalType::BOOLEAN}, 0),
+      statement_type(data.statement_type), properties(data.properties), plan(data.plan.get()), names(data.names) {
+	this->types = data.types;
+}
+
+unique_ptr<PhysicalResultCollector> PhysicalResultCollector::GetResultCollector(ClientContext &context,
+                                                                                PreparedStatementData &data) {
+	auto &config = DBConfig::GetConfig(context);
+	bool use_materialized_collector = !config.preserve_insertion_order || !data.plan->AllSourcesSupportBatchIndex();
+	if (use_materialized_collector) {
+		// parallel materialized collector only if we don't care about maintaining insertion order
+		return make_unique_base<PhysicalResultCollector, PhysicalMaterializedCollector>(
+		    data, !config.preserve_insertion_order);
+	} else {
+		// we care about maintaining insertion order and the sources all support batch indexes
+		// use a batch collector
+		return make_unique_base<PhysicalResultCollector, PhysicalBatchCollector>(data);
+	}
+}
+
+vector<PhysicalOperator *> PhysicalResultCollector::GetChildren() const {
+	return {plan};
+}
+
+void PhysicalResultCollector::BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) {
+	// operator is a sink, build a pipeline
+	sink_state.reset();
+
+	// single operator:
+	// the operator becomes the data source of the current pipeline
+	state.SetPipelineSource(current, this);
+	// we create a new pipeline starting from the child
+	D_ASSERT(children.size() == 0);
+	D_ASSERT(plan);
+
+	BuildChildPipeline(executor, current, state, plan);
+}
+
+} // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -61633,6 +62172,77 @@ void PhysicalSet::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSou
 }
 
 } // namespace duckdb
+
+
+
+namespace duckdb {
+
+PhysicalStreamingLimit::PhysicalStreamingLimit(vector<LogicalType> types, idx_t limit, idx_t offset,
+                                               unique_ptr<Expression> limit_expression,
+                                               unique_ptr<Expression> offset_expression, idx_t estimated_cardinality,
+                                               bool parallel)
+    : PhysicalOperator(PhysicalOperatorType::STREAMING_LIMIT, move(types), estimated_cardinality), limit_value(limit),
+      offset_value(offset), limit_expression(move(limit_expression)), offset_expression(move(offset_expression)),
+      parallel(parallel) {
+}
+
+//===--------------------------------------------------------------------===//
+// Operator
+//===--------------------------------------------------------------------===//
+class StreamingLimitOperatorState : public OperatorState {
+public:
+	explicit StreamingLimitOperatorState(const PhysicalStreamingLimit &op) {
+		this->limit = op.limit_expression ? DConstants::INVALID_INDEX : op.limit_value;
+		this->offset = op.offset_expression ? DConstants::INVALID_INDEX : op.offset_value;
+	}
+
+	idx_t limit;
+	idx_t offset;
+};
+
+class StreamingLimitGlobalState : public GlobalOperatorState {
+public:
+	StreamingLimitGlobalState() : current_offset(0) {
+	}
+
+	std::atomic<idx_t> current_offset;
+};
+
+unique_ptr<OperatorState> PhysicalStreamingLimit::GetOperatorState(ClientContext &context) const {
+	return make_unique<StreamingLimitOperatorState>(*this);
+}
+
+unique_ptr<GlobalOperatorState> PhysicalStreamingLimit::GetGlobalOperatorState(ClientContext &context) const {
+	return make_unique<StreamingLimitGlobalState>();
+}
+
+OperatorResultType PhysicalStreamingLimit::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
+                                                   GlobalOperatorState &gstate_p, OperatorState &state_p) const {
+	auto &gstate = (StreamingLimitGlobalState &)gstate_p;
+	auto &state = (StreamingLimitOperatorState &)state_p;
+	auto &limit = state.limit;
+	auto &offset = state.offset;
+	idx_t current_offset = gstate.current_offset.fetch_add(input.size());
+	idx_t max_element;
+	if (!PhysicalLimit::ComputeOffset(input, limit, offset, current_offset, max_element, limit_expression.get(),
+	                                  offset_expression.get())) {
+		return OperatorResultType::FINISHED;
+	}
+	if (PhysicalLimit::HandleOffset(input, current_offset, offset, limit)) {
+		chunk.Reference(input);
+	}
+	return OperatorResultType::NEED_MORE_INPUT;
+}
+
+bool PhysicalStreamingLimit::IsOrderDependent() const {
+	return !parallel;
+}
+
+bool PhysicalStreamingLimit::ParallelOperator() const {
+	return parallel;
+}
+
+} // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -62002,6 +62612,12 @@ public:
 	static void ConstructMarkJoinResult(DataChunk &join_keys, DataChunk &left, DataChunk &result, bool found_match[],
 	                                    bool has_null);
 	static void ConstructLeftJoinResult(DataChunk &left, DataChunk &result, bool found_match[]);
+
+public:
+	static void BuildJoinPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state,
+	                               PhysicalOperator &op);
+	void BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) override;
+	vector<const PhysicalOperator *> GetSources() const override;
 };
 
 } // namespace duckdb
@@ -62819,6 +63435,10 @@ public:
 	bool ParallelSink() const override {
 		return true;
 	}
+
+public:
+	void BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) override;
+	vector<const PhysicalOperator *> GetSources() const override;
 };
 
 } // namespace duckdb
@@ -62826,6 +63446,7 @@ public:
 
 
 
+
 namespace duckdb {
 
 PhysicalCrossProduct::PhysicalCrossProduct(vector<LogicalType> types, unique_ptr<PhysicalOperator> left,
@@ -62912,6 +63533,17 @@ OperatorResultType PhysicalCrossProduct::Execute(ExecutionContext &context, Data
 	return OperatorResultType::HAVE_MORE_OUTPUT;
 }
 
+//===--------------------------------------------------------------------===//
+// Pipeline Construction
+//===--------------------------------------------------------------------===//
+void PhysicalCrossProduct::BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) {
+	PhysicalJoin::BuildJoinPipelines(executor, current, state, *this);
+}
+
+vector<const PhysicalOperator *> PhysicalCrossProduct::GetSources() const {
+	return children[0]->GetSources();
+}
+
 } // namespace duckdb
 
 
@@ -62947,6 +63579,66 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+
+public:
+	void BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) override;
+};
+
+} // namespace duckdb
+
+
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/set/physical_recursive_cte.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+namespace duckdb {
+class Pipeline;
+class RecursiveCTEState;
+
+class PhysicalRecursiveCTE : public PhysicalOperator {
+public:
+	PhysicalRecursiveCTE(vector<LogicalType> types, bool union_all, unique_ptr<PhysicalOperator> top,
+	                     unique_ptr<PhysicalOperator> bottom, idx_t estimated_cardinality);
+	~PhysicalRecursiveCTE() override;
+
+	bool union_all;
+	std::shared_ptr<ChunkCollection> working_table;
+	vector<shared_ptr<Pipeline>> pipelines;
+
+public:
+	// Source interface
+	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
+	             LocalSourceState &lstate) const override;
+
+public:
+	// Sink interface
+	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
+	                    DataChunk &input) const override;
+
+	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
+
+	bool IsSink() const override {
+		return true;
+	}
+
+public:
+	void BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) override;
+
+	vector<const PhysicalOperator *> GetSources() const override;
+
+private:
+	//! Probe Hash Table and eliminate duplicate rows
+	idx_t ProbeHT(DataChunk &chunk, RecursiveCTEState &state) const;
+
+	void ExecuteRecursivePipelines(ExecutionContext &context) const;
 };
 
 } // namespace duckdb
@@ -62972,6 +63664,16 @@ PhysicalDelimJoin::PhysicalDelimJoin(vector<LogicalType> types, unique_ptr<Physi
 	join->children[0] = move(cached_chunk_scan);
 }
 
+vector<PhysicalOperator *> PhysicalDelimJoin::GetChildren() const {
+	vector<PhysicalOperator *> result;
+	for (auto &child : children) {
+		result.push_back(child.get());
+	}
+	result.push_back(join.get());
+	result.push_back(distinct.get());
+	return result;
+}
+
 //===--------------------------------------------------------------------===//
 // Sink
 //===--------------------------------------------------------------------===//
@@ -63045,6 +63747,40 @@ string PhysicalDelimJoin::ParamsToString() const {
 	return join->ParamsToString();
 }
 
+//===--------------------------------------------------------------------===//
+// Pipeline Construction
+//===--------------------------------------------------------------------===//
+void PhysicalDelimJoin::BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) {
+	op_state.reset();
+	sink_state.reset();
+
+	// duplicate eliminated join
+	auto pipeline = make_shared<Pipeline>(executor);
+	state.SetPipelineSink(*pipeline, this);
+	current.AddDependency(pipeline);
+
+	// recurse into the pipeline child
+	children[0]->BuildPipelines(executor, *pipeline, state);
+	if (type == PhysicalOperatorType::DELIM_JOIN) {
+		// recurse into the actual join
+		// any pipelines in there depend on the main pipeline
+		// any scan of the duplicate eliminated data on the RHS depends on this pipeline
+		// we add an entry to the mapping of (PhysicalOperator*) -> (Pipeline*)
+		for (auto &delim_scan : delim_scans) {
+			state.delim_join_dependencies[delim_scan] = pipeline.get();
+		}
+		join->BuildPipelines(executor, current, state);
+	}
+	if (!state.recursive_cte) {
+		// regular pipeline: schedule it
+		state.AddPipeline(executor, move(pipeline));
+	} else {
+		// CTE pipeline! add it to the CTE pipelines
+		auto &cte = (PhysicalRecursiveCTE &)*state.recursive_cte;
+		cte.pipelines.push_back(move(pipeline));
+	}
+}
+
 } // namespace duckdb
 
 
@@ -63616,6 +64352,9 @@ public:
 		return true;
 	}
 
+public:
+	void BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) override;
+
 private:
 	// resolve joins that can potentially output N*M elements (INNER, LEFT, FULL)
 	void ResolveComplexJoin(ExecutionContext &context, DataChunk &result, LocalSourceState &state) const;
@@ -64725,6 +65464,38 @@ void PhysicalIEJoin::GetData(ExecutionContext &context, DataChunk &result, Globa
 	}
 }
 
+//===--------------------------------------------------------------------===//
+// Pipeline Construction
+//===--------------------------------------------------------------------===//
+void PhysicalIEJoin::BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) {
+	D_ASSERT(children.size() == 2);
+	if (state.recursive_cte) {
+		throw NotImplementedException("IEJoins are not supported in recursive CTEs yet");
+	}
+
+	// Build the LHS
+	auto lhs_pipeline = make_shared<Pipeline>(executor);
+	state.SetPipelineSink(*lhs_pipeline, this);
+	D_ASSERT(children[0].get());
+	children[0]->BuildPipelines(executor, *lhs_pipeline, state);
+
+	// Build the RHS
+	auto rhs_pipeline = make_shared<Pipeline>(executor);
+	state.SetPipelineSink(*rhs_pipeline, this);
+	D_ASSERT(children[1].get());
+	children[1]->BuildPipelines(executor, *rhs_pipeline, state);
+
+	// RHS => LHS => current
+	current.AddDependency(rhs_pipeline);
+	rhs_pipeline->AddDependency(lhs_pipeline);
+
+	state.AddPipeline(executor, move(lhs_pipeline));
+	state.AddPipeline(executor, move(rhs_pipeline));
+
+	// Now build both and scan
+	state.SetPipelineSource(current, this);
+}
+
 } // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB
@@ -64790,6 +65561,10 @@ public:
 		return true;
 	}
 
+public:
+	void BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) override;
+	vector<const PhysicalOperator *> GetSources() const override;
+
 private:
 	void GetRHSMatches(ExecutionContext &context, DataChunk &input, OperatorState &state_p) const;
 	//! Fills result chunk
@@ -64849,10 +65624,16 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
+	idx_t GetBatchIndex(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
+	                    LocalSourceState &lstate) const override;
 
 	bool ParallelSource() const override {
 		return true;
 	}
+
+	bool SupportsBatchIndex() const override {
+		return function.supports_batch_index;
+	}
 };
 
 } // namespace duckdb
@@ -65097,9 +65878,25 @@ OperatorResultType PhysicalIndexJoin::Execute(ExecutionContext &context, DataChu
 	return OperatorResultType::HAVE_MORE_OUTPUT;
 }
 
+//===--------------------------------------------------------------------===//
+// Pipeline Construction
+//===--------------------------------------------------------------------===//
+void PhysicalIndexJoin::BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) {
+	// index join: we only continue into the LHS
+	// the right side is probed by the index join
+	// so we don't need to do anything in the pipeline with this child
+	state.AddPipelineOperator(current, this);
+	children[0]->BuildPipelines(executor, current, state);
+}
+
+vector<const PhysicalOperator *> PhysicalIndexJoin::GetSources() const {
+	return children[0]->GetSources();
+}
+
 } // namespace duckdb
 
 
+
 namespace duckdb {
 
 PhysicalJoin::PhysicalJoin(LogicalOperator &op, PhysicalOperatorType type, JoinType join_type,
@@ -65119,6 +65916,44 @@ bool PhysicalJoin::EmptyResultIfRHSIsEmpty() const {
 	}
 }
 
+//===--------------------------------------------------------------------===//
+// Pipeline Construction
+//===--------------------------------------------------------------------===//
+void PhysicalJoin::BuildJoinPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state,
+                                      PhysicalOperator &op) {
+	op.op_state.reset();
+	op.sink_state.reset();
+
+	// on the LHS (probe child), the operator becomes a regular operator
+	state.AddPipelineOperator(current, &op);
+	if (op.IsSource()) {
+		// FULL or RIGHT outer join
+		// schedule a scan of the node as a child pipeline
+		// this scan has to be performed AFTER all the probing has happened
+		if (state.recursive_cte) {
+			throw NotImplementedException("FULL and RIGHT outer joins are not supported in recursive CTEs yet");
+		}
+		state.AddChildPipeline(executor, current);
+	}
+	// continue building the pipeline on this child
+	op.children[0]->BuildPipelines(executor, current, state);
+
+	// on the RHS (build side), we construct a new child pipeline with this pipeline as its source
+	op.BuildChildPipeline(executor, current, state, op.children[1].get());
+}
+
+void PhysicalJoin::BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) {
+	PhysicalJoin::BuildJoinPipelines(executor, current, state, *this);
+}
+
+vector<const PhysicalOperator *> PhysicalJoin::GetSources() const {
+	auto result = children[0]->GetSources();
+	if (IsSource()) {
+		result.push_back(this);
+	}
+	return result;
+}
+
 } // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB
@@ -70046,6 +70881,10 @@ public:
 	bool IsSink() const override {
 		return true;
 	}
+
+public:
+	void BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) override;
+	vector<const PhysicalOperator *> GetSources() const override;
 };
 
 } // namespace duckdb
@@ -70057,6 +70896,7 @@ public:
 
 
 
+
 #include <algorithm>
 #include <sstream>
 
@@ -70221,6 +71061,23 @@ SinkResultType PhysicalExport::Sink(ExecutionContext &context, GlobalSinkState &
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
+//===--------------------------------------------------------------------===//
+// Pipeline Construction
+//===--------------------------------------------------------------------===//
+void PhysicalExport::BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) {
+	// EXPORT has an optional child
+	// we only need to schedule child pipelines if there is a child
+	state.SetPipelineSource(current, this);
+	if (children.empty()) {
+		return;
+	}
+	PhysicalOperator::BuildPipelines(executor, current, state);
+}
+
+vector<const PhysicalOperator *> PhysicalExport::GetSources() const {
+	return {this};
+}
+
 } // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB
@@ -71159,6 +72016,8 @@ OperatorResultType PhysicalUnnest::Execute(ExecutionContext &context, DataChunk
 } // namespace duckdb
 
 
+
+
 namespace duckdb {
 
 class PhysicalChunkScanState : public GlobalSourceState {
@@ -71190,6 +72049,38 @@ void PhysicalChunkScan::GetData(ExecutionContext &context, DataChunk &chunk, Glo
 	state.chunk_index++;
 }
 
+//===--------------------------------------------------------------------===//
+// Pipeline Construction
+//===--------------------------------------------------------------------===//
+void PhysicalChunkScan::BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) {
+	// check if there is any additional action we need to do depending on the type
+	switch (type) {
+	case PhysicalOperatorType::DELIM_SCAN: {
+		auto entry = state.delim_join_dependencies.find(this);
+		D_ASSERT(entry != state.delim_join_dependencies.end());
+		// this chunk scan introduces a dependency to the current pipeline
+		// namely a dependency on the duplicate elimination pipeline to finish
+		auto delim_dependency = entry->second->shared_from_this();
+		auto delim_sink = state.GetPipelineSink(*delim_dependency);
+		D_ASSERT(delim_sink);
+		D_ASSERT(delim_sink->type == PhysicalOperatorType::DELIM_JOIN);
+		auto &delim_join = (PhysicalDelimJoin &)*delim_sink;
+		current.AddDependency(delim_dependency);
+		state.SetPipelineSource(current, (PhysicalOperator *)delim_join.distinct.get());
+		return;
+	}
+	case PhysicalOperatorType::RECURSIVE_CTE_SCAN:
+		if (!state.recursive_cte) {
+			throw InternalException("Recursive CTE scan found without recursive CTE node");
+		}
+		break;
+	default:
+		break;
+	}
+	D_ASSERT(children.empty());
+	state.SetPipelineSource(current, this);
+}
+
 } // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB
@@ -71499,6 +72390,16 @@ void PhysicalTableScan::GetData(ExecutionContext &context, DataChunk &chunk, Glo
 	}
 }
 
+idx_t PhysicalTableScan::GetBatchIndex(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate_p,
+                                       LocalSourceState &lstate) const {
+	D_ASSERT(SupportsBatchIndex());
+	D_ASSERT(function.get_batch_index);
+	auto &gstate = (TableScanGlobalState &)gstate_p;
+	auto &state = (TableScanLocalState &)lstate;
+	return function.get_batch_index(context.client, bind_data.get(), state.operator_data.get(),
+	                                gstate.parallel_state.get());
+}
+
 string PhysicalTableScan::GetName() const {
 	return StringUtil::Upper(function.name);
 }
@@ -72325,56 +73226,7 @@ void PhysicalDrop::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSo
 }
 
 } // namespace duckdb
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/execution/operator/set/physical_recursive_cte.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-
-
-
-
-namespace duckdb {
-class Pipeline;
-class RecursiveCTEState;
-
-class PhysicalRecursiveCTE : public PhysicalOperator {
-public:
-	PhysicalRecursiveCTE(vector<LogicalType> types, bool union_all, unique_ptr<PhysicalOperator> top,
-	                     unique_ptr<PhysicalOperator> bottom, idx_t estimated_cardinality);
-	~PhysicalRecursiveCTE() override;
 
-	bool union_all;
-	std::shared_ptr<ChunkCollection> working_table;
-	vector<shared_ptr<Pipeline>> pipelines;
-
-public:
-	// Source interface
-	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-	             LocalSourceState &lstate) const override;
-
-public:
-	// Sink interface
-	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
-	                    DataChunk &input) const override;
-
-	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
-
-	bool IsSink() const override {
-		return true;
-	}
-
-private:
-	//! Probe Hash Table and eliminate duplicate rows
-	idx_t ProbeHT(DataChunk &chunk, RecursiveCTEState &state) const;
-
-	void ExecuteRecursivePipelines(ExecutionContext &context) const;
-};
-
-} // namespace duckdb
 
 
 
@@ -72523,6 +73375,40 @@ void PhysicalRecursiveCTE::ExecuteRecursivePipelines(ExecutionContext &context)
 	}
 }
 
+//===--------------------------------------------------------------------===//
+// Pipeline Construction
+//===--------------------------------------------------------------------===//
+void PhysicalRecursiveCTE::BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) {
+	op_state.reset();
+	sink_state.reset();
+
+	// recursive CTE
+	state.SetPipelineSource(current, this);
+	// the LHS of the recursive CTE is our initial state
+	// we build this pipeline as normal
+	auto pipeline_child = children[0].get();
+	// for the RHS, we gather all pipelines that depend on the recursive cte
+	// these pipelines need to be rerun
+	if (state.recursive_cte) {
+		throw InternalException("Recursive CTE detected WITHIN a recursive CTE node");
+	}
+	state.recursive_cte = this;
+
+	auto recursive_pipeline = make_shared<Pipeline>(executor);
+	state.SetPipelineSink(*recursive_pipeline, this);
+	children[1]->BuildPipelines(executor, *recursive_pipeline, state);
+
+	pipelines.push_back(move(recursive_pipeline));
+
+	state.recursive_cte = nullptr;
+
+	BuildChildPipeline(executor, current, state, pipeline_child);
+}
+
+vector<const PhysicalOperator *> PhysicalRecursiveCTE::GetSources() const {
+	return {this};
+}
+
 } // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB
@@ -72541,12 +73427,17 @@ class PhysicalUnion : public PhysicalOperator {
 public:
 	PhysicalUnion(vector<LogicalType> types, unique_ptr<PhysicalOperator> top, unique_ptr<PhysicalOperator> bottom,
 	              idx_t estimated_cardinality);
+
+public:
+	void BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) override;
+	vector<const PhysicalOperator *> GetSources() const override;
 };
 
 } // namespace duckdb
 
 
 
+
 namespace duckdb {
 
 PhysicalUnion::PhysicalUnion(vector<LogicalType> types, unique_ptr<PhysicalOperator> top,
@@ -72556,6 +73447,49 @@ PhysicalUnion::PhysicalUnion(vector<LogicalType> types, unique_ptr<PhysicalOpera
 	children.push_back(move(bottom));
 }
 
+//===--------------------------------------------------------------------===//
+// Pipeline Construction
+//===--------------------------------------------------------------------===//
+void PhysicalUnion::BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) {
+	if (state.recursive_cte) {
+		throw NotImplementedException("UNIONS are not supported in recursive CTEs yet");
+	}
+	op_state.reset();
+	sink_state.reset();
+
+	auto union_pipeline = make_shared<Pipeline>(executor);
+	auto pipeline_ptr = union_pipeline.get();
+	auto &child_pipelines = state.GetChildPipelines(executor);
+	auto &child_dependencies = state.GetChildDependencies(executor);
+	auto &union_pipelines = state.GetUnionPipelines(executor);
+	// set up dependencies for any child pipelines to this union pipeline
+	auto child_entry = child_pipelines.find(&current);
+	if (child_entry != child_pipelines.end()) {
+		for (auto &current_child : child_entry->second) {
+			D_ASSERT(child_dependencies.find(current_child.get()) != child_dependencies.end());
+			child_dependencies[current_child.get()].push_back(pipeline_ptr);
+		}
+	}
+	// for the current pipeline, continue building on the LHS
+	state.SetPipelineOperators(*union_pipeline, state.GetPipelineOperators(current));
+	children[0]->BuildPipelines(executor, current, state);
+	// insert the union pipeline as a union pipeline of the current node
+	union_pipelines[&current].push_back(move(union_pipeline));
+
+	// for the union pipeline, build on the RHS
+	state.SetPipelineSink(*pipeline_ptr, state.GetPipelineSink(current));
+	children[1]->BuildPipelines(executor, *pipeline_ptr, state);
+}
+
+vector<const PhysicalOperator *> PhysicalUnion::GetSources() const {
+	vector<const PhysicalOperator *> result;
+	for (auto &child : children) {
+		auto child_sources = child->GetSources();
+		result.insert(result.end(), child_sources.begin(), child_sources.end());
+	}
+	return result;
+}
+
 } // namespace duckdb
 
 
@@ -73027,6 +73961,7 @@ void PerfectAggregateHashTable::Destroy() {
 
 
 
+
 namespace duckdb {
 
 string PhysicalOperator::GetName() const {
@@ -73044,6 +73979,14 @@ void PhysicalOperator::Print() const {
 }
 // LCOV_EXCL_STOP
 
+vector<PhysicalOperator *> PhysicalOperator::GetChildren() const {
+	vector<PhysicalOperator *> result;
+	for (auto &child : children) {
+		result.push_back(child.get());
+	}
+	return result;
+}
+
 //===--------------------------------------------------------------------===//
 // Operator
 //===--------------------------------------------------------------------===//
@@ -73079,6 +74022,11 @@ void PhysicalOperator::GetData(ExecutionContext &context, DataChunk &chunk, Glob
                                LocalSourceState &lstate) const {
 	throw InternalException("Calling GetData on a node that is not a source!");
 }
+
+idx_t PhysicalOperator::GetBatchIndex(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
+                                      LocalSourceState &lstate) const {
+	throw InternalException("Calling GetBatchIndex on a node that does not support it");
+}
 // LCOV_EXCL_STOP
 
 //===--------------------------------------------------------------------===//
@@ -73107,6 +74055,99 @@ unique_ptr<GlobalSinkState> PhysicalOperator::GetGlobalSinkState(ClientContext &
 	return make_unique<GlobalSinkState>();
 }
 
+//===--------------------------------------------------------------------===//
+// Pipeline Construction
+//===--------------------------------------------------------------------===//
+void PhysicalOperator::AddPipeline(Executor &executor, shared_ptr<Pipeline> pipeline, PipelineBuildState &state) {
+	if (!state.recursive_cte) {
+		// regular pipeline: schedule it
+		state.AddPipeline(executor, move(pipeline));
+	} else {
+		// CTE pipeline! add it to the CTE pipelines
+		auto &cte = (PhysicalRecursiveCTE &)*state.recursive_cte;
+		cte.pipelines.push_back(move(pipeline));
+	}
+}
+
+void PhysicalOperator::BuildChildPipeline(Executor &executor, Pipeline &current, PipelineBuildState &state,
+                                          PhysicalOperator *pipeline_child) {
+	auto pipeline = make_shared<Pipeline>(executor);
+	state.SetPipelineSink(*pipeline, this);
+	// the current is dependent on this pipeline to complete
+	current.AddDependency(pipeline);
+	// recurse into the pipeline child
+	pipeline_child->BuildPipelines(executor, *pipeline, state);
+	AddPipeline(executor, move(pipeline), state);
+}
+
+void PhysicalOperator::BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) {
+	op_state.reset();
+	if (IsSink()) {
+		// operator is a sink, build a pipeline
+		sink_state.reset();
+
+		// single operator:
+		// the operator becomes the data source of the current pipeline
+		state.SetPipelineSource(current, this);
+		// we create a new pipeline starting from the child
+		D_ASSERT(children.size() == 1);
+
+		BuildChildPipeline(executor, current, state, children[0].get());
+	} else {
+		// operator is not a sink! recurse in children
+		if (children.empty()) {
+			// source
+			state.SetPipelineSource(current, this);
+		} else {
+			if (children.size() != 1) {
+				throw InternalException("Operator not supported in BuildPipelines");
+			}
+			state.AddPipelineOperator(current, this);
+			children[0]->BuildPipelines(executor, current, state);
+		}
+	}
+}
+
+vector<const PhysicalOperator *> PhysicalOperator::GetSources() const {
+	vector<const PhysicalOperator *> result;
+	if (IsSink()) {
+		D_ASSERT(children.size() == 1);
+		result.push_back(this);
+		return result;
+	} else {
+		if (children.empty()) {
+			// source
+			result.push_back(this);
+			return result;
+		} else {
+			if (children.size() != 1) {
+				throw InternalException("Operator not supported in GetSource");
+			}
+			return children[0]->GetSources();
+		}
+	}
+}
+
+bool PhysicalOperator::AllSourcesSupportBatchIndex() const {
+	auto sources = GetSources();
+	for (auto &source : sources) {
+		if (!source->SupportsBatchIndex()) {
+			return false;
+		}
+	}
+	return true;
+}
+
+void PhysicalOperator::Verify() {
+#ifdef DEBUG
+	auto sources = GetSources();
+	D_ASSERT(!sources.empty());
+	for (auto &child : children) {
+		child->Verify();
+	}
+#endif
+}
+
 } // namespace duckdb
 
 //===----------------------------------------------------------------------===//
@@ -75721,6 +76762,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalInsert &op
 } // namespace duckdb
 
 
+
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -75759,17 +76801,36 @@ protected:
 } // namespace duckdb
 
 
+
 namespace duckdb {
 
 unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalLimit &op) {
 	D_ASSERT(op.children.size() == 1);
 
 	auto plan = CreatePlan(*op.children[0]);
+	auto &config = DBConfig::GetConfig(context);
+	unique_ptr<PhysicalOperator> limit;
+	if (!config.preserve_insertion_order) {
+		// use parallel streaming limit if insertion order is not important
+		limit = make_unique<PhysicalStreamingLimit>(op.types, (idx_t)op.limit_val, op.offset_val, move(op.limit),
+		                                            move(op.offset), op.estimated_cardinality, true);
+	} else {
+		// maintaining insertion order is important
+		bool all_sources_support_batch_index = plan->AllSourcesSupportBatchIndex();
+
+		if (all_sources_support_batch_index) {
+			// source supports batch index: use parallel batch limit
+			limit = make_unique<PhysicalLimit>(op.types, (idx_t)op.limit_val, op.offset_val, move(op.limit),
+			                                   move(op.offset), op.estimated_cardinality);
+		} else {
+			// source does not support batch index: use a non-parallel streaming limit
+			limit = make_unique<PhysicalStreamingLimit>(op.types, (idx_t)op.limit_val, op.offset_val, move(op.limit),
+			                                            move(op.offset), op.estimated_cardinality, false);
+		}
+	}
 
-	auto limit = make_unique<PhysicalLimit>(op.types, (idx_t)op.limit_val, op.offset_val, move(op.limit),
-	                                        move(op.offset), op.estimated_cardinality);
 	limit->children.push_back(move(plan));
-	return move(limit);
+	return limit;
 }
 
 } // namespace duckdb
@@ -76877,6 +77938,8 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(unique_ptr<Logica
 	profiler.StartPhase("create_plan");
 	auto plan = CreatePlan(*op);
 	profiler.EndPhase();
+
+	plan->Verify();
 	return plan;
 }
 
@@ -109047,6 +110110,19 @@ double TableScanProgress(ClientContext &context, const FunctionData *bind_data_p
 	return percentage;
 }
 
+idx_t TableScanGetBatchIndex(ClientContext &context, const FunctionData *bind_data_p,
+                             FunctionOperatorData *operator_state, ParallelState *parallel_state_p) {
+	auto &bind_data = (const TableScanBindData &)*bind_data_p;
+	auto &state = (TableScanOperatorData &)*operator_state;
+	if (state.scan_state.row_group_scan_state.row_group) {
+		return state.scan_state.row_group_scan_state.row_group->start;
+	}
+	if (state.scan_state.local_state.max_index > 0) {
+		return bind_data.table->storage->GetTotalRows() + state.scan_state.local_state.chunk_index;
+	}
+	return 0;
+}
+
 void TableScanDependency(unordered_set<CatalogEntry *> &entries, const FunctionData *bind_data_p) {
 	auto &bind_data = (const TableScanBindData &)*bind_data_p;
 	entries.insert(bind_data.table);
@@ -109251,7 +110327,9 @@ void TableScanPushdownComplexFilter(ClientContext &context, LogicalGet &get, Fun
 				get.function.init_parallel_state = nullptr;
 				get.function.parallel_state_next = nullptr;
 				get.function.table_scan_progress = nullptr;
+				get.function.get_batch_index = nullptr;
 				get.function.filter_pushdown = false;
+				get.function.supports_batch_index = false;
 			} else {
 				bind_data.result_ids.clear();
 			}
@@ -109280,8 +110358,10 @@ TableFunction TableScanFunction::GetFunction() {
 	scan_function.parallel_init = TableScanParallelInit;
 	scan_function.parallel_state_next = TableScanParallelStateNext;
 	scan_function.table_scan_progress = TableScanProgress;
+	scan_function.get_batch_index = TableScanGetBatchIndex;
 	scan_function.projection_pushdown = true;
 	scan_function.filter_pushdown = true;
+	scan_function.supports_batch_index = true;
 	return scan_function;
 }
 
@@ -109474,7 +110554,7 @@ TableFunction::TableFunction(string name, vector<LogicalType> arguments, table_f
       cardinality(cardinality), pushdown_complex_filter(pushdown_complex_filter), to_string(to_string),
       max_threads(max_threads), init_parallel_state(init_parallel_state), parallel_function(parallel_function),
       parallel_init(parallel_init), parallel_state_next(parallel_state_next), table_scan_progress(query_progress),
-      projection_pushdown(projection_pushdown), filter_pushdown(filter_pushdown) {
+      projection_pushdown(projection_pushdown), filter_pushdown(filter_pushdown), supports_batch_index(false) {
 }
 
 TableFunction::TableFunction(const vector<LogicalType> &arguments, table_function_t function,
@@ -113024,6 +114104,7 @@ private:
 
 
 
+
 namespace duckdb {
 
 struct ActiveQueryContext {
@@ -113189,14 +114270,15 @@ const string &ClientContext::GetCurrentQuery() {
 	return active_query->query;
 }
 
-unique_ptr<QueryResult> ClientContext::FetchResultInternal(ClientContextLock &lock, PendingQueryResult &pending,
-                                                           bool allow_stream_result) {
+unique_ptr<QueryResult> ClientContext::FetchResultInternal(ClientContextLock &lock, PendingQueryResult &pending) {
 	D_ASSERT(active_query);
 	D_ASSERT(active_query->open_result == &pending);
 	D_ASSERT(active_query->prepared);
+	auto &executor = GetExecutor();
 	auto &prepared = *active_query->prepared;
-	bool create_stream_result = prepared.properties.allow_stream_result && allow_stream_result;
+	bool create_stream_result = prepared.properties.allow_stream_result && pending.allow_stream_result;
 	if (create_stream_result) {
+		D_ASSERT(!executor.HasResultCollector());
 		active_query->progress_bar.reset();
 		query_progress = -1;
 
@@ -113207,25 +114289,32 @@ unique_ptr<QueryResult> ClientContext::FetchResultInternal(ClientContextLock &lo
 		active_query->open_result = stream_result.get();
 		return move(stream_result);
 	}
-	// create a materialized result by continuously fetching
-	auto result = make_unique<MaterializedQueryResult>(pending.statement_type, pending.properties, pending.types,
-	                                                   pending.names, shared_from_this());
-	result->properties = pending.properties;
-	while (true) {
-		auto chunk = FetchInternal(lock, GetExecutor(), *result);
-		if (!chunk || chunk->size() == 0) {
-			break;
-		}
+	unique_ptr<QueryResult> result;
+	if (executor.HasResultCollector()) {
+		// we have a result collector - fetch the result directly from the result collector
+		result = executor.GetResult();
+		CleanupInternal(lock, result.get(), false);
+	} else {
+		// no result collector - create a materialized result by continuously fetching
+		auto materialized_result = make_unique<MaterializedQueryResult>(
+		    pending.statement_type, pending.properties, pending.types, pending.names, shared_from_this());
+		while (true) {
+			auto chunk = FetchInternal(lock, GetExecutor(), *materialized_result);
+			if (!chunk || chunk->size() == 0) {
+				break;
+			}
 #ifdef DEBUG
-		for (idx_t i = 0; i < chunk->ColumnCount(); i++) {
-			if (pending.types[i].id() == LogicalTypeId::VARCHAR) {
-				chunk->data[i].UTFVerify(chunk->size());
+			for (idx_t i = 0; i < chunk->ColumnCount(); i++) {
+				if (pending.types[i].id() == LogicalTypeId::VARCHAR) {
+					chunk->data[i].UTFVerify(chunk->size());
+				}
 			}
-		}
 #endif
-		result->collection.Append(*chunk);
+			materialized_result->collection.Append(*chunk);
+		}
+		result = move(materialized_result);
 	}
-	return move(result);
+	return result;
 }
 
 shared_ptr<PreparedStatementData> ClientContext::CreatePreparedStatement(ClientContextLock &lock, const string &query,
@@ -113288,7 +114377,7 @@ double ClientContext::GetProgress() {
 
 unique_ptr<PendingQueryResult> ClientContext::PendingPreparedStatement(ClientContextLock &lock,
                                                                        shared_ptr<PreparedStatementData> statement_p,
-                                                                       vector<Value> bound_values) {
+                                                                       PendingQueryParameters parameters) {
 	D_ASSERT(active_query);
 	auto &statement = *statement_p;
 	if (ActiveTransaction().IsInvalidated() && statement.properties.requires_valid_transaction) {
@@ -113301,7 +114390,7 @@ unique_ptr<PendingQueryResult> ClientContext::PendingPreparedStatement(ClientCon
 	}
 
 	// bind the bound values before execution
-	statement.Bind(move(bound_values));
+	statement.Bind(parameters.parameters ? *parameters.parameters : vector<Value>());
 
 	active_query->executor = make_unique<Executor>(*this);
 	auto &executor = *active_query->executor;
@@ -113310,12 +114399,23 @@ unique_ptr<PendingQueryResult> ClientContext::PendingPreparedStatement(ClientCon
 		active_query->progress_bar->Start();
 		query_progress = 0;
 	}
-	executor.Initialize(statement.plan.get());
+	auto stream_result = parameters.allow_stream_result && statement.properties.allow_stream_result;
+	if (!stream_result && statement.properties.return_type == StatementReturnType::QUERY_RESULT) {
+		unique_ptr<PhysicalResultCollector> collector;
+		auto &config = ClientConfig::GetConfig(*this);
+		auto get_method =
+		    config.result_collector ? config.result_collector : PhysicalResultCollector::GetResultCollector;
+		collector = get_method(*this, statement);
+		D_ASSERT(collector->type == PhysicalOperatorType::RESULT_COLLECTOR);
+		executor.Initialize(move(collector));
+	} else {
+		executor.Initialize(statement.plan.get());
+	}
 	auto types = executor.GetTypes();
 	D_ASSERT(types == statement.types);
 	D_ASSERT(!active_query->open_result);
 
-	auto pending_result = make_unique<PendingQueryResult>(shared_from_this(), *statement_p, move(types));
+	auto pending_result = make_unique<PendingQueryResult>(shared_from_this(), *statement_p, move(types), stream_result);
 	active_query->prepared = move(statement_p);
 	active_query->open_result = pending_result.get();
 	return pending_result;
@@ -113443,49 +114543,59 @@ unique_ptr<PreparedStatement> ClientContext::Prepare(const string &query) {
 
 unique_ptr<PendingQueryResult> ClientContext::PendingQueryPreparedInternal(ClientContextLock &lock, const string &query,
                                                                            shared_ptr<PreparedStatementData> &prepared,
-                                                                           vector<Value> &values) {
+                                                                           PendingQueryParameters parameters) {
 	try {
 		InitialCleanup(lock);
 	} catch (std::exception &ex) {
 		return make_unique<PendingQueryResult>(ex.what());
 	}
-	return PendingStatementOrPreparedStatementInternal(lock, query, nullptr, prepared, &values);
+	return PendingStatementOrPreparedStatementInternal(lock, query, nullptr, prepared, parameters);
 }
 
-unique_ptr<PendingQueryResult>
-ClientContext::PendingQuery(const string &query, shared_ptr<PreparedStatementData> &prepared, vector<Value> &values) {
+unique_ptr<PendingQueryResult> ClientContext::PendingQuery(const string &query,
+                                                           shared_ptr<PreparedStatementData> &prepared,
+                                                           PendingQueryParameters parameters) {
 	auto lock = LockContext();
-	return PendingQueryPreparedInternal(*lock, query, prepared, values);
+	return PendingQueryPreparedInternal(*lock, query, prepared, parameters);
 }
 
 unique_ptr<QueryResult> ClientContext::Execute(const string &query, shared_ptr<PreparedStatementData> &prepared,
-                                               vector<Value> &values, bool allow_stream_result) {
+                                               PendingQueryParameters parameters) {
 	auto lock = LockContext();
-	auto pending = PendingQueryPreparedInternal(*lock, query, prepared, values);
+	auto pending = PendingQueryPreparedInternal(*lock, query, prepared, parameters);
 	if (!pending->success) {
 		return make_unique<MaterializedQueryResult>(pending->error);
 	}
-	return pending->ExecuteInternal(*lock, allow_stream_result);
+	return pending->ExecuteInternal(*lock);
+}
+
+unique_ptr<QueryResult> ClientContext::Execute(const string &query, shared_ptr<PreparedStatementData> &prepared,
+                                               vector<Value> &values, bool allow_stream_result) {
+	PendingQueryParameters parameters;
+	parameters.parameters = &values;
+	parameters.allow_stream_result = allow_stream_result;
+	return Execute(query, prepared, parameters);
 }
 
 unique_ptr<PendingQueryResult> ClientContext::PendingStatementInternal(ClientContextLock &lock, const string &query,
-                                                                       unique_ptr<SQLStatement> statement) {
+                                                                       unique_ptr<SQLStatement> statement,
+                                                                       PendingQueryParameters parameters) {
 	// prepare the query for execution
 	auto prepared = CreatePreparedStatement(lock, query, move(statement));
-	// by default, no values are bound
-	vector<Value> bound_values;
 	// execute the prepared statement
-	return PendingPreparedStatement(lock, move(prepared), move(bound_values));
+	return PendingPreparedStatement(lock, move(prepared), parameters);
 }
 
 unique_ptr<QueryResult> ClientContext::RunStatementInternal(ClientContextLock &lock, const string &query,
                                                             unique_ptr<SQLStatement> statement,
                                                             bool allow_stream_result, bool verify) {
-	auto pending = PendingQueryInternal(lock, move(statement), verify);
+	PendingQueryParameters parameters;
+	parameters.allow_stream_result = allow_stream_result;
+	auto pending = PendingQueryInternal(lock, move(statement), parameters, verify);
 	if (!pending->success) {
 		return make_unique<MaterializedQueryResult>(move(pending->error));
 	}
-	return ExecutePendingQueryInternal(lock, *pending, allow_stream_result);
+	return ExecutePendingQueryInternal(lock, *pending);
 }
 
 bool ClientContext::IsActiveResult(ClientContextLock &lock, BaseQueryResult *result) {
@@ -113508,7 +114618,7 @@ static bool IsExplainAnalyze(SQLStatement *statement) {
 
 unique_ptr<PendingQueryResult> ClientContext::PendingStatementOrPreparedStatementInternal(
     ClientContextLock &lock, const string &query, unique_ptr<SQLStatement> statement,
-    shared_ptr<PreparedStatementData> &prepared, vector<Value> *values) {
+    shared_ptr<PreparedStatementData> &prepared, PendingQueryParameters parameters) {
 	// check if we are on AutoCommit. In this case we should start a transaction.
 	if (statement && config.query_verification_enabled) {
 		// query verification is enabled
@@ -113545,13 +114655,12 @@ unique_ptr<PendingQueryResult> ClientContext::PendingStatementOrPreparedStatemen
 			break;
 		}
 	}
-	return PendingStatementOrPreparedStatement(lock, query, move(statement), prepared, values);
+	return PendingStatementOrPreparedStatement(lock, query, move(statement), prepared, parameters);
 }
 
-unique_ptr<PendingQueryResult>
-ClientContext::PendingStatementOrPreparedStatement(ClientContextLock &lock, const string &query,
-                                                   unique_ptr<SQLStatement> statement,
-                                                   shared_ptr<PreparedStatementData> &prepared, vector<Value> *values) {
+unique_ptr<PendingQueryResult> ClientContext::PendingStatementOrPreparedStatement(
+    ClientContextLock &lock, const string &query, unique_ptr<SQLStatement> statement,
+    shared_ptr<PreparedStatementData> &prepared, PendingQueryParameters parameters) {
 	unique_ptr<PendingQueryResult> result;
 
 	BeginQueryInternal(lock, query);
@@ -113561,13 +114670,14 @@ ClientContext::PendingStatementOrPreparedStatement(ClientContextLock &lock, cons
 	bool invalidate_query = true;
 	try {
 		if (statement) {
-			result = PendingStatementInternal(lock, query, move(statement));
+			result = PendingStatementInternal(lock, query, move(statement), parameters);
 		} else {
 			auto &catalog = Catalog::GetCatalog(*this);
 			if (prepared->unbound_statement && (catalog.GetCatalogVersion() != prepared->catalog_version ||
 			                                    !prepared->properties.bound_all_parameters)) {
 				// catalog was modified: rebind the statement before execution
-				auto new_prepared = CreatePreparedStatement(lock, query, prepared->unbound_statement->Copy(), values);
+				auto new_prepared =
+				    CreatePreparedStatement(lock, query, prepared->unbound_statement->Copy(), parameters.parameters);
 				if (prepared->types != new_prepared->types && prepared->properties.bound_all_parameters) {
 					throw BinderException("Rebinding statement after catalog change resulted in change of types");
 				}
@@ -113576,7 +114686,7 @@ ClientContext::PendingStatementOrPreparedStatement(ClientContextLock &lock, cons
 				prepared = move(new_prepared);
 				prepared->properties.bound_all_parameters = false;
 			}
-			result = PendingPreparedStatement(lock, prepared, *values);
+			result = PendingPreparedStatement(lock, prepared, parameters);
 		}
 	} catch (StandardException &ex) {
 		// standard exceptions do not invalidate the current transaction
@@ -113618,8 +114728,8 @@ void ClientContext::LogQueryInternal(ClientContextLock &, const string &query) {
 }
 
 unique_ptr<QueryResult> ClientContext::Query(unique_ptr<SQLStatement> statement, bool allow_stream_result) {
-	auto pending_query = PendingQuery(move(statement));
-	return pending_query->Execute(allow_stream_result);
+	auto pending_query = PendingQuery(move(statement), allow_stream_result);
+	return pending_query->Execute();
 }
 
 unique_ptr<QueryResult> ClientContext::Query(const string &query, bool allow_stream_result) {
@@ -113644,13 +114754,14 @@ unique_ptr<QueryResult> ClientContext::Query(const string &query, bool allow_str
 	for (idx_t i = 0; i < statements.size(); i++) {
 		auto &statement = statements[i];
 		bool is_last_statement = i + 1 == statements.size();
-		bool stream_result = allow_stream_result && is_last_statement;
-		auto pending_query = PendingQueryInternal(*lock, move(statement));
+		PendingQueryParameters parameters;
+		parameters.allow_stream_result = allow_stream_result && is_last_statement;
+		auto pending_query = PendingQueryInternal(*lock, move(statement), parameters);
 		unique_ptr<QueryResult> current_result;
 		if (!pending_query->success) {
 			current_result = make_unique<MaterializedQueryResult>(pending_query->error);
 		} else {
-			current_result = ExecutePendingQueryInternal(*lock, *pending_query, stream_result);
+			current_result = ExecutePendingQueryInternal(*lock, *pending_query);
 		}
 		// now append the result to the list of results
 		if (!last_result) {
@@ -113679,7 +114790,7 @@ bool ClientContext::ParseStatements(ClientContextLock &lock, const string &query
 	}
 }
 
-unique_ptr<PendingQueryResult> ClientContext::PendingQuery(const string &query) {
+unique_ptr<PendingQueryResult> ClientContext::PendingQuery(const string &query, bool allow_stream_result) {
 	auto lock = LockContext();
 
 	string error;
@@ -113690,28 +114801,33 @@ unique_ptr<PendingQueryResult> ClientContext::PendingQuery(const string &query)
 	if (statements.size() != 1) {
 		return make_unique<PendingQueryResult>("PendingQuery can only take a single statement");
 	}
-	return PendingQueryInternal(*lock, move(statements[0]));
+	PendingQueryParameters parameters;
+	parameters.allow_stream_result = allow_stream_result;
+	return PendingQueryInternal(*lock, move(statements[0]), parameters);
 }
 
-unique_ptr<PendingQueryResult> ClientContext::PendingQuery(unique_ptr<SQLStatement> statement) {
+unique_ptr<PendingQueryResult> ClientContext::PendingQuery(unique_ptr<SQLStatement> statement,
+                                                           bool allow_stream_result) {
 	auto lock = LockContext();
-	return PendingQueryInternal(*lock, move(statement));
+	PendingQueryParameters parameters;
+	parameters.allow_stream_result = allow_stream_result;
+	return PendingQueryInternal(*lock, move(statement), parameters);
 }
 
 unique_ptr<PendingQueryResult> ClientContext::PendingQueryInternal(ClientContextLock &lock,
-                                                                   unique_ptr<SQLStatement> statement, bool verify) {
+                                                                   unique_ptr<SQLStatement> statement,
+                                                                   PendingQueryParameters parameters, bool verify) {
 	auto query = statement->query;
 	shared_ptr<PreparedStatementData> prepared;
 	if (verify) {
-		return PendingStatementOrPreparedStatementInternal(lock, query, move(statement), prepared, nullptr);
+		return PendingStatementOrPreparedStatementInternal(lock, query, move(statement), prepared, parameters);
 	} else {
-		return PendingStatementOrPreparedStatement(lock, query, move(statement), prepared, nullptr);
+		return PendingStatementOrPreparedStatement(lock, query, move(statement), prepared, parameters);
 	}
 }
 
-unique_ptr<QueryResult> ClientContext::ExecutePendingQueryInternal(ClientContextLock &lock, PendingQueryResult &query,
-                                                                   bool allow_stream_result) {
-	return query.ExecuteInternal(lock, allow_stream_result);
+unique_ptr<QueryResult> ClientContext::ExecutePendingQueryInternal(ClientContextLock &lock, PendingQueryResult &query) {
+	return query.ExecuteInternal(lock);
 }
 
 void ClientContext::Interrupt() {
@@ -114394,6 +115510,16 @@ struct PreserveIdentifierCase {
 	static Value GetSetting(ClientContext &context);
 };
 
+struct PreserveInsertionOrder {
+	static constexpr const char *Name = "preserve_insertion_order";
+	static constexpr const char *Description =
+	    "Whether or not to preserve insertion order. If set to false the system is allowed to re-order any results "
+	    "that do not contain ORDER BY clauses.";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static Value GetSetting(ClientContext &context);
+};
+
 struct ProfilerHistorySize {
 	static constexpr const char *Name = "profiler_history_size";
 	static constexpr const char *Description = "Sets the profiler history size";
@@ -114509,6 +115635,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_GLOBAL_ALIAS("null_order", DefaultNullOrderSetting),
                                                  DUCKDB_LOCAL(PerfectHashThresholdSetting),
                                                  DUCKDB_LOCAL(PreserveIdentifierCase),
+                                                 DUCKDB_GLOBAL(PreserveInsertionOrder),
                                                  DUCKDB_LOCAL(ProfilerHistorySize),
                                                  DUCKDB_LOCAL(ProfileOutputSetting),
                                                  DUCKDB_LOCAL(ProfilingModeSetting),
@@ -114978,12 +116105,12 @@ unique_ptr<MaterializedQueryResult> Connection::Query(unique_ptr<SQLStatement> s
 	return unique_ptr_cast<QueryResult, MaterializedQueryResult>(move(result));
 }
 
-unique_ptr<PendingQueryResult> Connection::PendingQuery(const string &query) {
-	return context->PendingQuery(query);
+unique_ptr<PendingQueryResult> Connection::PendingQuery(const string &query, bool allow_stream_result) {
+	return context->PendingQuery(query, allow_stream_result);
 }
 
-unique_ptr<PendingQueryResult> Connection::PendingQuery(unique_ptr<SQLStatement> statement) {
-	return context->PendingQuery(move(statement));
+unique_ptr<PendingQueryResult> Connection::PendingQuery(unique_ptr<SQLStatement> statement, bool allow_stream_result) {
+	return context->PendingQuery(move(statement), allow_stream_result);
 }
 
 unique_ptr<PreparedStatement> Connection::Prepare(const string &query) {
@@ -124082,10 +125209,10 @@ unique_ptr<DataChunk> MaterializedQueryResult::FetchRaw() {
 namespace duckdb {
 
 PendingQueryResult::PendingQueryResult(shared_ptr<ClientContext> context_p, PreparedStatementData &statement,
-                                       vector<LogicalType> types_p)
+                                       vector<LogicalType> types_p, bool allow_stream_result)
     : BaseQueryResult(QueryResultType::PENDING_RESULT, statement.statement_type, statement.properties, move(types_p),
                       statement.names),
-      context(move(context_p)) {
+      context(move(context_p)), allow_stream_result(allow_stream_result) {
 }
 
 PendingQueryResult::PendingQueryResult(string error) : BaseQueryResult(QueryResultType::PENDING_RESULT, move(error)) {
@@ -124123,21 +125250,21 @@ PendingExecutionResult PendingQueryResult::ExecuteTaskInternal(ClientContextLock
 	return context->ExecuteTaskInternal(lock, *this);
 }
 
-unique_ptr<QueryResult> PendingQueryResult::ExecuteInternal(ClientContextLock &lock, bool allow_streaming_result) {
+unique_ptr<QueryResult> PendingQueryResult::ExecuteInternal(ClientContextLock &lock) {
 	CheckExecutableInternal(lock);
 	while (ExecuteTaskInternal(lock) == PendingExecutionResult::RESULT_NOT_READY) {
 	}
 	if (!success) {
 		return make_unique<MaterializedQueryResult>(error);
 	}
-	auto result = context->FetchResultInternal(lock, *this, allow_streaming_result);
+	auto result = context->FetchResultInternal(lock, *this);
 	Close();
 	return result;
 }
 
-unique_ptr<QueryResult> PendingQueryResult::Execute(bool allow_streaming_result) {
+unique_ptr<QueryResult> PendingQueryResult::Execute() {
 	auto lock = LockContext();
-	return ExecuteInternal(*lock, allow_streaming_result);
+	return ExecuteInternal(*lock);
 }
 
 void PendingQueryResult::Close() {
@@ -124190,19 +125317,22 @@ const vector<string> &PreparedStatement::GetNames() {
 }
 
 unique_ptr<QueryResult> PreparedStatement::Execute(vector<Value> &values, bool allow_stream_result) {
-	auto pending = PendingQuery(values);
+	auto pending = PendingQuery(values, allow_stream_result);
 	if (!pending->success) {
 		return make_unique<MaterializedQueryResult>(pending->error);
 	}
-	return pending->Execute(allow_stream_result && data->properties.allow_stream_result);
+	return pending->Execute();
 }
 
-unique_ptr<PendingQueryResult> PreparedStatement::PendingQuery(vector<Value> &values) {
+unique_ptr<PendingQueryResult> PreparedStatement::PendingQuery(vector<Value> &values, bool allow_stream_result) {
 	if (!success) {
 		throw InvalidInputException("Attempting to execute an unsuccessfully prepared statement!");
 	}
 	D_ASSERT(data);
-	auto result = context->PendingQuery(query, data, values);
+	PendingQueryParameters parameters;
+	parameters.parameters = &values;
+	parameters.allow_stream_result = allow_stream_result && data->properties.allow_stream_result;
+	auto result = context->PendingQuery(query, data, parameters);
 	return result;
 }
 
@@ -124344,6 +125474,7 @@ bool QueryProfiler::OperatorRequiresProfiling(PhysicalOperatorType op_type) {
 	case PhysicalOperatorType::STREAMING_SAMPLE:
 	case PhysicalOperatorType::LIMIT:
 	case PhysicalOperatorType::LIMIT_PERCENT:
+	case PhysicalOperatorType::STREAMING_LIMIT:
 	case PhysicalOperatorType::TOP_N:
 	case PhysicalOperatorType::WINDOW:
 	case PhysicalOperatorType::UNNEST:
@@ -124847,27 +125978,10 @@ unique_ptr<QueryProfiler::TreeNode> QueryProfiler::CreateTree(PhysicalOperator *
 	node->extra_info = root->ParamsToString();
 	node->depth = depth;
 	tree_map[root] = node.get();
-	for (auto &child : root->children) {
-		auto child_node = CreateTree(child.get(), depth + 1);
-		node->children.push_back(move(child_node));
-	}
-	switch (root->type) {
-	case PhysicalOperatorType::DELIM_JOIN: {
-		auto &delim_join = (PhysicalDelimJoin &)*root;
-		auto child_node = CreateTree((PhysicalOperator *)delim_join.join.get(), depth + 1);
-		node->children.push_back(move(child_node));
-		child_node = CreateTree((PhysicalOperator *)delim_join.distinct.get(), depth + 1);
-		node->children.push_back(move(child_node));
-		break;
-	}
-	case PhysicalOperatorType::EXECUTE: {
-		auto &execute = (PhysicalExecute &)*root;
-		auto child_node = CreateTree((PhysicalOperator *)execute.plan, depth + 1);
+	auto children = root->GetChildren();
+	for (auto &child : children) {
+		auto child_node = CreateTree(child, depth + 1);
 		node->children.push_back(move(child_node));
-		break;
-	}
-	default:
-		break;
 	}
 	return node;
 }
@@ -128174,6 +129288,18 @@ Value PreserveIdentifierCase::GetSetting(ClientContext &context) {
 	return Value::BOOLEAN(ClientConfig::GetConfig(context).preserve_identifier_case);
 }
 
+//===--------------------------------------------------------------------===//
+// PreserveInsertionOrder
+//===--------------------------------------------------------------------===//
+void PreserveInsertionOrder::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	config.preserve_insertion_order = input.GetValue<bool>();
+}
+
+Value PreserveInsertionOrder::GetSetting(ClientContext &context) {
+	auto &config = DBConfig::GetConfig(context);
+	return Value::BOOLEAN(config.preserve_insertion_order);
+}
+
 //===--------------------------------------------------------------------===//
 // Profiler History Size
 //===--------------------------------------------------------------------===//
@@ -137610,11 +138736,6 @@ void Event::SetTasks(vector<unique_ptr<Task>> tasks) {
 
 
 
-
-
-
-
-
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -137709,7 +138830,9 @@ private:
 	//! Whether or not the pipeline has been finalized (used for verification only)
 	bool finalized = false;
 	//! Whether or not the pipeline has finished processing
-	bool finished_processing = false;
+	int32_t finished_processing_idx = -1;
+	//! Whether or not this pipeline requires keeping track of the batch index of the source
+	bool requires_batch_index = false;
 
 	//! Cached chunks for any operators that require caching
 	vector<unique_ptr<DataChunk>> cached_chunks;
@@ -137722,6 +138845,9 @@ private:
 	void GoToSource(idx_t &current_idx, idx_t initial_idx);
 	void FetchFromSource(DataChunk &result);
 
+	void FinishProcessing(int32_t operator_idx = -1);
+	bool IsFinished();
+
 	OperatorResultType ExecutePushInternal(DataChunk &input, idx_t initial_idx = 0);
 	//! Pushes a chunk through the pipeline and returns a single result chunk
 	//! Returns whether or not a new input chunk is needed, or whether or not we are finished
@@ -137822,6 +138948,8 @@ public:
 } // namespace duckdb
 
 
+
+
 #include <algorithm>
 
 namespace duckdb {
@@ -138062,8 +139190,18 @@ void Executor::VerifyPipelines() {
 #endif
 }
 
+void Executor::Initialize(unique_ptr<PhysicalOperator> physical_plan) {
+	Reset();
+	owned_plan = move(physical_plan);
+	InitializeInternal(owned_plan.get());
+}
+
 void Executor::Initialize(PhysicalOperator *plan) {
 	Reset();
+	InitializeInternal(plan);
+}
+
+void Executor::InitializeInternal(PhysicalOperator *plan) {
 
 	auto &scheduler = TaskScheduler::GetScheduler(context);
 	{
@@ -138076,7 +139214,9 @@ void Executor::Initialize(PhysicalOperator *plan) {
 
 		auto root_pipeline = make_shared<Pipeline>(*this);
 		root_pipeline->sink = nullptr;
-		BuildPipelines(physical_plan, root_pipeline.get());
+
+		PipelineBuildState state;
+		physical_plan->BuildPipelines(*this, *root_pipeline, state);
 
 		this->total_pipelines = pipelines.size();
 
@@ -138187,9 +139327,8 @@ PendingExecutionResult Executor::ExecuteTask() {
 
 void Executor::Reset() {
 	lock_guard<mutex> elock(executor_lock);
-	delim_join_dependencies.clear();
-	recursive_cte = nullptr;
 	physical_plan = nullptr;
+	owned_plan.reset();
 	root_executor.reset();
 	root_pipelines.clear();
 	root_pipeline_idx = 0;
@@ -138230,235 +139369,6 @@ void Executor::AddChildPipeline(Pipeline *current) {
 	child_pipelines[current].push_back(move(child_pipeline));
 }
 
-void Executor::BuildPipelines(PhysicalOperator *op, Pipeline *current) {
-	D_ASSERT(current);
-	op->op_state.reset();
-	if (op->IsSink()) {
-		// operator is a sink, build a pipeline
-		op->sink_state.reset();
-
-		PhysicalOperator *pipeline_child = nullptr;
-		switch (op->type) {
-		case PhysicalOperatorType::CREATE_TABLE_AS:
-		case PhysicalOperatorType::INSERT:
-		case PhysicalOperatorType::DELETE_OPERATOR:
-		case PhysicalOperatorType::UPDATE:
-		case PhysicalOperatorType::HASH_GROUP_BY:
-		case PhysicalOperatorType::SIMPLE_AGGREGATE:
-		case PhysicalOperatorType::PERFECT_HASH_GROUP_BY:
-		case PhysicalOperatorType::WINDOW:
-		case PhysicalOperatorType::ORDER_BY:
-		case PhysicalOperatorType::RESERVOIR_SAMPLE:
-		case PhysicalOperatorType::TOP_N:
-		case PhysicalOperatorType::COPY_TO_FILE:
-		case PhysicalOperatorType::LIMIT:
-		case PhysicalOperatorType::LIMIT_PERCENT:
-		case PhysicalOperatorType::EXPLAIN_ANALYZE:
-			D_ASSERT(op->children.size() == 1);
-			// single operator:
-			// the operator becomes the data source of the current pipeline
-			current->source = op;
-			// we create a new pipeline starting from the child
-			pipeline_child = op->children[0].get();
-			break;
-		case PhysicalOperatorType::EXPORT:
-			// EXPORT has an optional child
-			// we only need to schedule child pipelines if there is a child
-			current->source = op;
-			if (op->children.empty()) {
-				return;
-			}
-			D_ASSERT(op->children.size() == 1);
-			pipeline_child = op->children[0].get();
-			break;
-		case PhysicalOperatorType::NESTED_LOOP_JOIN:
-		case PhysicalOperatorType::BLOCKWISE_NL_JOIN:
-		case PhysicalOperatorType::HASH_JOIN:
-		case PhysicalOperatorType::PIECEWISE_MERGE_JOIN:
-		case PhysicalOperatorType::CROSS_PRODUCT:
-			// regular join, create a pipeline with RHS source that sinks into this pipeline
-			pipeline_child = op->children[1].get();
-			// on the LHS (probe child), the operator becomes a regular operator
-			current->operators.push_back(op);
-			if (op->IsSource()) {
-				// FULL or RIGHT outer join
-				// schedule a scan of the node as a child pipeline
-				// this scan has to be performed AFTER all the probing has happened
-				if (recursive_cte) {
-					throw NotImplementedException("FULL and RIGHT outer joins are not supported in recursive CTEs yet");
-				}
-				AddChildPipeline(current);
-			}
-			BuildPipelines(op->children[0].get(), current);
-			break;
-		case PhysicalOperatorType::IE_JOIN: {
-			D_ASSERT(op->children.size() == 2);
-			if (recursive_cte) {
-				throw NotImplementedException("IEJoins are not supported in recursive CTEs yet");
-			}
-
-			// Build the LHS
-			auto lhs_pipeline = make_shared<Pipeline>(*this);
-			lhs_pipeline->sink = op;
-			D_ASSERT(op->children[0].get());
-			BuildPipelines(op->children[0].get(), lhs_pipeline.get());
-
-			// Build the RHS
-			auto rhs_pipeline = make_shared<Pipeline>(*this);
-			rhs_pipeline->sink = op;
-			D_ASSERT(op->children[1].get());
-			BuildPipelines(op->children[1].get(), rhs_pipeline.get());
-
-			// RHS => LHS => current
-			current->AddDependency(rhs_pipeline);
-			rhs_pipeline->AddDependency(lhs_pipeline);
-
-			pipelines.emplace_back(move(lhs_pipeline));
-			pipelines.emplace_back(move(rhs_pipeline));
-
-			// Now build both and scan
-			current->source = op;
-			return;
-		}
-		case PhysicalOperatorType::DELIM_JOIN: {
-			// duplicate eliminated join
-			// for delim joins, recurse into the actual join
-			pipeline_child = op->children[0].get();
-			break;
-		}
-		case PhysicalOperatorType::RECURSIVE_CTE: {
-			auto &cte_node = (PhysicalRecursiveCTE &)*op;
-
-			// recursive CTE
-			current->source = op;
-			// the LHS of the recursive CTE is our initial state
-			// we build this pipeline as normal
-			pipeline_child = op->children[0].get();
-			// for the RHS, we gather all pipelines that depend on the recursive cte
-			// these pipelines need to be rerun
-			if (recursive_cte) {
-				throw InternalException("Recursive CTE detected WITHIN a recursive CTE node");
-			}
-			recursive_cte = op;
-
-			auto recursive_pipeline = make_shared<Pipeline>(*this);
-			recursive_pipeline->sink = op;
-			op->sink_state.reset();
-			BuildPipelines(op->children[1].get(), recursive_pipeline.get());
-
-			cte_node.pipelines.push_back(move(recursive_pipeline));
-
-			recursive_cte = nullptr;
-			break;
-		}
-		default:
-			throw InternalException("Unimplemented sink type!");
-		}
-		// the current is dependent on this pipeline to complete
-		auto pipeline = make_shared<Pipeline>(*this);
-		pipeline->sink = op;
-		current->AddDependency(pipeline);
-		D_ASSERT(pipeline_child);
-		// recurse into the pipeline child
-		BuildPipelines(pipeline_child, pipeline.get());
-		if (op->type == PhysicalOperatorType::DELIM_JOIN) {
-			// for delim joins, recurse into the actual join
-			// any pipelines in there depend on the main pipeline
-			auto &delim_join = (PhysicalDelimJoin &)*op;
-			// any scan of the duplicate eliminated data on the RHS depends on this pipeline
-			// we add an entry to the mapping of (PhysicalOperator*) -> (Pipeline*)
-			for (auto &delim_scan : delim_join.delim_scans) {
-				delim_join_dependencies[delim_scan] = pipeline.get();
-			}
-			BuildPipelines(delim_join.join.get(), current);
-		}
-		if (!recursive_cte) {
-			// regular pipeline: schedule it
-			pipelines.push_back(move(pipeline));
-		} else {
-			// CTE pipeline! add it to the CTE pipelines
-			D_ASSERT(recursive_cte);
-			auto &cte = (PhysicalRecursiveCTE &)*recursive_cte;
-			cte.pipelines.push_back(move(pipeline));
-		}
-	} else {
-		// operator is not a sink! recurse in children
-		// first check if there is any additional action we need to do depending on the type
-		switch (op->type) {
-		case PhysicalOperatorType::DELIM_SCAN: {
-			D_ASSERT(op->children.empty());
-			auto entry = delim_join_dependencies.find(op);
-			D_ASSERT(entry != delim_join_dependencies.end());
-			// this chunk scan introduces a dependency to the current pipeline
-			// namely a dependency on the duplicate elimination pipeline to finish
-			auto delim_dependency = entry->second->shared_from_this();
-			D_ASSERT(delim_dependency->sink->type == PhysicalOperatorType::DELIM_JOIN);
-			auto &delim_join = (PhysicalDelimJoin &)*delim_dependency->sink;
-			current->AddDependency(delim_dependency);
-			current->source = (PhysicalOperator *)delim_join.distinct.get();
-			return;
-		}
-		case PhysicalOperatorType::EXECUTE: {
-			// EXECUTE statement: build pipeline on child
-			auto &execute = (PhysicalExecute &)*op;
-			BuildPipelines(execute.plan, current);
-			return;
-		}
-		case PhysicalOperatorType::RECURSIVE_CTE_SCAN: {
-			if (!recursive_cte) {
-				throw InternalException("Recursive CTE scan found without recursive CTE node");
-			}
-			break;
-		}
-		case PhysicalOperatorType::INDEX_JOIN: {
-			// index join: we only continue into the LHS
-			// the right side is probed by the index join
-			// so we don't need to do anything in the pipeline with this child
-			current->operators.push_back(op);
-			BuildPipelines(op->children[0].get(), current);
-			return;
-		}
-		case PhysicalOperatorType::UNION: {
-			if (recursive_cte) {
-				throw NotImplementedException("UNIONS are not supported in recursive CTEs yet");
-			}
-			auto union_pipeline = make_shared<Pipeline>(*this);
-			auto pipeline_ptr = union_pipeline.get();
-			// set up dependencies for any child pipelines to this union pipeline
-			auto child_entry = child_pipelines.find(current);
-			if (child_entry != child_pipelines.end()) {
-				for (auto &current_child : child_entry->second) {
-					D_ASSERT(child_dependencies.find(current_child.get()) != child_dependencies.end());
-					child_dependencies[current_child.get()].push_back(pipeline_ptr);
-				}
-			}
-			// for the current pipeline, continue building on the LHS
-			union_pipeline->operators = current->operators;
-			BuildPipelines(op->children[0].get(), current);
-			// insert the union pipeline as a union pipeline of the current node
-			union_pipelines[current].push_back(move(union_pipeline));
-
-			// for the union pipeline, build on the RHS
-			pipeline_ptr->sink = current->sink;
-			BuildPipelines(op->children[1].get(), pipeline_ptr);
-			return;
-		}
-		default:
-			break;
-		}
-		if (op->children.empty()) {
-			// source
-			current->source = op;
-		} else {
-			if (op->children.size() != 1) {
-				throw InternalException("Operator not supported yet");
-			}
-			current->operators.push_back(op);
-			BuildPipelines(op->children[0].get(), current);
-		}
-	}
-}
-
 vector<LogicalType> Executor::GetTypes() {
 	D_ASSERT(physical_plan);
 	return physical_plan->GetTypes();
@@ -138526,6 +139436,17 @@ bool Executor::GetPipelinesProgress(double &current_progress) { // LCOV_EXCL_STA
 	}
 } // LCOV_EXCL_STOP
 
+bool Executor::HasResultCollector() {
+	return physical_plan->type == PhysicalOperatorType::RESULT_COLLECTOR;
+}
+
+unique_ptr<QueryResult> Executor::GetResult() {
+	D_ASSERT(HasResultCollector());
+	auto &result_collector = (PhysicalResultCollector &)*physical_plan;
+	D_ASSERT(result_collector.sink_state);
+	return result_collector.GetResult(*result_collector.sink_state);
+}
+
 unique_ptr<DataChunk> Executor::FetchChunk() {
 	D_ASSERT(physical_plan);
 
@@ -138591,10 +139512,6 @@ TaskExecutionResult ExecutorTask::Execute(TaskExecutionMode mode) {
 
 
 
-
-
-
-
 
 namespace duckdb {
 
@@ -138689,6 +139606,7 @@ void Pipeline::ScheduleSequentialTask(shared_ptr<Event> &event) {
 }
 
 bool Pipeline::ScheduleParallel(shared_ptr<Event> &event) {
+	// check if the sink, source and all intermediate operators support parallelism
 	if (!sink->ParallelSink()) {
 		return false;
 	}
@@ -138700,10 +139618,35 @@ bool Pipeline::ScheduleParallel(shared_ptr<Event> &event) {
 			return false;
 		}
 	}
+	if (sink->RequiresBatchIndex()) {
+		if (!source->SupportsBatchIndex()) {
+			throw InternalException(
+			    "Attempting to schedule a pipeline where the sink requires batch index but source does not support it");
+		}
+	}
 	idx_t max_threads = source_state->MaxThreads();
 	return LaunchScanTasks(event, max_threads);
 }
 
+bool Pipeline::IsOrderDependent() const {
+	auto &config = DBConfig::GetConfig(executor.context);
+	if (!config.preserve_insertion_order) {
+		return false;
+	}
+	if (sink && sink->IsOrderDependent()) {
+		return true;
+	}
+	if (source->IsOrderDependent()) {
+		return true;
+	}
+	for (auto &op : operators) {
+		if (op->IsOrderDependent()) {
+			return true;
+		}
+	}
+	return false;
+}
+
 void Pipeline::Schedule(shared_ptr<Event> &event) {
 	D_ASSERT(ready);
 	D_ASSERT(sink);
@@ -138801,6 +139744,59 @@ vector<PhysicalOperator *> Pipeline::GetOperators() const {
 	return result;
 }
 
+//===--------------------------------------------------------------------===//
+// Pipeline Build State
+//===--------------------------------------------------------------------===//
+void PipelineBuildState::SetPipelineSource(Pipeline &pipeline, PhysicalOperator *op) {
+	pipeline.source = op;
+}
+
+void PipelineBuildState::SetPipelineSink(Pipeline &pipeline, PhysicalOperator *op) {
+	pipeline.sink = op;
+	// set the base batch index of this pipeline based on how many other pipelines have this node as their sink
+	pipeline.base_batch_index = BATCH_INCREMENT * sink_pipeline_count[op];
+	// increment the number of nodes that have this pipeline as their sink
+	sink_pipeline_count[op]++;
+}
+
+void PipelineBuildState::AddPipelineOperator(Pipeline &pipeline, PhysicalOperator *op) {
+	pipeline.operators.push_back(op);
+}
+
+void PipelineBuildState::AddPipeline(Executor &executor, shared_ptr<Pipeline> pipeline) {
+	executor.pipelines.push_back(move(pipeline));
+}
+
+PhysicalOperator *PipelineBuildState::GetPipelineSource(Pipeline &pipeline) {
+	return pipeline.source;
+}
+
+PhysicalOperator *PipelineBuildState::GetPipelineSink(Pipeline &pipeline) {
+	return pipeline.sink;
+}
+
+void PipelineBuildState::SetPipelineOperators(Pipeline &pipeline, vector<PhysicalOperator *> operators) {
+	pipeline.operators = move(operators);
+}
+
+void PipelineBuildState::AddChildPipeline(Executor &executor, Pipeline &pipeline) {
+	executor.AddChildPipeline(&pipeline);
+}
+
+unordered_map<Pipeline *, vector<shared_ptr<Pipeline>>> &PipelineBuildState::GetUnionPipelines(Executor &executor) {
+	return executor.union_pipelines;
+}
+unordered_map<Pipeline *, vector<shared_ptr<Pipeline>>> &PipelineBuildState::GetChildPipelines(Executor &executor) {
+	return executor.child_pipelines;
+}
+unordered_map<Pipeline *, vector<Pipeline *>> &PipelineBuildState::GetChildDependencies(Executor &executor) {
+	return executor.child_dependencies;
+}
+
+vector<PhysicalOperator *> PipelineBuildState::GetPipelineOperators(Pipeline &pipeline) {
+	return pipeline.operators;
+}
+
 } // namespace duckdb
 
 
@@ -138842,6 +139838,7 @@ void PipelineEvent::FinishEvent() {
 
 
 
+
 namespace duckdb {
 
 PipelineExecutor::PipelineExecutor(ClientContext &context_p, Pipeline &pipeline_p)
@@ -138850,7 +139847,9 @@ PipelineExecutor::PipelineExecutor(ClientContext &context_p, Pipeline &pipeline_
 	local_source_state = pipeline.source->GetLocalSourceState(context, *pipeline.source_state);
 	if (pipeline.sink) {
 		local_sink_state = pipeline.sink->GetLocalSinkState(context);
+		requires_batch_index = pipeline.sink->RequiresBatchIndex() && pipeline.source->SupportsBatchIndex();
 	}
+	bool can_cache_in_pipeline = pipeline.sink && !pipeline.IsOrderDependent() && !requires_batch_index;
 	intermediate_chunks.reserve(pipeline.operators.size());
 	intermediate_states.reserve(pipeline.operators.size());
 	cached_chunks.resize(pipeline.operators.size());
@@ -138861,7 +139860,7 @@ PipelineExecutor::PipelineExecutor(ClientContext &context_p, Pipeline &pipeline_
 		chunk->Initialize(prev_operator->GetTypes());
 		intermediate_chunks.push_back(move(chunk));
 		intermediate_states.push_back(current_operator->GetOperatorState(context.client));
-		if (pipeline.sink && !pipeline.sink->SinkOrderMatters() && current_operator->RequiresCache()) {
+		if (can_cache_in_pipeline && current_operator->RequiresCache()) {
 			auto &cache_types = current_operator->GetTypes();
 			bool can_cache = true;
 			for (auto &type : cache_types) {
@@ -138879,7 +139878,7 @@ PipelineExecutor::PipelineExecutor(ClientContext &context_p, Pipeline &pipeline_
 		if (current_operator->IsSink() && current_operator->sink_state->state == SinkFinalizeType::NO_OUTPUT_POSSIBLE) {
 			// one of the operators has already figured out no output is possible
 			// we can skip executing the pipeline
-			finished_processing = true;
+			FinishProcessing();
 		}
 	}
 	InitializeChunk(final_chunk);
@@ -138890,7 +139889,7 @@ bool PipelineExecutor::Execute(idx_t max_chunks) {
 	bool exhausted_source = false;
 	auto &source_chunk = pipeline.operators.empty() ? final_chunk : *intermediate_chunks[0];
 	for (idx_t i = 0; i < max_chunks; i++) {
-		if (finished_processing) {
+		if (IsFinished()) {
 			break;
 		}
 		source_chunk.Reset();
@@ -138901,11 +139900,11 @@ bool PipelineExecutor::Execute(idx_t max_chunks) {
 		}
 		auto result = ExecutePushInternal(source_chunk);
 		if (result == OperatorResultType::FINISHED) {
-			finished_processing = true;
+			D_ASSERT(IsFinished());
 			break;
 		}
 	}
-	if (!exhausted_source && !finished_processing) {
+	if (!exhausted_source && !IsFinished()) {
 		return false;
 	}
 	PushFinalize();
@@ -138920,6 +139919,15 @@ OperatorResultType PipelineExecutor::ExecutePush(DataChunk &input) { // LCOV_EXC
 	return ExecutePushInternal(input);
 } // LCOV_EXCL_STOP
 
+void PipelineExecutor::FinishProcessing(int32_t operator_idx) {
+	finished_processing_idx = operator_idx < 0 ? NumericLimits<int32_t>::Maximum() : operator_idx;
+	in_process_operators = stack<idx_t>();
+}
+
+bool PipelineExecutor::IsFinished() {
+	return finished_processing_idx >= 0;
+}
+
 OperatorResultType PipelineExecutor::ExecutePushInternal(DataChunk &input, idx_t initial_idx) {
 	D_ASSERT(pipeline.sink);
 	if (input.size() == 0) { // LCOV_EXCL_START
@@ -138944,6 +139952,7 @@ OperatorResultType PipelineExecutor::ExecutePushInternal(DataChunk &input, idx_t
 			auto sink_result = pipeline.sink->Sink(context, *pipeline.sink->sink_state, *local_sink_state, sink_chunk);
 			EndOperator(pipeline.sink, nullptr);
 			if (sink_result == SinkResultType::FINISHED) {
+				FinishProcessing();
 				return OperatorResultType::FINISHED;
 			}
 		}
@@ -138951,7 +139960,6 @@ OperatorResultType PipelineExecutor::ExecutePushInternal(DataChunk &input, idx_t
 			return OperatorResultType::NEED_MORE_INPUT;
 		}
 	}
-	return OperatorResultType::FINISHED;
 }
 
 void PipelineExecutor::PushFinalize() {
@@ -138960,13 +139968,15 @@ void PipelineExecutor::PushFinalize() {
 	}
 	finalized = true;
 	// flush all caches
-	if (!finished_processing) {
-		D_ASSERT(in_process_operators.empty());
-		for (idx_t i = 0; i < cached_chunks.size(); i++) {
-			if (cached_chunks[i] && cached_chunks[i]->size() > 0) {
-				ExecutePushInternal(*cached_chunks[i], i + 1);
-				cached_chunks[i].reset();
-			}
+	// note that even if an operator has finished, we might still need to flush caches AFTER that operator
+	// e.g. if we have SOURCE -> LIMIT -> CROSS_PRODUCT -> SINK, if the LIMIT reports no more rows will be passed on
+	// we still need to flush caches from the CROSS_PRODUCT
+	D_ASSERT(in_process_operators.empty());
+	idx_t start_idx = IsFinished() ? idx_t(finished_processing_idx) : 0;
+	for (idx_t i = start_idx; i < cached_chunks.size(); i++) {
+		if (cached_chunks[i] && cached_chunks[i]->size() > 0) {
+			ExecutePushInternal(*cached_chunks[i], i + 1);
+			cached_chunks[i].reset();
 		}
 	}
 	D_ASSERT(local_sink_state);
@@ -139022,7 +140032,7 @@ void PipelineExecutor::CacheChunk(DataChunk &current_chunk, idx_t operator_idx)
 }
 
 void PipelineExecutor::ExecutePull(DataChunk &result) {
-	if (finished_processing) {
+	if (IsFinished()) {
 		return;
 	}
 	auto &executor = pipeline.executor;
@@ -139038,7 +140048,10 @@ void PipelineExecutor::ExecutePull(DataChunk &result) {
 				}
 			}
 			if (!pipeline.operators.empty()) {
-				Execute(source_chunk, result);
+				auto state = Execute(source_chunk, result);
+				if (state == OperatorResultType::FINISHED) {
+					break;
+				}
 			}
 		}
 	} catch (std::exception &ex) { // LCOV_EXCL_START
@@ -139122,6 +140135,7 @@ OperatorResultType PipelineExecutor::Execute(DataChunk &input, DataChunk &result
 				in_process_operators.push(current_idx);
 			} else if (result == OperatorResultType::FINISHED) {
 				D_ASSERT(current_chunk.size() == 0);
+				FinishProcessing(current_idx);
 				return OperatorResultType::FINISHED;
 			}
 			current_chunk.Verify();
@@ -139155,6 +140169,14 @@ OperatorResultType PipelineExecutor::Execute(DataChunk &input, DataChunk &result
 void PipelineExecutor::FetchFromSource(DataChunk &result) {
 	StartOperator(pipeline.source);
 	pipeline.source->GetData(context, result, *pipeline.source_state, *local_source_state);
+	if (result.size() != 0 && requires_batch_index) {
+		auto next_batch_index =
+		    pipeline.source->GetBatchIndex(context, result, *pipeline.source_state, *local_source_state);
+		next_batch_index += pipeline.base_batch_index;
+		D_ASSERT(local_sink_state->batch_index <= next_batch_index ||
+		         local_sink_state->batch_index == DConstants::INVALID_INDEX);
+		local_sink_state->batch_index = next_batch_index;
+	}
 	EndOperator(pipeline.source, &result);
 }