duckdb 0.7.2-dev2867.0 → 0.7.2-dev2995.0

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_schema.hpp

@@ -28,9 +28,7 @@ public:
 
 public:
 	// Source interface
-	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
-	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-	             LocalSourceState &lstate) const override;
+	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
 
 	bool IsSource() const override {
 		return true;

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_sequence.hpp

@@ -28,9 +28,7 @@ public:
 
 public:
 	// Source interface
-	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
-	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-	             LocalSourceState &lstate) const override;
+	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
 
 	bool IsSource() const override {
 		return true;

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_table.hpp

@@ -29,9 +29,7 @@ public:
 
 public:
 	// Source interface
-	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
-	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-	             LocalSourceState &lstate) const override;
+	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
 
 	bool IsSource() const override {
 		return true;

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_type.hpp

@@ -25,9 +25,7 @@ public:
 
 public:
 	// Source interface
-	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
-	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-	             LocalSourceState &lstate) const override;
+	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
 
 	bool IsSource() const override {
 		return true;
@@ -37,8 +35,7 @@ public:
 	// Sink interface
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
 
-	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &gstate_p, LocalSinkState &lstate_p,
-	                    DataChunk &input) const override;
+	SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
 
 	bool IsSink() const override {
 		return info->query != nullptr;

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_view.hpp

@@ -28,9 +28,7 @@ public:
 
 public:
 	// Source interface
-	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
-	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-	             LocalSourceState &lstate) const override;
+	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
 
 	bool IsSource() const override {
 		return true;

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_detach.hpp

@@ -27,9 +27,7 @@ public:
 
 public:
 	// Source interface
-	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
-	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-	             LocalSourceState &lstate) const override;
+	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
 
 	bool IsSource() const override {
 		return true;

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_drop.hpp

@@ -28,9 +28,7 @@ public:
 
 public:
 	// Source interface
-	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
-	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-	             LocalSourceState &lstate) const override;
+	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
 
 	bool IsSource() const override {
 		return true;

package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp

@@ -30,8 +30,7 @@ public:
 
 public:
 	// Source interface
-	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-	             LocalSourceState &lstate) const override;
+	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
 
 	bool IsSource() const override {
 		return true;
@@ -39,8 +38,7 @@ public:
 
 public:
 	// Sink interface
-	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
-	                    DataChunk &input) const override;
+	SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
 
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
 

package/src/duckdb/src/include/duckdb/execution/physical_operator.hpp

@@ -15,6 +15,7 @@
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/execution/execution_context.hpp"
 #include "duckdb/optimizer/join_order/join_node.hpp"
+#include "duckdb/common/optional_idx.hpp"
 #include "duckdb/execution/physical_operator_states.hpp"
 #include "duckdb/common/enums/order_preservation_type.hpp"
 
@@ -102,8 +103,8 @@ public:
 	virtual unique_ptr<LocalSourceState> GetLocalSourceState(ExecutionContext &context,
 	                                                         GlobalSourceState &gstate) const;
 	virtual unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const;
-	virtual void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-	                     LocalSourceState &lstate) const;
+	virtual SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const;
+
 	virtual idx_t GetBatchIndex(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	                            LocalSourceState &lstate) const;
 
@@ -132,8 +133,7 @@ public:
 
 	//! The sink method is called constantly with new input, as long as new input is available. Note that this method
 	//! CAN be called in parallel, proper locking is needed when accessing data inside the GlobalSinkState.
-	virtual SinkResultType Sink(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate,
-	                            DataChunk &input) const;
+	virtual SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const;
 	// The combine is called when a single thread has completed execution of its part of the pipeline, it is the final
 	// time that a specific LocalSinkState is accessible. This method can be called in parallel while other Sink() or
 	// Combine() calls are active on the same GlobalSinkState.
@@ -143,6 +143,9 @@ public:
 	//! If Finalize returns SinkResultType::FINISHED, the sink is marked as finished
 	virtual SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
 	                                  GlobalSinkState &gstate) const;
+	//! For sinks with RequiresBatchIndex set to true, when a new batch starts being processed this method is called
+	//! This allows flushing of the current batch (e.g. to disk)
+	virtual void NextBatch(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate_p) const;
 
 	virtual unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const;
 	virtual unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const;

package/src/duckdb/src/include/duckdb/execution/physical_operator_states.hpp

@@ -23,6 +23,18 @@ class PhysicalOperator;
 class Pipeline;
 class PipelineBuildState;
 class MetaPipeline;
+class InterruptState;
+
+struct SourcePartitionInfo {
+	//! The current batch index
+	//! This is only set in case RequiresBatchIndex() is true, and the source has support for it (SupportsBatchIndex())
+	//! Otherwise this is left on INVALID_INDEX
+	//! The batch index is a globally unique, increasing index that should be used to maintain insertion order
+	//! //! in conjunction with parallelism
+	optional_idx batch_index;
+	//! The minimum batch index that any thread is currently actively reading
+	optional_idx min_batch_index;
+};
 
 // LCOV_EXCL_START
 class OperatorState {
@@ -88,12 +100,8 @@ public:
 	virtual ~LocalSinkState() {
 	}
 
-	//! The current batch index
-	//! This is only set in case RequiresBatchIndex() is true, and the source has support for it (SupportsBatchIndex())
-	//! Otherwise this is left on INVALID_INDEX
-	//! The batch index is a globally unique, increasing index that should be used to maintain insertion order
-	//! //! in conjunction with parallelism
-	idx_t batch_index = DConstants::INVALID_INDEX;
+	//! Source partition info
+	SourcePartitionInfo partition_info;
 
 	template <class TARGET>
 	TARGET &Cast() {
@@ -145,6 +153,18 @@ public:
 	}
 };
 
+struct OperatorSinkInput {
+	GlobalSinkState &global_state;
+	LocalSinkState &local_state;
+	InterruptState &interrupt_state;
+};
+
+struct OperatorSourceInput {
+	GlobalSourceState &global_state;
+	LocalSourceState &local_state;
+	InterruptState &interrupt_state;
+};
+
 // LCOV_EXCL_STOP
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp

@@ -41,20 +41,20 @@ public:
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const;
 	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const;
 
-	void Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate, DataChunk &input,
-	          DataChunk &aggregate_input_chunk, const vector<idx_t> &filter) const;
+	void Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input, DataChunk &aggregate_input_chunk,
+	          const vector<idx_t> &filter) const;
 	void Combine(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate) const;
 	bool Finalize(ClientContext &context, GlobalSinkState &gstate_p) const;
 
 	void ScheduleTasks(Executor &executor, const shared_ptr<Event> &event, GlobalSinkState &state,
-	                   vector<unique_ptr<Task>> &tasks) const;
+	                   vector<shared_ptr<Task>> &tasks) const;
 
 	//! Source interface
 	idx_t Size(GlobalSinkState &sink_state) const;
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const;
 	unique_ptr<LocalSourceState> GetLocalSourceState(ExecutionContext &context) const;
-	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSinkState &sink_state, GlobalSourceState &gstate_p,
-	             LocalSourceState &lstate_p) const;
+	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, GlobalSinkState &sink_state,
+	                         OperatorSourceInput &input) const;
 
 	static void SetMultiScan(GlobalSinkState &state);
 	bool ForceSingleHT(GlobalSinkState &state) const;

package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp

@@ -25,7 +25,8 @@ class BoundAggregateExpression;
 
 struct AggregateInputData {
 	AggregateInputData(FunctionData *bind_data_p, Allocator &allocator_p)
-	    : bind_data(bind_data_p), allocator(allocator_p) {};
+	    : bind_data(bind_data_p), allocator(allocator_p) {
+	}
 	FunctionData *bind_data;
 	Allocator &allocator;
 };

package/src/duckdb/src/include/duckdb/function/table_function.hpp

@@ -182,7 +182,6 @@ typedef unique_ptr<LocalTableFunctionState> (*table_function_init_local_t)(Execu
 typedef unique_ptr<BaseStatistics> (*table_statistics_t)(ClientContext &context, const FunctionData *bind_data,
                                                          column_t column_index);
 typedef void (*table_function_t)(ClientContext &context, TableFunctionInput &data, DataChunk &output);
-
 typedef OperatorResultType (*table_in_out_function_t)(ExecutionContext &context, TableFunctionInput &data,
                                                       DataChunk &input, DataChunk &output);
 typedef OperatorFinalizeResultType (*table_in_out_function_final_t)(ExecutionContext &context, TableFunctionInput &data,

package/src/duckdb/src/include/duckdb/main/client_config.hpp

@@ -63,6 +63,8 @@ struct ClientConfig {
 	bool verify_serializer = false;
 	//! Enable the running of optimizers
 	bool enable_optimizer = true;
+	//! Enable caching operators
+	bool enable_caching_operators = true;
 	//! Force parallelism of small tables, used for testing
 	bool verify_parallelism = false;
 	//! Force index join independent of table cardinality, used for testing

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -149,6 +149,8 @@ struct DBConfigOptions {
 	DebugInitialize debug_initialize = DebugInitialize::NO_INITIALIZE;
 	//! The set of unrecognized (other) options
 	unordered_map<string, Value> unrecognized_options;
+	//! Whether to print bindings when printing the plan (debug mode only)
+	static bool debug_print_bindings;
 
 	bool operator==(const DBConfigOptions &other) const;
 };

package/src/duckdb/src/include/duckdb/parallel/event.hpp

@@ -41,7 +41,7 @@ public:
 
 	void CompleteDependency();
 
-	void SetTasks(vector<unique_ptr<Task>> tasks);
+	void SetTasks(vector<shared_ptr<Task>> tasks);
 
 	void InsertEvent(shared_ptr<Event> replacement_event);
 

package/src/duckdb/src/include/duckdb/parallel/interrupt.hpp

@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// src/include/duckdb/parallel/interrupt.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/atomic.hpp"
+#include "duckdb/common/mutex.hpp"
+#include "duckdb/parallel/task.hpp"
+#include <condition_variable>
+#include <memory>
+
+namespace duckdb {
+
+//! InterruptMode specifies how operators should block/unblock, note that this will happen transparently to the
+//! operator, as the operator only needs to return a BLOCKED result and call the callback using the InterruptState.
+//! NO_INTERRUPTS: No blocking mode is specified, an error will be thrown when the operator blocks. Should only be used
+//!                when manually calling operators of which is known they will never block.
+//! TASK:          A weak pointer to a task is provided. On the callback, this task will be signalled. If the Task has
+//!                been deleted, this callback becomes a NOP. This is the preferred way to await blocked pipelines.
+//! BLOCKING:	   The caller has blocked awaiting some synchronization primitive to wait for the callback.
+enum class InterruptMode : uint8_t { NO_INTERRUPTS, TASK, BLOCKING };
+
+//! Synchronization primitive used to await a callback in InterruptMode::BLOCKING.
+struct InterruptDoneSignalState {
+	//! Called by the callback to signal the interrupt is over
+	void Signal();
+	//! Await the callback signalling the interrupt is over
+	void Await();
+
+protected:
+	mutex lock;
+	std::condition_variable cv;
+	bool done = false;
+};
+
+//! State required to make the callback after some asynchronous operation within an operator source / sink.
+class InterruptState {
+public:
+	//! Default interrupt state will be set to InterruptMode::NO_INTERRUPTS and throw an error on use of Callback()
+	InterruptState();
+	//! Register the task to be interrupted and set mode to InterruptMode::TASK, the preferred way to handle interrupts
+	InterruptState(weak_ptr<Task> task);
+	//! Register signal state and set mode to InterruptMode::BLOCKING, used for code paths without Task.
+	InterruptState(weak_ptr<InterruptDoneSignalState> done_signal);
+
+	//! Perform the callback to indicate the Interrupt is over
+	DUCKDB_API void Callback() const;
+
+protected:
+	//! Current interrupt mode
+	InterruptMode mode;
+	//! Task ptr for InterruptMode::TASK
+	weak_ptr<Task> current_task;
+	//! Signal state for InterruptMode::BLOCKING
+	weak_ptr<InterruptDoneSignalState> signal_state;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp

@@ -9,6 +9,8 @@
 #pragma once
 
 #include "duckdb/common/atomic.hpp"
+#include "duckdb/common/unordered_set.hpp"
+#include "duckdb/common/set.hpp"
 #include "duckdb/execution/physical_operator.hpp"
 #include "duckdb/function/table_function.hpp"
 #include "duckdb/parallel/task_scheduler.hpp"
@@ -64,9 +66,7 @@ public:
 	void Reset();
 	void ResetSink();
 	void ResetSource(bool force);
-	void ClearSource() {
-		source_state.reset();
-	}
+	void ClearSource();
 	void Schedule(shared_ptr<Event> &event);
 
 	//! Finalize this pipeline
@@ -94,6 +94,12 @@ public:
 	//! Returns whether any of the operators in the pipeline care about preserving order
 	bool IsOrderDependent() const;
 
+	//! Registers a new batch index for a pipeline executor - returns the current minimum batch index
+	idx_t RegisterNewBatchIndex();
+
+	//! Updates the batch index of a pipeline (and returns the new minimum batch index)
+	idx_t UpdateBatchIndex(idx_t old_index, idx_t new_index);
+
 private:
 	//! Whether or not the pipeline has been readied
 	bool ready;
@@ -116,6 +122,13 @@ private:
 
 	//! The base batch index of this pipeline
 	idx_t base_batch_index = 0;
+	//! Lock for accessing the set of batch indexes
+	mutex batch_lock;
+	//! The set of batch indexes that are currently being processed
+	//! Despite batch indexes being unique - this is a multiset
+	//! The reason is that when we start a new pipeline we insert the current minimum batch index as a placeholder
+	//! Which leads to duplicate entries in the set of active batch indexes
+	multiset<idx_t> batch_indexes;
 
 private:
 	void ScheduleSequentialTask(shared_ptr<Event> &event);

package/src/duckdb/src/include/duckdb/parallel/pipeline_executor.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/common/types/data_chunk.hpp"
+#include "duckdb/parallel/interrupt.hpp"
 #include "duckdb/parallel/pipeline.hpp"
 #include "duckdb/execution/physical_operator.hpp"
 #include "duckdb/parallel/thread_context.hpp"
@@ -20,16 +21,27 @@
 namespace duckdb {
 class Executor;
 
+//! The result of executing a PipelineExecutor
+enum class PipelineExecuteResult {
+	//! PipelineExecutor is fully executed: the source is completely exhausted
+	FINISHED,
+	//! PipelineExecutor is not yet fully executed and can be called again immediately
+	NOT_FINISHED,
+	//! The PipelineExecutor was interrupted and should not be called again until the interrupt is handled as specified
+	//! in the InterruptMode
+	INTERRUPTED
+};
+
 //! The Pipeline class represents an execution pipeline
 class PipelineExecutor {
 public:
 	PipelineExecutor(ClientContext &context, Pipeline &pipeline);
 
 	//! Fully execute a pipeline with a source and a sink until the source is completely exhausted
-	void Execute();
-	//! Execute a pipeline with a source and a sink until finished, or until max_chunks have been processed
+	PipelineExecuteResult Execute();
+	//! Execute a pipeline with a source and a sink until finished, or until max_chunks were processed from the source
 	//! Returns true if execution is finished, false if Execute should be called again
-	bool Execute(idx_t max_chunks);
+	PipelineExecuteResult Execute(idx_t max_chunks);
 
 	//! Push a single input DataChunk into the pipeline.
 	//! Returns either OperatorResultType::NEED_MORE_INPUT or OperatorResultType::FINISHED
@@ -48,6 +60,9 @@ public:
 	//! This flushes profiler states
 	void PullFinalize();
 
+	//! Registers the task in the interrupt_state to allow Source/Sink operators to block the task
+	void SetTaskForInterrupts(weak_ptr<Task> current_task);
+
 private:
 	//! The pipeline to process
 	Pipeline &pipeline;
@@ -65,6 +80,8 @@ private:
 	unique_ptr<LocalSourceState> local_source_state;
 	//! The local sink state (if any)
 	unique_ptr<LocalSinkState> local_sink_state;
+	//! The interrupt state, holding required information for sink/source operators to block
+	InterruptState interrupt_state;
 
 	//! The final chunk used for moving data into the sink
 	DataChunk final_chunk;
@@ -79,28 +96,55 @@ private:
 	//! Whether or not this pipeline requires keeping track of the batch index of the source
 	bool requires_batch_index = false;
 
+	//! Source has indicated it is exhausted
+	bool exhausted_source = false;
+	//! Flushing of intermediate operators has started
+	bool started_flushing = false;
+	//! Flushing of caching operators is done
+	bool done_flushing = false;
+
+	//! This flag is set when the pipeline gets interrupted by the Sink -> the final_chunk should be re-sink-ed.
+	bool remaining_sink_chunk = false;
+
+	//! Current operator being flushed
+	idx_t flushing_idx;
+	//! Whether the current flushing_idx should be flushed: this needs to be stored to make flushing code re-entrant
+	bool should_flush_current_idx = true;
+
 private:
 	void StartOperator(PhysicalOperator &op);
 	void EndOperator(PhysicalOperator &op, optional_ptr<DataChunk> chunk);
 
 	//! Reset the operator index to the first operator
 	void GoToSource(idx_t &current_idx, idx_t initial_idx);
-	void FetchFromSource(DataChunk &result);
+	SourceResultType FetchFromSource(DataChunk &result);
 
 	void FinishProcessing(int32_t operator_idx = -1);
 	bool IsFinished();
 
+	//! Wrappers for sink/source calls to respective operators
+	SourceResultType GetData(DataChunk &chunk, OperatorSourceInput &input);
+	SinkResultType Sink(DataChunk &chunk, OperatorSinkInput &input);
+
 	OperatorResultType ExecutePushInternal(DataChunk &input, idx_t initial_idx = 0);
 	//! Pushes a chunk through the pipeline and returns a single result chunk
 	//! Returns whether or not a new input chunk is needed, or whether or not we are finished
 	OperatorResultType Execute(DataChunk &input, DataChunk &result, idx_t initial_index = 0);
 
-	//! FlushCachedOperators methods push/pull any remaining cached results through the pipeline
-	void FlushCachingOperatorsPull(DataChunk &result);
-	void FlushCachingOperatorsPush();
+	//! Tries to flush all state from intermediate operators. Will return true if all state is flushed, false in the
+	//! case of a blocked sink.
+	bool TryFlushCachingOperators();
 
 	static bool CanCacheType(const LogicalType &type);
 	void CacheChunk(DataChunk &input, idx_t operator_idx);
+
+#ifdef DUCKDB_DEBUG_ASYNC_SINK_SOURCE
+	//! Debugging state: number of times blocked
+	int debug_blocked_sink_count = 0;
+	int debug_blocked_source_count = 0;
+	//! Number of times the Sink/Source will block before actually returning data
+	int debug_blocked_target_count = 1;
+#endif
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/parallel/task.hpp

@@ -13,13 +13,16 @@
 namespace duckdb {
 class ClientContext;
 class Executor;
+class Task;
+class DatabaseInstance;
+struct ProducerToken;
 
 enum class TaskExecutionMode : uint8_t { PROCESS_ALL, PROCESS_PARTIAL };
 
-enum class TaskExecutionResult : uint8_t { TASK_FINISHED, TASK_NOT_FINISHED, TASK_ERROR };
+enum class TaskExecutionResult : uint8_t { TASK_FINISHED, TASK_NOT_FINISHED, TASK_ERROR, TASK_BLOCKED };
 
 //! Generic parallel task
-class Task {
+class Task : public std::enable_shared_from_this<Task> {
 public:
 	virtual ~Task() {
 	}
@@ -28,7 +31,20 @@ public:
 	//! If mode is PROCESS_ALL, Execute should always finish processing and return TASK_FINISHED
 	//! If mode is PROCESS_PARTIAL, Execute can return TASK_NOT_FINISHED, in which case Execute will be called again
 	//! In case of an error, TASK_ERROR is returned
+	//! In case the task has interrupted, BLOCKED is returned.
 	virtual TaskExecutionResult Execute(TaskExecutionMode mode) = 0;
+
+	//! Descheduling a task ensures the task is not executed, but remains available for rescheduling as long as
+	//! required, generally until some code in an operator calls the InterruptState::Callback() method of a state of the
+	//! InterruptMode::TASK mode.
+	virtual void Deschedule() {
+		throw InternalException("Cannot deschedule task of base Task class");
+	};
+
+	//! Ensures a task is rescheduled to the correct queue
+	virtual void Reschedule() {
+		throw InternalException("Cannot reschedule task of base Task class");
+	}
 };
 
 //! Execute a task within an executor, including exception handling
@@ -39,6 +55,9 @@ public:
 	ExecutorTask(ClientContext &context);
 	virtual ~ExecutorTask();
 
+	void Deschedule() override;
+	void Reschedule() override;
+
 	Executor &executor;
 
 public:

package/src/duckdb/src/include/duckdb/parallel/task_counter.hpp

@@ -18,7 +18,7 @@ public:
 	    : scheduler(scheduler_p), token(scheduler_p.CreateProducer()), task_count(0), tasks_completed(0) {
 	}
 
-	virtual void AddTask(unique_ptr<Task> task) {
+	virtual void AddTask(shared_ptr<Task> task) {
 		++task_count;
 		scheduler.ScheduleTask(*token, std::move(task));
 	}
@@ -29,7 +29,7 @@ public:
 
 	virtual void Finish() {
 		while (tasks_completed < task_count) {
-			unique_ptr<Task> task;
+			shared_ptr<Task> task;
 			if (scheduler.GetTaskFromProducer(*token, task)) {
 				task->Execute();
 				task.reset();

package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp

@@ -47,9 +47,9 @@ public:
 
 	unique_ptr<ProducerToken> CreateProducer();
 	//! Schedule a task to be executed by the task scheduler
-	void ScheduleTask(ProducerToken &producer, unique_ptr<Task> task);
+	void ScheduleTask(ProducerToken &producer, shared_ptr<Task> task);
 	//! Fetches a task from a specific producer, returns true if successful or false if no tasks were available
-	bool GetTaskFromProducer(ProducerToken &token, unique_ptr<Task> &task);
+	bool GetTaskFromProducer(ProducerToken &token, shared_ptr<Task> &task);
 	//! Run tasks forever until "marker" is set to false, "marker" must remain valid until the thread is joined
 	void ExecuteForever(atomic<bool> *marker);
 	//! Run tasks until `marker` is set to false, `max_tasks` have been completed, or until there are no more tasks

package/src/duckdb/src/include/duckdb/planner/column_binding.hpp

@@ -9,6 +9,8 @@
 #pragma once
 
 #include "duckdb/common/common.hpp"
+#include "duckdb/common/to_string.hpp"
+
 #include <functional>
 
 namespace duckdb {
@@ -23,6 +25,10 @@ struct ColumnBinding {
 	ColumnBinding(idx_t table, idx_t column) : table_index(table), column_index(column) {
 	}
 
+	string ToString() const {
+		return "#[" + to_string(table_index) + "." + to_string(column_index) + "]";
+	}
+
 	bool operator==(const ColumnBinding &rhs) const {
 		return table_index == rhs.table_index && column_index == rhs.column_index;
 	}

package/src/duckdb/src/include/duckdb/planner/expression/bound_columnref_expression.hpp

@@ -42,6 +42,7 @@ public:
 	}
 
 	string ToString() const override;
+	string GetName() const override;
 
 	bool Equals(const BaseExpression *other) const override;
 	hash_t Hash() const override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_aggregate.hpp

@@ -47,6 +47,7 @@ public:
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	idx_t EstimateCardinality(ClientContext &context) override;
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_column_data_get.hpp

@@ -34,6 +34,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override {

package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp

@@ -41,6 +41,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override {