duckdb 0.3.1-dev11.0 → 0.3.1-dev210.0

package/src/duckdb.hpp

@@ -10,8 +10,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 #pragma once
 #define DUCKDB_AMALGAMATION 1
-#define DUCKDB_SOURCE_ID "42decfe98"
-#define DUCKDB_VERSION "0.3.1-dev11"
+#define DUCKDB_SOURCE_ID "f05f6f50f"
+#define DUCKDB_VERSION "0.3.1-dev210"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -567,11 +567,12 @@ using std::vector;
 #endif
 
 
+
 namespace duckdb {
 
 class Serializer;
 class Deserializer;
-
+class Value;
 //! Type used to represent dates (days since 1970-01-01)
 struct date_t {
 	int32_t days;
@@ -861,12 +862,12 @@ enum class PhysicalType : uint8_t {
 	/// Like LIST, but with 64-bit offsets
 	LARGE_LIST = 33,
 
-	// DuckDB Extensions
+	/// DuckDB Extensions
 	VARCHAR = 200, // our own string representation, different from STRING and LARGE_STRING above
 	INT128 = 204, // 128-bit integers
-
+	UNKNOWN = 205, // Unknown physical type of user defined types
 	/// Boolean as 1 bit, LSB bit-packed ordering
-	BIT = 205,
+	BIT = 206,
 
 	INVALID = 255
 };
@@ -879,7 +880,7 @@ enum class LogicalTypeId : uint8_t {
 	SQLNULL = 1, /* NULL type, used for constant NULL */
 	UNKNOWN = 2, /* unknown type, used for parameter expressions */
 	ANY = 3,     /* ANY type, used for functions that accept any type as parameter */
-
+	USER = 4, /* A User Defined Type (e.g., ENUMs before the binder) */
 	BOOLEAN = 10,
 	TINYINT = 11,
 	SMALLINT = 12,
@@ -913,7 +914,8 @@ enum class LogicalTypeId : uint8_t {
 	STRUCT = 100,
 	LIST = 101,
 	MAP = 102,
-	TABLE = 103
+	TABLE = 103,
+	ENUM = 104
 };
 
 struct ExtraTypeInfo;
@@ -922,6 +924,7 @@ struct LogicalType {
 	DUCKDB_API LogicalType();
 	DUCKDB_API LogicalType(LogicalTypeId id); // NOLINT: Allow implicit conversion from `LogicalTypeId`
 	DUCKDB_API LogicalType(LogicalTypeId id, shared_ptr<ExtraTypeInfo> type_info);
+
 	DUCKDB_API LogicalType(const LogicalType &other) :
 		id_(other.id_), physical_type_(other.physical_type_), type_info_(other.type_info_) {}
 
@@ -1018,10 +1021,11 @@ public:
 	// explicitly allowing these functions to be capitalized to be in-line with the remaining functions
 	DUCKDB_API static LogicalType DECIMAL(int width, int scale);                 // NOLINT
 	DUCKDB_API static LogicalType VARCHAR_COLLATION(string collation);           // NOLINT
-	DUCKDB_API static LogicalType LIST(LogicalType child);                       // NOLINT
-	DUCKDB_API static LogicalType STRUCT(child_list_t<LogicalType> children);    // NOLINT
-	DUCKDB_API static LogicalType MAP(child_list_t<LogicalType> children);       // NOLINT
-
+	DUCKDB_API static LogicalType LIST( LogicalType child);                       // NOLINT
+	DUCKDB_API static LogicalType STRUCT( child_list_t<LogicalType> children);    // NOLINT
+	DUCKDB_API static LogicalType MAP( child_list_t<LogicalType> children);       // NOLINT
+	DUCKDB_API static LogicalType ENUM(const string &enum_name, const vector<string> &ordered_data); // NOLINT
+	DUCKDB_API static LogicalType USER(const string &user_type_name); // NOLINT
 	//! A list of all NUMERIC types (integral and floating point types)
 	DUCKDB_API static const vector<LogicalType> NUMERIC;
 	//! A list of all INTEGRAL types
@@ -1043,6 +1047,18 @@ struct ListType {
 	DUCKDB_API static const LogicalType &GetChildType(const LogicalType &type);
 };
 
+struct UserType{
+	DUCKDB_API static const string &GetTypeName(const LogicalType &type);
+};
+
+struct EnumType{
+	DUCKDB_API static const string &GetTypeName(const LogicalType &type);
+	DUCKDB_API static int64_t GetPos(const LogicalType &type, const string& key);
+	DUCKDB_API static const vector<string> &GetValuesInsertOrder(const LogicalType &type);
+	DUCKDB_API static idx_t GetSize(const LogicalType &type);
+	DUCKDB_API static const string& GetValue(const Value &val);
+};
+
 struct StructType {
 	DUCKDB_API static const child_list_t<LogicalType> &GetChildTypes(const LogicalType &type);
 	DUCKDB_API static const LogicalType &GetChildType(const LogicalType &type, idx_t index);
@@ -1497,7 +1513,7 @@ public:
 	//! Write a string with a length prefix
 	void WriteString(const string &val) {
 		Write<uint32_t>((uint32_t)val.size());
-		if (val.size() > 0) {
+		if (!val.empty()) {
 			WriteData((const_data_ptr_t)val.c_str(), val.size());
 		}
 	}
@@ -2568,6 +2584,9 @@ public:
 	DUCKDB_API static Value INTERVAL(int32_t months, int32_t days, int64_t micros);
 	DUCKDB_API static Value INTERVAL(interval_t interval);
 
+	// Create a enum Value from a specified uint value
+	DUCKDB_API static Value ENUM(uint64_t value, const LogicalType &original_type);
+
 	// Decimal values
 	DUCKDB_API static Value DECIMAL(int16_t value, uint8_t width, uint8_t scale);
 	DUCKDB_API static Value DECIMAL(int32_t value, uint8_t width, uint8_t scale);
@@ -3293,6 +3312,9 @@ public:
 	//! Without changing the type of this vector
 	void Reinterpret(Vector &other);
 
+	//! Causes this vector to reference the data held by the other vector, changes the type if required.
+	void ReferenceAndSetType(Vector &other);
+
 	//! Resets a vector from a vector cache.
 	//! This turns the vector back into an empty FlatVector with STANDARD_VECTOR_SIZE entries.
 	//! The VectorCache is used so this can be done without requiring any allocations.
@@ -7134,6 +7156,7 @@ enum class CatalogType : uint8_t {
 	PREPARED_STATEMENT = 5,
 	SEQUENCE_ENTRY = 6,
 	COLLATION_ENTRY = 7,
+	TYPE_ENTRY = 8,
 
 	// functions
 	TABLE_FUNCTION_ENTRY = 25,
@@ -7302,6 +7325,7 @@ struct CreateFunctionInfo;
 struct CreateViewInfo;
 struct CreateSequenceInfo;
 struct CreateCollationInfo;
+struct CreateTypeInfo;
 
 class ClientContext;
 class Transaction;
@@ -7390,6 +7414,8 @@ public:
 	CatalogEntry *CreateView(ClientContext &context, CreateViewInfo *info);
 	//! Creates a table in the catalog.
 	CatalogEntry *CreateSequence(ClientContext &context, CreateSequenceInfo *info);
+	//! Creates a Enum in the catalog.
+	CatalogEntry *CreateType(ClientContext &context, CreateTypeInfo *info);
 	//! Creates a collation in the catalog
 	CatalogEntry *CreateCollation(ClientContext &context, CreateCollationInfo *info);
 
@@ -7409,6 +7435,8 @@ public:
 	CatalogEntry *CreateView(ClientContext &context, SchemaCatalogEntry *schema, CreateViewInfo *info);
 	//! Creates a table in the catalog.
 	CatalogEntry *CreateSequence(ClientContext &context, SchemaCatalogEntry *schema, CreateSequenceInfo *info);
+	//! Creates a enum in the catalog.
+	CatalogEntry *CreateType(ClientContext &context, SchemaCatalogEntry *schema, CreateTypeInfo *info);
 	//! Creates a collation in the catalog
 	CatalogEntry *CreateCollation(ClientContext &context, SchemaCatalogEntry *schema, CreateCollationInfo *info);
 
@@ -7555,6 +7583,7 @@ enum class LogicalOperatorType : uint8_t {
 	LOGICAL_DROP = 132,
 	LOGICAL_PRAGMA = 133,
 	LOGICAL_TRANSACTION = 134,
+	LOGICAL_CREATE_TYPE = 135,
 
 	// -----------------------------
 	// Explain
@@ -9249,7 +9278,7 @@ public:
 	//! Delete from a table, can only be used on a TableRelation
 	DUCKDB_API virtual void Delete(const string &condition = string());
 	//! Create a relation from calling a table in/out function on the input relation
-	DUCKDB_API shared_ptr<Relation> TableFunction(const std::string &fname, vector<Value> values);
+	DUCKDB_API shared_ptr<Relation> TableFunction(const std::string &fname, const vector<Value> &values);
 
 public:
 	//! Whether or not the relation inherits column bindings from its child or not, only relevant for binding
@@ -9504,7 +9533,7 @@ public:
 	DUCKDB_API shared_ptr<Relation> ReadCSV(const string &csv_file);
 	DUCKDB_API shared_ptr<Relation> ReadCSV(const string &csv_file, const vector<string> &columns);
 	//! Returns a relation from a query
-	DUCKDB_API shared_ptr<Relation> RelationFromQuery(string query, string alias = "queryrelation");
+	DUCKDB_API shared_ptr<Relation> RelationFromQuery(const string &query, const string &alias = "queryrelation");
 
 	DUCKDB_API void BeginTransaction();
 	DUCKDB_API void Commit();
@@ -12138,6 +12167,7 @@ struct CreateSequenceInfo;
 struct CreateSchemaInfo;
 struct CreateTableFunctionInfo;
 struct CreateCopyFunctionInfo;
+struct CreateTypeInfo;
 
 struct DropInfo;
 
@@ -12165,6 +12195,8 @@ private:
 	CatalogSet sequences;
 	//! The catalog set holding the collations
 	CatalogSet collations;
+	//! The catalog set holding the types
+	CatalogSet types;
 
 public:
 	//! Scan the specified catalog set, invoking the callback method for every entry
@@ -12199,6 +12231,8 @@ private:
 	CatalogEntry *CreatePragmaFunction(ClientContext &context, CreatePragmaFunctionInfo *info);
 	//! Create a collation within the given schema
 	CatalogEntry *CreateCollation(ClientContext &context, CreateCollationInfo *info);
+	//! Create a enum within the given schema
+	CatalogEntry *CreateType(ClientContext &context, CreateTypeInfo *info);
 
 	//! Drops an entry from the schema
 	void DropEntry(ClientContext &context, DropInfo *info);
@@ -12307,16 +12341,6 @@ using std::thread;
 
 
 
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/execution/physical_sink.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-
-
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -12407,6 +12431,7 @@ enum class PhysicalOperatorType : uint8_t {
 	DROP,
 	PRAGMA,
 	TRANSACTION,
+	CREATE_TYPE,
 
 	// -----------------------------
 	// Helpers
@@ -12442,60 +12467,110 @@ string PhysicalOperatorToString(PhysicalOperatorType type);
 namespace duckdb {
 class ClientContext;
 class ThreadContext;
-class TaskContext;
 
 class ExecutionContext {
 public:
-	ExecutionContext(ClientContext &client_p, ThreadContext &thread_p, TaskContext &task_p)
-	    : client(client_p), thread(thread_p), task(task_p) {
+	ExecutionContext(ClientContext &client_p, ThreadContext &thread_p) : client(client_p), thread(thread_p) {
 	}
 
 	//! The client-global context; caution needs to be taken when used in parallel situations
 	ClientContext &client;
 	//! The thread-local context for this execution
 	ThreadContext &thread;
-	//! The task context for this execution
-	TaskContext &task;
 };
 
 } // namespace duckdb
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/enums/operator_result_type.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
 
 
-#include <functional>
-#include <utility>
 
 namespace duckdb {
-class ExpressionExecutor;
+
+//! The OperatorResultType is used to indicate how data should flow around a regular (i.e. non-sink and non-source)
+//! physical operator
+//! There are three possible results:
+//! NEED_MORE_INPUT means the operator is done with the current input and can consume more input if available
+//! If there is more input the operator will be called with more input, otherwise the operator will not be called again.
+//! HAVE_MORE_OUTPUT means the operator is not finished yet with the current input.
+//! The operator will be called again with the same input.
+//! FINISHED means the operator has finished the entire pipeline and no more processing is necessary.
+//! The operator will not be called again, and neither will any other operators in this pipeline.
+enum class OperatorResultType : uint8_t { NEED_MORE_INPUT, HAVE_MORE_OUTPUT, FINISHED };
+
+//! The SinkResultType is used to indicate the result of data flowing into a sink
+//! There are two possible results:
+//! NEED_MORE_INPUT means the sink needs more input
+//! FINISHED means the sink is finished executing, and more input will not change the result any further
+enum class SinkResultType : uint8_t { NEED_MORE_INPUT, FINISHED };
+
+//! The SinkFinalizeType is used to indicate the result of a Finalize call on a sink
+//! There are two possible results:
+//! READY means the sink is ready for further processing
+//! NO_OUTPUT_POSSIBLE means the sink will never provide output, and any pipelines involving the sink can be skipped
+enum class SinkFinalizeType : uint8_t { READY, NO_OUTPUT_POSSIBLE };
+
+} // namespace duckdb
+
+
+namespace duckdb {
+class Event;
 class PhysicalOperator;
+class Pipeline;
+
+// LCOV_EXCL_START
+class OperatorState {
+public:
+	virtual ~OperatorState() {
+	}
 
-//! The current state/context of the operator. The PhysicalOperatorState is
-//! updated using the GetChunk function, and allows the caller to repeatedly
-//! call the GetChunk function and get new batches of data everytime until the
-//! data source is exhausted.
-class PhysicalOperatorState {
+	virtual void Finalize(PhysicalOperator *op, ExecutionContext &context) {
+	}
+};
+
+class GlobalSinkState {
 public:
-	PhysicalOperatorState(PhysicalOperator &op, PhysicalOperator *child);
-	virtual ~PhysicalOperatorState() = default;
+	GlobalSinkState() : state(SinkFinalizeType::READY) {
+	}
+	virtual ~GlobalSinkState() {
+	}
 
-	//! Flag indicating whether or not the operator is finished [note: not all
-	//! operators use this flag]
-	bool finished;
-	//! DataChunk that stores data from the child of this operator
-	DataChunk child_chunk;
-	//! State of the child of this operator
-	unique_ptr<PhysicalOperatorState> child_state;
+	SinkFinalizeType state;
 };
 
+class LocalSinkState {
+public:
+	virtual ~LocalSinkState() {
+	}
+};
+
+class GlobalSourceState {
+public:
+	virtual ~GlobalSourceState() {
+	}
+
+	virtual idx_t MaxThreads() {
+		return 1;
+	}
+};
+
+class LocalSourceState {
+public:
+	virtual ~LocalSourceState() {
+	}
+};
+// LCOV_EXCL_STOP
+
 //! PhysicalOperator is the base class of the physical operators present in the
 //! execution plan
-/*!
-    The execution model is a pull-based execution model. GetChunk is called on
-   the root node, which causes the root node to be executed, and presumably call
-   GetChunk again on its child nodes. Every node in the operator chain has a
-   state that is updated as GetChunk is called: PhysicalOperatorState (different
-   operators subclass this state and add different properties).
-*/
 class PhysicalOperator {
 public:
 	PhysicalOperator(PhysicalOperatorType type, vector<LogicalType> types, idx_t estimated_cardinality)
@@ -12512,6 +12587,8 @@ public:
 	vector<LogicalType> types;
 	//! The extimated cardinality of this physical operator
 	idx_t estimated_cardinality;
+	//! The global sink state of this operator
+	unique_ptr<GlobalSinkState> sink_state;
 
 public:
 	virtual string GetName() const;
@@ -12519,97 +12596,78 @@ public:
 		return "";
 	}
 	virtual string ToString() const;
-	void Print();
+	void Print() const;
 
 	//! Return a vector of the types that will be returned by this operator
-	vector<LogicalType> &GetTypes() {
+	const vector<LogicalType> &GetTypes() const {
 		return types;
 	}
-	//! Initialize a given chunk to the types that will be returned by this
-	//! operator, this will prepare chunk for a call to GetChunk. This method
-	//! only has to be called once for any amount of calls to GetChunk.
-	virtual void InitializeChunk(DataChunk &chunk) {
-		auto &types = GetTypes();
-		chunk.Initialize(types);
-	}
-	//! Retrieves a chunk from this operator and stores it in the chunk
-	//! variable.
-	virtual void GetChunkInternal(ExecutionContext &context, DataChunk &chunk, PhysicalOperatorState *state) const = 0;
 
-	void GetChunk(ExecutionContext &context, DataChunk &chunk, PhysicalOperatorState *state) const;
-
-	//! Create a new empty instance of the operator state
-	virtual unique_ptr<PhysicalOperatorState> GetOperatorState() {
-		return make_unique<PhysicalOperatorState>(*this, children.size() == 0 ? nullptr : children[0].get());
+	virtual bool Equals(const PhysicalOperator &other) const {
+		return false;
 	}
 
-	virtual void FinalizeOperatorState(PhysicalOperatorState &state, ExecutionContext &context) {
-		if (!children.empty() && state.child_state) {
-			children[0]->FinalizeOperatorState(*state.child_state, context);
-		}
-	}
+public:
+	// Operator interface
+	virtual unique_ptr<OperatorState> GetOperatorState(ClientContext &context) const;
+	virtual OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
+	                                   OperatorState &state) const;
 
-	virtual bool IsSink() const {
+	virtual bool ParallelOperator() const {
 		return false;
 	}
-};
-
-} // namespace duckdb
 
-
-namespace duckdb {
-
-class Pipeline;
-
-class GlobalOperatorState {
-public:
-	virtual ~GlobalOperatorState() {
+	virtual bool RequiresCache() const {
+		return false;
 	}
-};
 
-class LocalSinkState {
 public:
-	virtual ~LocalSinkState() {
-	}
-};
+	// Source interface
+	virtual unique_ptr<LocalSourceState> GetLocalSourceState(ExecutionContext &context,
+	                                                         GlobalSourceState &gstate) const;
+	virtual unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const;
+	virtual void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
+	                     LocalSourceState &lstate) const;
 
-class PhysicalSink : public PhysicalOperator {
-public:
-	PhysicalSink(PhysicalOperatorType type, vector<LogicalType> types, idx_t estimated_cardinality)
-	    : PhysicalOperator(type, move(types), estimated_cardinality) {
+	virtual bool IsSource() const {
+		return false;
 	}
 
-	unique_ptr<GlobalOperatorState> sink_state;
+	virtual bool ParallelSource() const {
+		return false;
+	}
 
 public:
+	// Sink interface
+
 	//! The sink method is called constantly with new input, as long as new input is available. Note that this method
-	//! CAN be called in parallel, proper locking is needed when accessing data inside the GlobalOperatorState.
-	virtual void Sink(ExecutionContext &context, GlobalOperatorState &gstate, LocalSinkState &lstate,
-	                  DataChunk &input) const = 0;
+	//! CAN be called in parallel, proper locking is needed when accessing data inside the GlobalSinkState.
+	virtual SinkResultType Sink(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate,
+	                            DataChunk &input) const;
 	// The combine is called when a single thread has completed execution of its part of the pipeline, it is the final
 	// time that a specific LocalSinkState is accessible. This method can be called in parallel while other Sink() or
-	// Combine() calls are active on the same GlobalOperatorState.
-	virtual void Combine(ExecutionContext &context, GlobalOperatorState &gstate, LocalSinkState &lstate) {
-	}
+	// Combine() calls are active on the same GlobalSinkState.
+	virtual void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const;
 	//! The finalize is called when ALL threads are finished execution. It is called only once per pipeline, and is
 	//! entirely single threaded.
-	virtual bool Finalize(Pipeline &pipeline, ClientContext &context, unique_ptr<GlobalOperatorState> gstate) {
-		this->sink_state = move(gstate);
-		return true;
-	}
+	//! If Finalize returns SinkResultType::FINISHED, the sink is marked as finished
+	virtual SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
+	                                  GlobalSinkState &gstate) const;
 
-	virtual unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) {
-		return make_unique<LocalSinkState>();
-	}
-	virtual unique_ptr<GlobalOperatorState> GetGlobalState(ClientContext &context) {
-		return make_unique<GlobalOperatorState>();
+	virtual unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const;
+	virtual unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const;
+
+	virtual bool IsSink() const {
+		return false;
 	}
 
-	bool IsSink() const override {
-		return true;
+	virtual bool ParallelSink() const {
+		return false;
 	}
 
-	void Schedule(ClientContext &context);
+	virtual bool SinkOrderMatters() const {
+		return false;
+	}
 };
 
 } // namespace duckdb
@@ -12668,7 +12726,9 @@ typedef void (*table_function_cleanup_t)(ClientContext &context, const FunctionD
                                          FunctionOperatorData *operator_state);
 typedef idx_t (*table_function_max_threads_t)(ClientContext &context, const FunctionData *bind_data);
 typedef unique_ptr<ParallelState> (*table_function_init_parallel_state_t)(ClientContext &context,
-                                                                          const FunctionData *bind_data);
+                                                                          const FunctionData *bind_data,
+                                                                          const vector<column_t> &column_ids,
+                                                                          TableFilterCollection *filters);
 typedef unique_ptr<FunctionOperatorData> (*table_function_init_parallel_t)(ClientContext &context,
                                                                            const FunctionData *bind_data,
                                                                            ParallelState *state,
@@ -12820,7 +12880,10 @@ struct ParallelState {
 
 
 namespace duckdb {
+class ClientContext;
+class Executor;
 
+//! Generic parallel task
 class Task {
 public:
 	virtual ~Task() {
@@ -12830,6 +12893,21 @@ public:
 	virtual void Execute() = 0;
 };
 
+//! Execute a task within an executor, including exception handling
+//! This should be used within queries
+class ExecutorTask : public Task {
+public:
+	ExecutorTask(Executor &executor);
+	ExecutorTask(ClientContext &context);
+	virtual ~ExecutorTask();
+
+	Executor &executor;
+
+public:
+	virtual void ExecuteTask() = 0;
+	void Execute() override;
+};
+
 } // namespace duckdb
 
 
@@ -12894,109 +12972,87 @@ private:
 
 namespace duckdb {
 class Executor;
-class TaskContext;
+class Event;
 
 //! The Pipeline class represents an execution pipeline
 class Pipeline : public std::enable_shared_from_this<Pipeline> {
 	friend class Executor;
+	friend class PipelineExecutor;
+	friend class PipelineEvent;
+	friend class PipelineFinishEvent;
 
 public:
-	Pipeline(Executor &execution_context, ProducerToken &token);
+	Pipeline(Executor &execution_context);
 
 	Executor &executor;
-	ProducerToken &token;
 
 public:
-	//! Execute a task within the pipeline on a single thread
-	void Execute(TaskContext &task);
+	ClientContext &GetClientContext();
 
 	void AddDependency(shared_ptr<Pipeline> &pipeline);
-	void CompleteDependency();
-	bool HasDependencies() {
-		return !dependencies.empty();
-	}
 
-	void Reset(ClientContext &context);
-	void Schedule();
+	void Ready();
+	void Reset();
+	void ResetSource();
+	void Schedule(shared_ptr<Event> &event);
 
-	//! Finish a single task of this pipeline
-	void FinishTask();
-	//! Finish executing this pipeline
-	void Finish();
+	//! Finalize this pipeline
+	void Finalize(Event &event);
 
 	string ToString() const;
 	void Print() const;
 
-	void SetRecursiveCTE(PhysicalOperator *op) {
-		this->recursive_cte = op;
-	}
-	PhysicalOperator *GetRecursiveCTE() {
-		return recursive_cte;
-	}
-	void ClearParents();
-
-	void IncrementTasks(idx_t amount) {
-		this->total_tasks += amount;
-	}
-
-	bool IsFinished() {
-		return finished;
-	}
 	//! Returns query progress
 	bool GetProgress(int &current_percentage);
 
-public:
-	//! The current threads working on the pipeline
-	atomic<idx_t> finished_tasks;
-	//! The maximum amount of threads that can work on the pipeline
-	atomic<idx_t> total_tasks;
+	//! Returns a list of all operators (including source and sink) involved in this pipeline
+	vector<PhysicalOperator *> GetOperators() const;
+
+	PhysicalOperator *GetSink() {
+		return sink;
+	}
 
 private:
-	//! The child from which to pull chunks
-	PhysicalOperator *child;
-	//! The global sink state
-	unique_ptr<GlobalOperatorState> sink_state;
+	//! Whether or not the pipeline has been readied
+	bool ready;
+	//! The source of this pipeline
+	PhysicalOperator *source;
+	//! The chain of intermediate operators
+	vector<PhysicalOperator *> operators;
 	//! The sink (i.e. destination) for data; this is e.g. a hash table to-be-built
-	PhysicalSink *sink;
+	PhysicalOperator *sink;
+
+	//! The global source state
+	unique_ptr<GlobalSourceState> source_state;
+
 	//! The parent pipelines (i.e. pipelines that are dependent on this pipeline to finish)
-	unordered_map<Pipeline *, weak_ptr<Pipeline>> parents;
+	vector<weak_ptr<Pipeline>> parents;
 	//! The dependencies of this pipeline
-	unordered_map<Pipeline *, weak_ptr<Pipeline>> dependencies;
-	//! The amount of completed dependencies (the pipeline can only be started after the dependencies have finished
-	//! executing)
-	atomic<idx_t> finished_dependencies;
-
-	//! The parallel operator (if any)
-	PhysicalOperator *parallel_node;
-	//! The parallel state (if any)
-	unique_ptr<ParallelState> parallel_state;
-
-	//! Whether or not the pipeline is finished executing
-	bool finished;
-	//! The recursive CTE node that this pipeline belongs to, and may be executed multiple times
-	PhysicalOperator *recursive_cte;
+	vector<weak_ptr<Pipeline>> dependencies;
 
 private:
-	bool GetProgress(ClientContext &context, PhysicalOperator *op, int &current_percentage);
-	void ScheduleSequentialTask();
-	bool LaunchScanTasks(PhysicalOperator *op, idx_t max_threads, unique_ptr<ParallelState> parallel_state);
-	bool ScheduleOperator(PhysicalOperator *op);
+	bool GetProgressInternal(ClientContext &context, PhysicalOperator *op, int &current_percentage);
+	void ScheduleSequentialTask(shared_ptr<Event> &event);
+	bool LaunchScanTasks(shared_ptr<Event> &event, idx_t max_threads);
+
+	bool ScheduleParallel(shared_ptr<Event> &event);
 };
 
 } // namespace duckdb
 
 
 
-#include <queue>
 
 namespace duckdb {
 class ClientContext;
 class DataChunk;
 class PhysicalOperator;
-class PhysicalOperatorState;
+class PipelineExecutor;
+class OperatorState;
 class ThreadContext;
 class Task;
 
+struct PipelineEventStack;
 struct ProducerToken;
 
 class Executor {
@@ -13010,8 +13066,10 @@ public:
 	ClientContext &context;
 
 public:
+	static Executor &Get(ClientContext &context);
+
 	void Initialize(PhysicalOperator *physical_plan);
-	void BuildPipelines(PhysicalOperator *op, Pipeline *parent);
+	void BuildPipelines(PhysicalOperator *op, Pipeline *current);
 
 	void Reset();
 
@@ -13020,8 +13078,15 @@ public:
 	unique_ptr<DataChunk> FetchChunk();
 
 	//! Push a new error
-	void PushError(const string &exception);
-	bool GetError(string &exception);
+	void PushError(ExceptionType type, const string &exception);
+	//! True if an error has been thrown
+	bool HasError();
+	//! Throw the exception that was pushed using PushError.
+	//! Should only be called if HasError returns true
+	void ThrowException();
+
+	//! Work on tasks for this specific executor, until there are no tasks remaining
+	void WorkOnTasks();
 
 	//! Flush a thread context into the client context
 	void Flush(ThreadContext &context);
@@ -13029,24 +13094,79 @@ public:
 	//! Returns the progress of the pipelines
 	bool GetPipelinesProgress(int &current_progress);
 
+	void CompletePipeline() {
+		completed_pipelines++;
+	}
+	ProducerToken &GetToken() {
+		return *producer;
+	}
+	void AddEvent(shared_ptr<Event> event);
+
+	void ReschedulePipelines(const vector<shared_ptr<Pipeline>> &pipelines, vector<shared_ptr<Event>> &events);
+
+private:
+	void ScheduleEvents();
+	void ScheduleEventsInternal(const vector<shared_ptr<Pipeline>> &pipelines,
+	                            unordered_map<Pipeline *, vector<shared_ptr<Pipeline>>> &child_pipelines,
+	                            vector<shared_ptr<Event>> &events, bool main_schedule = true);
+
+	void SchedulePipeline(const shared_ptr<Pipeline> &pipeline,
+	                      unordered_map<Pipeline *, PipelineEventStack> &event_map, vector<shared_ptr<Event>> &events,
+	                      bool complete_pipeline);
+	void ScheduleUnionPipeline(const shared_ptr<Pipeline> &pipeline, PipelineEventStack &stack,
+	                           unordered_map<Pipeline *, PipelineEventStack> &event_map,
+	                           vector<shared_ptr<Event>> &events);
+	void ScheduleChildPipeline(Pipeline *parent, const shared_ptr<Pipeline> &pipeline,
+	                           unordered_map<Pipeline *, PipelineEventStack> &event_map,
+	                           vector<shared_ptr<Event>> &events);
+	void ExtractPipelines(shared_ptr<Pipeline> &pipeline, vector<shared_ptr<Pipeline>> &result);
+	bool NextExecutor();
+
+	void AddChildPipeline(Pipeline *current);
+
+	void VerifyPipeline(Pipeline &pipeline);
+	void VerifyPipelines();
+	void ThrowExceptionInternal();
+
 private:
 	PhysicalOperator *physical_plan;
-	unique_ptr<PhysicalOperatorState> physical_state;
 
 	mutex executor_lock;
 	//! The pipelines of the current query
 	vector<shared_ptr<Pipeline>> pipelines;
+	//! The root pipeline of the query
+	vector<shared_ptr<Pipeline>> root_pipelines;
+	//! The pipeline executor for the root pipeline
+	unique_ptr<PipelineExecutor> root_executor;
+	//! The current root pipeline index
+	idx_t root_pipeline_idx;
 	//! The producer of this query
 	unique_ptr<ProducerToken> producer;
 	//! Exceptions that occurred during the execution of the current query
-	vector<string> exceptions;
+	vector<pair<ExceptionType, string>> exceptions;
+	//! List of events
+	vector<shared_ptr<Event>> events;
 
 	//! The amount of completed pipelines of the query
 	atomic<idx_t> completed_pipelines;
 	//! The total amount of pipelines in the query
 	idx_t total_pipelines;
 
+	//! The adjacent union pipelines of each pipeline
+	//! Union pipelines have the same sink, but can be run concurrently along with this pipeline
+	unordered_map<Pipeline *, vector<shared_ptr<Pipeline>>> union_pipelines;
+	//! Child pipelines of this pipeline
+	//! Like union pipelines, child pipelines share the same sink
+	//! Unlike union pipelines, child pipelines should be run AFTER their dependencies are completed
+	//! i.e. they should be run after the dependencies are completed, but before finalize is called on the sink
+	unordered_map<Pipeline *, vector<shared_ptr<Pipeline>>> child_pipelines;
+	//! Dependencies of child pipelines
+	unordered_map<Pipeline *, vector<Pipeline *>> child_dependencies;
+
+	//! Duplicate eliminated join scan dependencies
 	unordered_map<PhysicalOperator *, Pipeline *> delim_join_dependencies;
+
+	//! Active recursive CTE node (if any)
 	PhysicalOperator *recursive_cte;
 };
 } // namespace duckdb