duckdb 0.9.3-dev0.0 → 0.9.3-dev14.0

package/src/duckdb/src/include/duckdb/main/secret/secret_storage.hpp

@@ -0,0 +1,164 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/main/secret/secret_storage.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/enums/on_entry_not_found.hpp"
+
+namespace duckdb {
+
+class BaseSecret;
+class Catalog;
+class CatalogSet;
+struct CatalogTransaction;
+class DatabaseInstance;
+struct SecretMatch;
+struct SecretEntry;
+
+//! Base class for SecretStorage API
+class SecretStorage {
+	friend class SecretManager;
+
+public:
+	SecretStorage(const string &name) : storage_name(name), persistent(false) {};
+	virtual ~SecretStorage() = default;
+
+public:
+	//! SecretStorage API
+
+	//! Get the storage name (e.g. local_file, :memory:)
+	virtual string &GetName() {
+		return storage_name;
+	};
+
+	//! Store a secret
+	virtual unique_ptr<SecretEntry> StoreSecret(unique_ptr<const BaseSecret> secret, OnCreateConflict on_conflict,
+	                                            optional_ptr<CatalogTransaction> transaction = nullptr) = 0;
+	//! Get all secrets
+	virtual vector<SecretEntry> AllSecrets(optional_ptr<CatalogTransaction> transaction = nullptr) = 0;
+	//! Drop secret by name
+	virtual void DropSecretByName(const string &name, OnEntryNotFound on_entry_not_found,
+	                              optional_ptr<CatalogTransaction> transaction = nullptr) = 0;
+	//! Get best match
+	virtual SecretMatch LookupSecret(const string &path, const string &type,
+	                                 optional_ptr<CatalogTransaction> transaction = nullptr) = 0;
+	//! Get a secret by name
+	virtual unique_ptr<SecretEntry> GetSecretByName(const string &name,
+	                                                optional_ptr<CatalogTransaction> transaction = nullptr) = 0;
+
+	//! Return the offset associated to this storage for tie-breaking secrets between storages
+	virtual int64_t GetTieBreakOffset() = 0;
+
+	//! Returns include_in_lookups, used to create secret storage
+	virtual bool IncludeInLookups() {
+		return true;
+	}
+
+	virtual bool Persistent() const {
+		return persistent;
+	}
+
+protected:
+	//! Helper function to select the best matching secret within a storage. Tie-breaks within a storage are broken
+	//! by secret name by default.
+	SecretMatch SelectBestMatch(SecretEntry &secret_entry, const string &path, SecretMatch &current_best);
+
+	//! Offsets the score to tie-break secrets giving preference to the storage with the lowest storage_penalty
+	//! the base implementation will be chosen last in a tie-break
+	int64_t OffsetMatchScore(int64_t score) {
+		return 100 * score - GetTieBreakOffset();
+	}
+
+	//! Name of the storage backend (e.g. temporary, file, etc)
+	string storage_name;
+	//! Whether entries in this storage will survive duckdb reboots
+	bool persistent;
+};
+
+//! Wrapper struct around a SecretEntry to allow storing it
+struct SecretCatalogEntry : public InCatalogEntry {
+public:
+	SecretCatalogEntry(unique_ptr<SecretEntry> secret_p, Catalog &catalog);
+	SecretCatalogEntry(unique_ptr<const BaseSecret> secret_p, Catalog &catalog);
+
+	//! The secret to store in a catalog
+	unique_ptr<SecretEntry> secret;
+};
+
+//! Base Implementation for catalog set based secret storage
+class CatalogSetSecretStorage : public SecretStorage {
+public:
+	CatalogSetSecretStorage(DatabaseInstance &db_instance, const string &name_p)
+	    : SecretStorage(name_p), db(db_instance) {};
+
+public:
+	//! SecretStorage API
+	string &GetName() override {
+		return storage_name;
+	};
+
+	virtual unique_ptr<SecretEntry> StoreSecret(unique_ptr<const BaseSecret> secret, OnCreateConflict on_conflict,
+	                                            optional_ptr<CatalogTransaction> transaction = nullptr) override;
+	vector<SecretEntry> AllSecrets(optional_ptr<CatalogTransaction> transaction = nullptr) override;
+	void DropSecretByName(const string &name, OnEntryNotFound on_entry_not_found,
+	                      optional_ptr<CatalogTransaction> transaction = nullptr) override;
+	SecretMatch LookupSecret(const string &path, const string &type,
+	                         optional_ptr<CatalogTransaction> transaction = nullptr) override;
+	unique_ptr<SecretEntry> GetSecretByName(const string &name,
+	                                        optional_ptr<CatalogTransaction> transaction = nullptr) override;
+
+protected:
+	//! Callback called on Store to allow child classes to implement persistence.
+	virtual void WriteSecret(const BaseSecret &secret, OnCreateConflict on_conflict);
+	virtual void RemoveSecret(const string &name, OnEntryNotFound on_entry_not_found);
+	//! Returns the CatalogTransaction in `transaction` if not set, return the System transaction
+	CatalogTransaction GetTransactionOrDefault(optional_ptr<CatalogTransaction> transaction);
+
+	//! CatalogSet containing the secrets
+	unique_ptr<CatalogSet> secrets;
+	//! DB instance for accessing the system catalog transaction
+	DatabaseInstance &db;
+};
+
+class TemporarySecretStorage : public CatalogSetSecretStorage {
+public:
+	TemporarySecretStorage(const string &name_p, DatabaseInstance &db_p) : CatalogSetSecretStorage(db_p, name_p) {
+		secrets = make_uniq<CatalogSet>(Catalog::GetSystemCatalog(db));
+		persistent = false;
+	}
+
+	int64_t GetTieBreakOffset() override {
+		return 10;
+	}
+
+protected:
+};
+
+class LocalFileSecretStorage : public CatalogSetSecretStorage {
+public:
+	LocalFileSecretStorage(SecretManager &manager, DatabaseInstance &db, const string &name_p,
+	                       const string &secret_path);
+
+	int64_t GetTieBreakOffset() override {
+		return 20;
+	}
+
+protected:
+	//! Implements the writes to disk
+	void WriteSecret(const BaseSecret &secret, OnCreateConflict on_conflict) override;
+	//! Implements the deletes from disk
+	virtual void RemoveSecret(const string &secret, OnEntryNotFound on_entry_not_found) override;
+
+	//! Set of persistent secrets that are lazily loaded
+	case_insensitive_set_t persistent_secrets;
+	//! Path that is searched for secrets;
+	string secret_path;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/main/settings.hpp

@@ -16,6 +16,8 @@ class ClientContext;
 class DatabaseInstance;
 struct DBConfig;
 
+const string GetDefaultUserAgent();
+
 struct AccessModeSetting {
 	static constexpr const char *Name = "access_mode";
 	static constexpr const char *Description = "Access mode of the database (AUTOMATIC, READ_ONLY or READ_WRITE)";
@@ -25,6 +27,16 @@ struct AccessModeSetting {
 	static Value GetSetting(ClientContext &context);
 };
 
+struct AllowPersistentSecrets {
+	static constexpr const char *Name = "allow_persistent_secrets";
+	static constexpr const char *Description =
+	    "Allow the creation of persistent secrets, that are stored and loaded on restarts";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(ClientContext &context);
+};
+
 struct CheckpointThresholdSetting {
 	static constexpr const char *Name = "checkpoint_threshold";
 	static constexpr const char *Description =
@@ -131,6 +143,15 @@ struct DefaultNullOrderSetting {
 	static Value GetSetting(ClientContext &context);
 };
 
+struct DefaultSecretStorage {
+	static constexpr const char *Name = "default_secret_storage";
+	static constexpr const char *Description = "Allows switching the default storage for secrets";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(ClientContext &context);
+};
+
 struct DisabledFileSystemsSetting {
 	static constexpr const char *Name = "disabled_filesystems";
 	static constexpr const char *Description = "Disable specific file systems preventing access (e.g. LocalFileSystem)";
@@ -183,8 +204,8 @@ struct CustomExtensionRepository {
 	static constexpr const char *Name = "custom_extension_repository";
 	static constexpr const char *Description = "Overrides the custom endpoint for remote extension installation";
 	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
-	static void SetLocal(ClientContext &context, const Value &parameter);
-	static void ResetLocal(ClientContext &context);
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
 	static Value GetSetting(ClientContext &context);
 };
 
@@ -193,8 +214,8 @@ struct AutoloadExtensionRepository {
 	static constexpr const char *Description =
 	    "Overrides the custom endpoint for extension installation on autoloading";
 	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
-	static void SetLocal(ClientContext &context, const Value &parameter);
-	static void ResetLocal(ClientContext &context);
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
 	static Value GetSetting(ClientContext &context);
 };
 
@@ -266,6 +287,15 @@ struct EnableProgressBarPrintSetting {
 	static Value GetSetting(ClientContext &context);
 };
 
+struct ErrorsAsJsonSetting {
+	static constexpr const char *Name = "errors_as_json";
+	static constexpr const char *Description = "Output error messages as structured JSON instead of as a raw string";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
+	static void SetLocal(ClientContext &context, const Value &parameter);
+	static void ResetLocal(ClientContext &context);
+	static Value GetSetting(ClientContext &context);
+};
+
 struct ExplainOutputSetting {
 	static constexpr const char *Name = "explain_output";
 	static constexpr const char *Description = "Output of EXPLAIN statements (ALL, OPTIMIZED_ONLY, PHYSICAL_ONLY)";
@@ -388,6 +418,15 @@ struct MaximumMemorySetting {
 	static Value GetSetting(ClientContext &context);
 };
 
+struct OldImplicitCasting {
+	static constexpr const char *Name = "old_implicit_casting";
+	static constexpr const char *Description = "Allow implicit casting to/from VARCHAR";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(ClientContext &context);
+};
+
 struct PasswordSetting {
 	static constexpr const char *Name = "password";
 	static constexpr const char *Description = "The password to use. Ignored for legacy compatibility.";
@@ -457,15 +496,6 @@ struct ExportLargeBufferArrow {
 	static Value GetSetting(ClientContext &context);
 };
 
-struct ProfilerHistorySize {
-	static constexpr const char *Name = "profiler_history_size";
-	static constexpr const char *Description = "Sets the profiler history size";
-	static constexpr const LogicalTypeId InputType = LogicalTypeId::BIGINT;
-	static void SetLocal(ClientContext &context, const Value &parameter);
-	static void ResetLocal(ClientContext &context);
-	static Value GetSetting(ClientContext &context);
-};
-
 struct ProfileOutputSetting {
 	static constexpr const char *Name = "profile_output";
 	static constexpr const char *Description =
@@ -515,6 +545,15 @@ struct SearchPathSetting {
 	static Value GetSetting(ClientContext &context);
 };
 
+struct SecretDirectorySetting {
+	static constexpr const char *Name = "secret_directory";
+	static constexpr const char *Description = "Set the directory to which persistent secrets are stored";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(ClientContext &context);
+};
+
 struct TempDirectorySetting {
 	static constexpr const char *Name = "temp_directory";
 	static constexpr const char *Description = "Set the directory to which to write temp files";

package/src/duckdb/src/include/duckdb/main/stream_query_result.hpp

@@ -10,6 +10,9 @@
 
 #include "duckdb/common/winapi.hpp"
 #include "duckdb/main/query_result.hpp"
+#include "duckdb/parallel/interrupt.hpp"
+#include "duckdb/common/queue.hpp"
+#include "duckdb/main/buffered_data/simple_buffered_data.hpp"
 
 namespace duckdb {
 
@@ -29,7 +32,8 @@ public:
 	//! Create a successful StreamQueryResult. StreamQueryResults should always be successful initially (it makes no
 	//! sense to stream an error).
 	DUCKDB_API StreamQueryResult(StatementType statement_type, StatementProperties properties,
-	                             shared_ptr<ClientContext> context, vector<LogicalType> types, vector<string> names);
+	                             vector<LogicalType> types, vector<string> names, ClientProperties client_properties,
+	                             shared_ptr<BufferedData> buffered_data);
 	DUCKDB_API ~StreamQueryResult() override;
 
 public:
@@ -49,9 +53,13 @@ public:
 	shared_ptr<ClientContext> context;
 
 private:
+	unique_ptr<DataChunk> FetchInternal(ClientContextLock &lock);
 	unique_ptr<ClientContextLock> LockContext();
 	void CheckExecutableInternal(ClientContextLock &lock);
 	bool IsOpenInternal(ClientContextLock &lock);
+
+private:
+	shared_ptr<BufferedData> buffered_data;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp

@@ -25,13 +25,16 @@ public:
 private:
 	//! Finds DelimJoins and their corresponding DelimGets
 	void FindCandidates(unique_ptr<LogicalOperator> &op, vector<DelimCandidate> &candidates);
-	void FindJoinWithDelimGet(unique_ptr<LogicalOperator> &op, DelimCandidate &candidate);
+	void FindJoinWithDelimGet(unique_ptr<LogicalOperator> &op, DelimCandidate &candidate, idx_t depth = 0);
+	//! Whether the DelimJoin is selective
+	bool HasSelection(const LogicalOperator &delim_join);
 	//! Remove joins with a DelimGet
 	bool RemoveJoinWithDelimGet(LogicalComparisonJoin &delim_join, const idx_t delim_get_count,
 	                            unique_ptr<LogicalOperator> &join, bool &all_equality_conditions);
 	bool RemoveInequalityJoinWithDelimGet(LogicalComparisonJoin &delim_join, const idx_t delim_get_count,
 	                                      unique_ptr<LogicalOperator> &join,
 	                                      const vector<ReplacementBinding> &replacement_bindings);
+	void TrySwitchSingleToLeft(LogicalComparisonJoin &delim_join);
 
 private:
 	optional_ptr<LogicalOperator> root;

package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp

@@ -65,6 +65,8 @@ private:
 	// Pushdown a left join
 	unique_ptr<LogicalOperator> PushdownLeftJoin(unique_ptr<LogicalOperator> op, unordered_set<idx_t> &left_bindings,
 	                                             unordered_set<idx_t> &right_bindings);
+
+	unique_ptr<LogicalOperator> PushdownSemiAntiJoin(unique_ptr<LogicalOperator> op);
 	// Pushdown a mark join
 	unique_ptr<LogicalOperator> PushdownMarkJoin(unique_ptr<LogicalOperator> op, unordered_set<idx_t> &left_bindings,
 	                                             unordered_set<idx_t> &right_bindings);

package/src/duckdb/src/include/duckdb/optimizer/join_order/join_node.hpp

@@ -46,7 +46,7 @@ public:
 
 private:
 public:
-	void PrintJoinNode();
+	void Print();
 	string ToString();
 };
 

package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp

@@ -49,6 +49,8 @@ public:
 	JoinRelationSet &Union(JoinRelationSet &left, JoinRelationSet &right);
 	// //! Create the set difference of left \ right (i.e. all elements in left that are not in right)
 	// JoinRelationSet *Difference(JoinRelationSet *left, JoinRelationSet *right);
+	string ToString() const;
+	void Print();
 
 private:
 	JoinRelationTreeNode root;

package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph_manager.hpp

@@ -11,9 +11,11 @@
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/optional_ptr.hpp"
 #include "duckdb/common/pair.hpp"
+#include "duckdb/common/enums/join_type.hpp"
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/common/unordered_set.hpp"
 #include "duckdb/common/vector.hpp"
+
 #include "duckdb/optimizer/join_order/join_node.hpp"
 #include "duckdb/optimizer/join_order/join_relation.hpp"
 #include "duckdb/optimizer/join_order/query_graph.hpp"
@@ -88,6 +90,7 @@ public:
 	//! (Basically we put lower expected cardinality columns on the build side, and larger
 	//! tables on the probe side)
 	unique_ptr<LogicalOperator> LeftRightOptimizations(unique_ptr<LogicalOperator> op);
+	void TryFlipChildren(LogicalOperator &op, idx_t cardinality_ratio = 1);
 
 private:
 	vector<reference<LogicalOperator>> filter_operators;

package/src/duckdb/src/include/duckdb/optimizer/join_order/relation_statistics_helper.hpp

@@ -60,6 +60,7 @@ public:
 	static RelationStats ExtractProjectionStats(LogicalProjection &proj, RelationStats &child_stats);
 	static RelationStats ExtractAggregationStats(LogicalAggregate &aggr, RelationStats &child_stats);
 	static RelationStats ExtractWindowStats(LogicalWindow &window, RelationStats &child_stats);
+	static RelationStats ExtractEmptyResultStats(LogicalEmptyResult &empty);
 	//! Called after reordering a query plan with potentially 2+ relations.
 	static RelationStats CombineStatsOfReorderableOperator(vector<ColumnBinding> &bindings,
 	                                                       vector<RelationStats> relation_stats);

package/src/duckdb/src/include/duckdb/parallel/event.hpp

@@ -37,7 +37,7 @@ public:
 	bool HasDependencies() const {
 		return total_dependencies != 0;
 	}
-	const vector<Event *> &GetParentsVerification() const;
+	const vector<reference<Event>> &GetParentsVerification() const;
 
 	void CompleteDependency();
 
@@ -79,7 +79,7 @@ protected:
 	//! The events that depend on this event to run
 	vector<weak_ptr<Event>> parents;
 	//! Raw pointers to the parents (used for verification only)
-	vector<Event *> parents_raw;
+	vector<reference<Event>> parents_raw;
 
 	//! Whether or not the event is finished executing
 	atomic<bool> finished;

package/src/duckdb/src/include/duckdb/parallel/meta_pipeline.hpp

@@ -8,19 +8,11 @@
 
 #pragma once
 
+#include "duckdb/common/reference_map.hpp"
 #include "duckdb/execution/physical_operator.hpp"
 
 namespace duckdb {
 
-class PhysicalRecursiveCTE;
-
-struct PipelineFinishGroup {
-	explicit PipelineFinishGroup(Pipeline *group_base_p) : group_base(group_base_p) {
-	}
-	Pipeline *group_base;
-	unordered_set<Pipeline *> group_members;
-};
-
 //! MetaPipeline represents a set of pipelines that all have the same sink
 class MetaPipeline : public std::enable_shared_from_this<MetaPipeline> {
 	//! We follow these rules when building:
@@ -34,7 +26,7 @@ class MetaPipeline : public std::enable_shared_from_this<MetaPipeline> {
 	//!         * And all pipelines that were added to the MetaPipeline after 'current'
 public:
 	//! Create a MetaPipeline with the given sink
-	explicit MetaPipeline(Executor &executor, PipelineBuildState &state, PhysicalOperator *sink);
+	MetaPipeline(Executor &executor, PipelineBuildState &state, optional_ptr<PhysicalOperator> sink);
 
 public:
 	//! Get the Executor for this MetaPipeline
@@ -51,22 +43,22 @@ public:
 	//! Get the MetaPipeline children of this MetaPipeline
 	void GetMetaPipelines(vector<shared_ptr<MetaPipeline>> &result, bool recursive, bool skip);
 	//! Get the dependencies (within this MetaPipeline) of the given Pipeline
-	const vector<Pipeline *> *GetDependencies(Pipeline *dependant) const;
+	optional_ptr<const vector<reference<Pipeline>>> GetDependencies(Pipeline &dependant) const;
 	//! Whether this MetaPipeline has a recursive CTE
 	bool HasRecursiveCTE() const;
 	//! Set the flag that this MetaPipeline is a recursive CTE pipeline
 	void SetRecursiveCTE();
 	//! Assign a batch index to the given pipeline
-	void AssignNextBatchIndex(Pipeline *pipeline);
+	void AssignNextBatchIndex(Pipeline &pipeline);
 	//! Let 'dependant' depend on all pipeline that were created since 'start',
 	//! where 'including' determines whether 'start' is added to the dependencies
-	void AddDependenciesFrom(Pipeline *dependant, Pipeline *start, bool including);
+	void AddDependenciesFrom(Pipeline &dependant, Pipeline &start, bool including);
 	//! Make sure that the given pipeline has its own PipelineFinishEvent (e.g., for IEJoin - double Finalize)
-	void AddFinishEvent(Pipeline *pipeline);
+	void AddFinishEvent(Pipeline &pipeline);
 	//! Whether the pipeline needs its own PipelineFinishEvent
-	bool HasFinishEvent(Pipeline *pipeline) const;
+	bool HasFinishEvent(Pipeline &pipeline) const;
 	//! Whether this pipeline is part of a PipelineFinishEvent
-	optional_ptr<Pipeline> GetFinishGroup(Pipeline *pipeline) const;
+	optional_ptr<Pipeline> GetFinishGroup(Pipeline &pipeline) const;
 
 public:
 	//! Build the MetaPipeline with 'op' as the first operator (excl. the shared sink)
@@ -75,12 +67,12 @@ public:
 	void Ready();
 
 	//! Create an empty pipeline within this MetaPipeline
-	Pipeline *CreatePipeline();
+	Pipeline &CreatePipeline();
 	//! Create a union pipeline (clone of 'current')
-	Pipeline *CreateUnionPipeline(Pipeline &current, bool order_matters);
+	Pipeline &CreateUnionPipeline(Pipeline &current, bool order_matters);
 	//! Create a child pipeline op 'current' starting at 'op',
 	//! where 'last_pipeline' is the last pipeline added before building out 'current'
-	void CreateChildPipeline(Pipeline &current, PhysicalOperator &op, Pipeline *last_pipeline);
+	void CreateChildPipeline(Pipeline &current, PhysicalOperator &op, Pipeline &last_pipeline);
 	//! Create a MetaPipeline child that 'current' depends on
 	MetaPipeline &CreateChildMetaPipeline(Pipeline &current, PhysicalOperator &op);
 
@@ -96,15 +88,15 @@ private:
 	//! All pipelines with a different source, but the same sink
 	vector<shared_ptr<Pipeline>> pipelines;
 	//! Dependencies within this MetaPipeline
-	unordered_map<Pipeline *, vector<Pipeline *>> dependencies;
+	reference_map_t<Pipeline, vector<reference<Pipeline>>> dependencies;
 	//! Other MetaPipelines that this MetaPipeline depends on
 	vector<shared_ptr<MetaPipeline>> children;
 	//! Next batch index
 	idx_t next_batch_index;
 	//! Pipelines (other than the base pipeline) that need their own PipelineFinishEvent (e.g., for IEJoin)
-	unordered_set<Pipeline *> finish_pipelines;
+	reference_set_t<Pipeline> finish_pipelines;
 	//! Mapping from pipeline (e.g., child or union) to finish pipeline
-	unordered_map<Pipeline *, Pipeline *> finish_map;
+	reference_map_t<Pipeline, Pipeline &> finish_map;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp

@@ -21,6 +21,26 @@ namespace duckdb {
 class Executor;
 class Event;
 class MetaPipeline;
+class PipelineExecutor;
+class Pipeline;
+
+class PipelineTask : public ExecutorTask {
+	static constexpr const idx_t PARTIAL_CHUNK_COUNT = 50;
+
+public:
+	explicit PipelineTask(Pipeline &pipeline_p, shared_ptr<Event> event_p);
+
+	Pipeline &pipeline;
+	shared_ptr<Event> event;
+	unique_ptr<PipelineExecutor> pipeline_executor;
+
+public:
+	const PipelineExecutor &GetPipelineExecutor() const;
+	bool TaskBlockedOnResult() const override;
+
+public:
+	TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override;
+};
 
 class PipelineBuildState {
 public:

package/src/duckdb/src/include/duckdb/parallel/pipeline_executor.hpp

@@ -51,14 +51,12 @@ public:
 	//! This should only be called once per PipelineExecutor.
 	PipelineExecuteResult PushFinalize();
 
-	//! Initializes a chunk with the types that will flow out of ExecutePull
+	bool RemainingSinkChunk() const;
+
+	//! Initializes a chunk with the types that will flow out of the chunk
 	void InitializeChunk(DataChunk &chunk);
 	//! Execute a pipeline without a sink, and retrieve a single DataChunk
 	//! Returns an empty chunk when finished.
-	void ExecutePull(DataChunk &result);
-	//! Called after depleting the source using ExecutePull
-	//! This flushes profiler states
-	void PullFinalize();
 
 	//! Registers the task in the interrupt_state to allow Source/Sink operators to block the task
 	void SetTaskForInterrupts(weak_ptr<Task> current_task);

package/src/duckdb/src/include/duckdb/parallel/task.hpp

@@ -27,6 +27,7 @@ public:
 	virtual ~Task() {
 	}
 
+public:
 	//! Execute the task in the specified execution mode
 	//! If mode is PROCESS_ALL, Execute should always finish processing and return TASK_FINISHED
 	//! If mode is PROCESS_PARTIAL, Execute can return TASK_NOT_FINISHED, in which case Execute will be called again
@@ -45,6 +46,10 @@ public:
 	virtual void Reschedule() {
 		throw InternalException("Cannot reschedule task of base Task class");
 	}
+
+	virtual bool TaskBlockedOnResult() const {
+		return false;
+	}
 };
 
 //! Execute a task within an executor, including exception handling
@@ -55,9 +60,11 @@ public:
 	ExecutorTask(ClientContext &context);
 	virtual ~ExecutorTask();
 
+public:
 	void Deschedule() override;
 	void Reschedule() override;
 
+public:
 	Executor &executor;
 
 public:

package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp

@@ -58,9 +58,13 @@ public:
 	//! Run tasks until `max_tasks` have been completed, or until there are no more tasks available
 	void ExecuteTasks(idx_t max_tasks);
 
-	//! Sets the amount of active threads executing tasks for the system; n-1 background threads will be launched.
-	//! The main thread will also be used for execution
-	void SetThreads(int32_t n);
+	//! Sets the amount of background threads to be used for execution, based on the number of total threads
+	//! and the number of external threads. External threads, e.g. the main thread, will also be used for execution.
+	//! Launches `total_threads - external_threads` background worker threads.
+	void SetThreads(idx_t total_threads, idx_t external_threads);
+
+	void RelaunchThreads();
+
 	//! Returns the number of threads
 	DUCKDB_API int32_t NumberOfThreads();
 
@@ -74,7 +78,7 @@ public:
 	void SetAllocatorFlushTreshold(idx_t threshold);
 
 private:
-	void SetThreadsInternal(int32_t n);
+	void RelaunchThreadsInternal(int32_t n);
 
 private:
 	DatabaseInstance &db;
@@ -88,6 +92,8 @@ private:
 	vector<unique_ptr<atomic<bool>>> markers;
 	//! The threshold after which to flush the allocator after completing a task
 	atomic<idx_t> allocator_flush_threshold;
+	//! Requested thread count
+	atomic<int32_t> thread_count;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/parser/base_expression.hpp

@@ -11,6 +11,7 @@
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/enums/expression_type.hpp"
 #include "duckdb/common/exception.hpp"
+#include "duckdb/common/optional_idx.hpp"
 
 namespace duckdb {
 
@@ -40,6 +41,8 @@ public:
 	ExpressionClass expression_class;
 	//! The alias of the expression,
 	string alias;
+	//! The location in the query (if any)
+	optional_idx query_location;
 
 public:
 	//! Returns true if this expression is an aggregate or not.