duckdb 1.3.4 → 1.4.1-dev2.0

package/src/duckdb/src/execution/operator/projection/physical_tableinout_function.cpp

@@ -11,6 +11,7 @@ public:
 	idx_t row_index;
 	bool new_row;
 	DataChunk input_chunk;
+	idx_t current_ordinality_idx = 1;
 };
 
 class TableInOutGlobalState : public GlobalOperatorState {
@@ -21,11 +22,11 @@ public:
 	unique_ptr<GlobalTableFunctionState> global_state;
 };
 
-PhysicalTableInOutFunction::PhysicalTableInOutFunction(vector<LogicalType> types, TableFunction function_p,
-                                                       unique_ptr<FunctionData> bind_data_p,
+PhysicalTableInOutFunction::PhysicalTableInOutFunction(PhysicalPlan &physical_plan, vector<LogicalType> types,
+                                                       TableFunction function_p, unique_ptr<FunctionData> bind_data_p,
                                                        vector<ColumnIndex> column_ids_p, idx_t estimated_cardinality,
                                                        vector<column_t> project_input_p)
-    : PhysicalOperator(PhysicalOperatorType::INOUT_FUNCTION, std::move(types), estimated_cardinality),
+    : PhysicalOperator(physical_plan, PhysicalOperatorType::INOUT_FUNCTION, std::move(types), estimated_cardinality),
       function(std::move(function_p)), bind_data(std::move(bind_data_p)), column_ids(std::move(column_ids_p)),
       projected_input(std::move(project_input_p)) {
 }
@@ -61,6 +62,15 @@ unique_ptr<GlobalOperatorState> PhysicalTableInOutFunction::GetGlobalOperatorSta
 	return std::move(result);
 }
 
+void PhysicalTableInOutFunction::SetOrdinality(DataChunk &chunk, const optional_idx &ordinality_column_idx,
+                                               const idx_t &ordinality_idx, const idx_t &ordinality) {
+	D_ASSERT(ordinality_column_idx.IsValid());
+	if (ordinality > 0) {
+		constexpr idx_t step = 1;
+		chunk.data[ordinality_column_idx.GetIndex()].Sequence(static_cast<int64_t>(ordinality_idx), step, ordinality);
+	}
+}
+
 OperatorResultType PhysicalTableInOutFunction::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
                                                        GlobalOperatorState &gstate_p, OperatorState &state_p) const {
 	auto &gstate = gstate_p.Cast<TableInOutGlobalState>();
@@ -68,7 +78,13 @@ OperatorResultType PhysicalTableInOutFunction::Execute(ExecutionContext &context
 	TableFunctionInput data(bind_data.get(), state.local_state.get(), gstate.global_state.get());
 	if (projected_input.empty()) {
 		// straightforward case - no need to project input
-		return function.in_out_function(context, data, input, chunk);
+		auto result = function.in_out_function(context, data, input, chunk);
+		if (this->ordinality_idx.IsValid()) {
+			const idx_t ordinality = chunk.size();
+			SetOrdinality(chunk, this->ordinality_idx, state.current_ordinality_idx, ordinality);
+			state.current_ordinality_idx += ordinality;
+		}
+		return result;
 	}
 	// when project_input is set we execute the input function row-by-row
 	if (state.new_row) {
@@ -87,6 +103,7 @@ OperatorResultType PhysicalTableInOutFunction::Execute(ExecutionContext &context
 		state.input_chunk.SetCardinality(1);
 		state.row_index++;
 		state.new_row = false;
+		state.current_ordinality_idx = 1;
 	}
 	// set up the output data in "chunk"
 	D_ASSERT(chunk.ColumnCount() > projected_input.size());
@@ -98,6 +115,11 @@ OperatorResultType PhysicalTableInOutFunction::Execute(ExecutionContext &context
 		ConstantVector::Reference(chunk.data[target_idx], input.data[source_idx], state.row_index - 1, 1);
 	}
 	auto result = function.in_out_function(context, data, state.input_chunk, chunk);
+	if (this->ordinality_idx.IsValid()) {
+		const idx_t ordinality = chunk.size();
+		SetOrdinality(chunk, this->ordinality_idx, state.current_ordinality_idx, ordinality);
+		state.current_ordinality_idx += ordinality;
+	}
 	if (result == OperatorResultType::FINISHED) {
 		return result;
 	}

package/src/duckdb/src/execution/operator/projection/physical_unnest.cpp

@@ -62,9 +62,11 @@ void UnnestOperatorState::Reset() {
 	first_fetch = true;
 }
 
-PhysicalUnnest::PhysicalUnnest(vector<LogicalType> types, vector<unique_ptr<Expression>> select_list,
-                               idx_t estimated_cardinality, PhysicalOperatorType type)
-    : PhysicalOperator(type, std::move(types), estimated_cardinality), select_list(std::move(select_list)) {
+PhysicalUnnest::PhysicalUnnest(PhysicalPlan &physical_plan, vector<LogicalType> types,
+                               vector<unique_ptr<Expression>> select_list, idx_t estimated_cardinality,
+                               PhysicalOperatorType type)
+    : PhysicalOperator(physical_plan, type, std::move(types), estimated_cardinality),
+      select_list(std::move(select_list)) {
 	D_ASSERT(!this->select_list.empty());
 }
 

package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp

@@ -8,16 +8,18 @@
 
 namespace duckdb {
 
-PhysicalColumnDataScan::PhysicalColumnDataScan(vector<LogicalType> types, PhysicalOperatorType op_type,
-                                               idx_t estimated_cardinality,
+PhysicalColumnDataScan::PhysicalColumnDataScan(PhysicalPlan &physical_plan, vector<LogicalType> types,
+                                               PhysicalOperatorType op_type, idx_t estimated_cardinality,
                                                optionally_owned_ptr<ColumnDataCollection> collection_p)
-    : PhysicalOperator(op_type, std::move(types), estimated_cardinality), collection(std::move(collection_p)),
-      cte_index(DConstants::INVALID_INDEX) {
+    : PhysicalOperator(physical_plan, op_type, std::move(types), estimated_cardinality),
+      collection(std::move(collection_p)), cte_index(DConstants::INVALID_INDEX) {
 }
 
-PhysicalColumnDataScan::PhysicalColumnDataScan(vector<LogicalType> types, PhysicalOperatorType op_type,
-                                               idx_t estimated_cardinality, idx_t cte_index)
-    : PhysicalOperator(op_type, std::move(types), estimated_cardinality), collection(nullptr), cte_index(cte_index) {
+PhysicalColumnDataScan::PhysicalColumnDataScan(PhysicalPlan &physical_plan, vector<LogicalType> types,
+                                               PhysicalOperatorType op_type, idx_t estimated_cardinality,
+                                               idx_t cte_index)
+    : PhysicalOperator(physical_plan, op_type, std::move(types), estimated_cardinality), collection(nullptr),
+      cte_index(cte_index) {
 }
 
 class PhysicalColumnDataGlobalScanState : public GlobalSourceState {

package/src/duckdb/src/execution/operator/scan/physical_positional_scan.cpp

@@ -10,9 +10,9 @@
 
 namespace duckdb {
 
-PhysicalPositionalScan::PhysicalPositionalScan(vector<LogicalType> types, PhysicalOperator &left,
-                                               PhysicalOperator &right)
-    : PhysicalOperator(PhysicalOperatorType::POSITIONAL_SCAN, std::move(types),
+PhysicalPositionalScan::PhysicalPositionalScan(PhysicalPlan &physical_plan, vector<LogicalType> types,
+                                               PhysicalOperator &left, PhysicalOperator &right)
+    : PhysicalOperator(physical_plan, PhysicalOperatorType::POSITIONAL_SCAN, std::move(types),
                        MaxValue(left.estimated_cardinality, right.estimated_cardinality)) {
 
 	// Manage the children ourselves

package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp

@@ -9,13 +9,13 @@
 
 namespace duckdb {
 
-PhysicalTableScan::PhysicalTableScan(vector<LogicalType> types, TableFunction function_p,
+PhysicalTableScan::PhysicalTableScan(PhysicalPlan &physical_plan, vector<LogicalType> types, TableFunction function_p,
                                      unique_ptr<FunctionData> bind_data_p, vector<LogicalType> returned_types_p,
                                      vector<ColumnIndex> column_ids_p, vector<idx_t> projection_ids_p,
                                      vector<string> names_p, unique_ptr<TableFilterSet> table_filters_p,
                                      idx_t estimated_cardinality, ExtraOperatorInfo extra_info,
                                      vector<Value> parameters_p, virtual_column_map_t virtual_columns_p)
-    : PhysicalOperator(PhysicalOperatorType::TABLE_SCAN, std::move(types), estimated_cardinality),
+    : PhysicalOperator(physical_plan, PhysicalOperatorType::TABLE_SCAN, std::move(types), estimated_cardinality),
       function(std::move(function_p)), bind_data(std::move(bind_data_p)), returned_types(std::move(returned_types_p)),
       column_ids(std::move(column_ids_p)), projection_ids(std::move(projection_ids_p)), names(std::move(names_p)),
       table_filters(std::move(table_filters_p)), extra_info(std::move(extra_info)), parameters(std::move(parameters_p)),
@@ -32,7 +32,7 @@ public:
 		if (op.function.init_global) {
 			auto filters = table_filters ? *table_filters : GetTableFilters(op);
 			TableFunctionInitInput input(op.bind_data.get(), op.column_ids, op.projection_ids, filters,
-			                             op.extra_info.sample_options);
+			                             op.extra_info.sample_options, &op);
 
 			global_state = op.function.init_global(context, input);
 			if (global_state) {
@@ -47,7 +47,7 @@ public:
 			for (auto &param : op.parameters) {
 				input_types.push_back(param.type());
 			}
-			input_chunk.Initialize(context, input_types);
+			input_chunk.Initialize(BufferAllocator::Get(context), input_types);
 			for (idx_t c = 0; c < op.parameters.size(); c++) {
 				input_chunk.data[c].Reference(op.parameters[c]);
 			}
@@ -76,7 +76,7 @@ public:
 	                          const PhysicalTableScan &op) {
 		if (op.function.init_local) {
 			TableFunctionInitInput input(op.bind_data.get(), op.column_ids, op.projection_ids,
-			                             gstate.GetTableFilters(op), op.extra_info.sample_options);
+			                             gstate.GetTableFilters(op), op.extra_info.sample_options, &op);
 			local_state = op.function.init_local(context, input, gstate.global_state.get());
 		}
 	}
@@ -110,6 +110,7 @@ SourceResultType PhysicalTableScan::GetData(ExecutionContext &context, DataChunk
 		function.in_out_function_final(context, data, chunk);
 	}
 	switch (function.in_out_function(context, data, g_state.input_chunk, chunk)) {
+
 	case OperatorResultType::BLOCKED: {
 		auto guard = g_state.Lock();
 		return g_state.BlockSource(guard, input.interrupt_state);

package/src/duckdb/src/execution/operator/schema/physical_attach.cpp

@@ -17,7 +17,8 @@ SourceResultType PhysicalAttach::GetData(ExecutionContext &context, DataChunk &c
                                          OperatorSourceInput &input) const {
 	// parse the options
 	auto &config = DBConfig::GetConfig(context.client);
-	AttachOptions options(info, config.options.access_mode);
+	// construct the options
+	AttachOptions options(info->options, config.options.access_mode);
 
 	// get the name and path of the database
 	auto &name = info->name;
@@ -35,7 +36,7 @@ SourceResultType PhysicalAttach::GetData(ExecutionContext &context, DataChunk &c
 	if (info->on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT ||
 	    info->on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) {
 		// constant-time lookup in the catalog for the db name
-		auto existing_db = db_manager.GetDatabase(context.client, name);
+		auto existing_db = db_manager.GetDatabase(name);
 		if (existing_db) {
 			if ((existing_db->IsReadOnly() && options.access_mode == AccessMode::READ_WRITE) ||
 			    (!existing_db->IsReadOnly() && options.access_mode == AccessMode::READ_ONLY)) {
@@ -50,10 +51,8 @@ SourceResultType PhysicalAttach::GetData(ExecutionContext &context, DataChunk &c
 				existing_db->GetCatalog().SetDefaultTable(options.default_table.schema, options.default_table.name);
 			}
 			if (info->on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) {
-				// same path, name and type, DB does not need replacing
-				auto const db_type = options.db_type.empty() ? "duckdb" : options.db_type;
-				if (existing_db->GetCatalog().GetDBPath() == path &&
-				    existing_db->GetCatalog().GetCatalogType() == db_type) {
+				// allow custom catalogs to override this behavior
+				if (!existing_db->GetCatalog().HasConflictingAttachOptions(path, options)) {
 					return SourceResultType::FINISHED;
 				}
 			} else {
@@ -62,17 +61,20 @@ SourceResultType PhysicalAttach::GetData(ExecutionContext &context, DataChunk &c
 		}
 	}
 
-	// Get the database type and attach the database.
-	db_manager.GetDatabaseType(context.client, *info, config, options);
+	// attach the database.
 	auto attached_db = db_manager.AttachDatabase(context.client, *info, options);
+	if (!attached_db) {
+		return SourceResultType::FINISHED;
+	}
 
 	//! Initialize the database.
-	const auto storage_options = info->GetStorageOptions();
-	attached_db->Initialize(context.client, storage_options);
+	attached_db->Initialize(context.client);
 	if (!options.default_table.name.empty()) {
 		attached_db->GetCatalog().SetDefaultTable(options.default_table.schema, options.default_table.name);
 	}
 	attached_db->FinalizeLoad(context.client);
+
+	db_manager.FinalizeAttach(context.client, *info, std::move(attached_db));
 	return SourceResultType::FINISHED;
 }
 

package/src/duckdb/src/execution/operator/schema/physical_create_art_index.cpp

@@ -14,12 +14,13 @@
 
 namespace duckdb {
 
-PhysicalCreateARTIndex::PhysicalCreateARTIndex(LogicalOperator &op, TableCatalogEntry &table_p,
-                                               const vector<column_t> &column_ids, unique_ptr<CreateIndexInfo> info,
+PhysicalCreateARTIndex::PhysicalCreateARTIndex(PhysicalPlan &physical_plan, LogicalOperator &op,
+                                               TableCatalogEntry &table_p, const vector<column_t> &column_ids,
+                                               unique_ptr<CreateIndexInfo> info,
                                                vector<unique_ptr<Expression>> unbound_expressions,
                                                idx_t estimated_cardinality, const bool sorted,
                                                unique_ptr<AlterTableInfo> alter_table_info)
-    : PhysicalOperator(PhysicalOperatorType::CREATE_INDEX, op.types, estimated_cardinality),
+    : PhysicalOperator(physical_plan, PhysicalOperatorType::CREATE_INDEX, op.types, estimated_cardinality),
       table(table_p.Cast<DuckTableEntry>()), info(std::move(info)), unbound_expressions(std::move(unbound_expressions)),
       sorted(sorted), alter_table_info(std::move(alter_table_info)) {
 
@@ -113,7 +114,7 @@ SinkResultType PhysicalCreateARTIndex::SinkSorted(OperatorSinkInput &input) cons
 	auto art = make_uniq<ART>(info->index_name, l_index->GetConstraintType(), l_index->GetColumnIds(),
 	                          l_index->table_io_manager, l_index->unbound_expressions, storage.db,
 	                          l_index->Cast<ART>().allocators);
-	if (!art->Construct(l_state.keys, l_state.row_ids, l_state.key_chunk.size())) {
+	if (art->Build(l_state.keys, l_state.row_ids, l_state.key_chunk.size()) != ARTConflictType::NO_CONFLICT) {
 		throw ConstraintException("Data contains duplicates on indexed column(s)");
 	}
 

package/src/duckdb/src/execution/operator/schema/physical_create_table.cpp

@@ -7,10 +7,10 @@
 
 namespace duckdb {
 
-PhysicalCreateTable::PhysicalCreateTable(LogicalOperator &op, SchemaCatalogEntry &schema,
+PhysicalCreateTable::PhysicalCreateTable(PhysicalPlan &physical_plan, LogicalOperator &op, SchemaCatalogEntry &schema,
                                          unique_ptr<BoundCreateTableInfo> info, idx_t estimated_cardinality)
-    : PhysicalOperator(PhysicalOperatorType::CREATE_TABLE, op.types, estimated_cardinality), schema(schema),
-      info(std::move(info)) {
+    : PhysicalOperator(physical_plan, PhysicalOperatorType::CREATE_TABLE, op.types, estimated_cardinality),
+      schema(schema), info(std::move(info)) {
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/execution/operator/schema/physical_create_type.cpp

@@ -7,8 +7,9 @@
 
 namespace duckdb {
 
-PhysicalCreateType::PhysicalCreateType(unique_ptr<CreateTypeInfo> info_p, idx_t estimated_cardinality)
-    : PhysicalOperator(PhysicalOperatorType::CREATE_TYPE, {LogicalType::BIGINT}, estimated_cardinality),
+PhysicalCreateType::PhysicalCreateType(PhysicalPlan &physical_plan, unique_ptr<CreateTypeInfo> info_p,
+                                       idx_t estimated_cardinality)
+    : PhysicalOperator(physical_plan, PhysicalOperatorType::CREATE_TYPE, {LogicalType::BIGINT}, estimated_cardinality),
       info(std::move(info_p)) {
 }
 

package/src/duckdb/src/execution/operator/schema/physical_drop.cpp

@@ -17,8 +17,9 @@ SourceResultType PhysicalDrop::GetData(ExecutionContext &context, DataChunk &chu
 	case CatalogType::PREPARED_STATEMENT: {
 		// DEALLOCATE silently ignores errors
 		auto &statements = ClientData::Get(context.client).prepared_statements;
-		if (statements.find(info->name) != statements.end()) {
-			statements.erase(info->name);
+		auto stmt_iter = statements.find(info->name);
+		if (stmt_iter != statements.end()) {
+			statements.erase(stmt_iter);
 		}
 		break;
 	}

package/src/duckdb/src/execution/operator/set/physical_cte.cpp

@@ -9,10 +9,10 @@
 
 namespace duckdb {
 
-PhysicalCTE::PhysicalCTE(string ctename, idx_t table_index, vector<LogicalType> types, PhysicalOperator &top,
-                         PhysicalOperator &bottom, idx_t estimated_cardinality)
-    : PhysicalOperator(PhysicalOperatorType::CTE, std::move(types), estimated_cardinality), table_index(table_index),
-      ctename(std::move(ctename)) {
+PhysicalCTE::PhysicalCTE(PhysicalPlan &physical_plan, string ctename, idx_t table_index, vector<LogicalType> types,
+                         PhysicalOperator &top, PhysicalOperator &bottom, idx_t estimated_cardinality)
+    : PhysicalOperator(physical_plan, PhysicalOperatorType::CTE, std::move(types), estimated_cardinality),
+      table_index(table_index), ctename(std::move(ctename)) {
 	children.push_back(top);
 	children.push_back(bottom);
 }
@@ -106,6 +106,7 @@ InsertionOrderPreservingMap<string> PhysicalCTE::ParamsToString() const {
 	InsertionOrderPreservingMap<string> result;
 	result["CTE Name"] = ctename;
 	result["Table Index"] = StringUtil::Format("%llu", table_index);
+	SetEstimatedCardinality(result, estimated_cardinality);
 	return result;
 }
 

package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp

@@ -15,9 +15,10 @@
 
 namespace duckdb {
 
-PhysicalRecursiveCTE::PhysicalRecursiveCTE(string ctename, idx_t table_index, vector<LogicalType> types, bool union_all,
-                                           PhysicalOperator &top, PhysicalOperator &bottom, idx_t estimated_cardinality)
-    : PhysicalOperator(PhysicalOperatorType::RECURSIVE_CTE, std::move(types), estimated_cardinality),
+PhysicalRecursiveCTE::PhysicalRecursiveCTE(PhysicalPlan &physical_plan, string ctename, idx_t table_index,
+                                           vector<LogicalType> types, bool union_all, PhysicalOperator &top,
+                                           PhysicalOperator &bottom, idx_t estimated_cardinality)
+    : PhysicalOperator(physical_plan, PhysicalOperatorType::RECURSIVE_CTE, std::move(types), estimated_cardinality),
       ctename(std::move(ctename)), table_index(table_index), union_all(union_all) {
 	children.push_back(top);
 	children.push_back(bottom);
@@ -319,6 +320,7 @@ InsertionOrderPreservingMap<string> PhysicalRecursiveCTE::ParamsToString() const
 	InsertionOrderPreservingMap<string> result;
 	result["CTE Name"] = ctename;
 	result["Table Index"] = StringUtil::Format("%llu", table_index);
+	SetEstimatedCardinality(result, estimated_cardinality);
 	return result;
 }
 

package/src/duckdb/src/execution/operator/set/physical_union.cpp

@@ -6,9 +6,9 @@
 
 namespace duckdb {
 
-PhysicalUnion::PhysicalUnion(vector<LogicalType> types, PhysicalOperator &top, PhysicalOperator &bottom,
-                             idx_t estimated_cardinality, bool allow_out_of_order)
-    : PhysicalOperator(PhysicalOperatorType::UNION, std::move(types), estimated_cardinality),
+PhysicalUnion::PhysicalUnion(PhysicalPlan &physical_plan, vector<LogicalType> types, PhysicalOperator &top,
+                             PhysicalOperator &bottom, idx_t estimated_cardinality, bool allow_out_of_order)
+    : PhysicalOperator(physical_plan, PhysicalOperatorType::UNION, std::move(types), estimated_cardinality),
       allow_out_of_order(allow_out_of_order) {
 	children.push_back(top);
 	children.push_back(bottom);

package/src/duckdb/src/execution/physical_operator.cpp

@@ -6,6 +6,7 @@
 #include "duckdb/common/tree_renderer.hpp"
 #include "duckdb/execution/execution_context.hpp"
 #include "duckdb/execution/operator/set/physical_recursive_cte.hpp"
+#include "duckdb/execution/physical_plan_generator.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/parallel/meta_pipeline.hpp"
 #include "duckdb/parallel/pipeline.hpp"
@@ -15,6 +16,12 @@
 
 namespace duckdb {
 
+PhysicalOperator::PhysicalOperator(PhysicalPlan &physical_plan, PhysicalOperatorType type, vector<LogicalType> types,
+                                   idx_t estimated_cardinality)
+    : children(physical_plan.ArenaRef()), type(type), types(std::move(types)),
+      estimated_cardinality(estimated_cardinality) {
+}
+
 string PhysicalOperator::GetName() const {
 	return PhysicalOperatorToString(type);
 }
@@ -202,50 +209,53 @@ void PhysicalOperator::BuildPipelines(Pipeline &current, MetaPipeline &meta_pipe
 	op_state.reset();
 
 	auto &state = meta_pipeline.GetState();
+	if (!IsSink() && children.empty()) {
+		// Operator is a source.
+		state.SetPipelineSource(current, *this);
+		return;
+	}
+
+	if (children.size() != 1) {
+		throw InternalException("Operator not supported in BuildPipelines");
+	}
+
 	if (IsSink()) {
-		// operator is a sink, build a pipeline
+		// Operator is a sink.
 		sink_state.reset();
-		D_ASSERT(children.size() == 1);
 
-		// single operator: the operator becomes the data source of the current pipeline
+		// It becomes the data source of the current pipeline.
 		state.SetPipelineSource(current, *this);
 
-		// we create a new pipeline starting from the child
+		// Create a new pipeline starting at the child.
 		auto &child_meta_pipeline = meta_pipeline.CreateChildMetaPipeline(current, *this);
-		child_meta_pipeline.Build(children[0]);
-	} else {
-		// operator is not a sink! recurse in children
-		if (children.empty()) {
-			// source
-			state.SetPipelineSource(current, *this);
-		} else {
-			if (children.size() != 1) {
-				throw InternalException("Operator not supported in BuildPipelines");
-			}
-			state.AddPipelineOperator(current, *this);
-			children[0].get().BuildPipelines(current, meta_pipeline);
-		}
+		child_meta_pipeline.Build(children[0].get());
+		return;
 	}
+
+	// Recurse into the child.
+	state.AddPipelineOperator(current, *this);
+	children[0].get().BuildPipelines(current, meta_pipeline);
 }
 
 vector<const_reference<PhysicalOperator>> PhysicalOperator::GetSources() const {
 	vector<const_reference<PhysicalOperator>> result;
+	if (!IsSink() && children.empty()) {
+		// Operator is a source.
+		result.push_back(*this);
+		return result;
+	}
+
+	if (children.size() != 1) {
+		throw InternalException("Operator not supported in GetSource");
+	}
+
 	if (IsSink()) {
-		D_ASSERT(children.size() == 1);
 		result.push_back(*this);
 		return result;
-	} else {
-		if (children.empty()) {
-			// source
-			result.push_back(*this);
-			return result;
-		} else {
-			if (children.size() != 1) {
-				throw InternalException("Operator not supported in GetSource");
-			}
-			return children[0].get().GetSources();
-		}
 	}
+
+	// Recurse into the child.
+	return children[0].get().GetSources();
 }
 
 bool PhysicalOperator::AllSourcesSupportBatchIndex() const {
@@ -288,9 +298,9 @@ bool CachingPhysicalOperator::CanCacheType(const LogicalType &type) {
 	}
 }
 
-CachingPhysicalOperator::CachingPhysicalOperator(PhysicalOperatorType type, vector<LogicalType> types_p,
-                                                 idx_t estimated_cardinality)
-    : PhysicalOperator(type, std::move(types_p), estimated_cardinality) {
+CachingPhysicalOperator::CachingPhysicalOperator(PhysicalPlan &physical_plan, PhysicalOperatorType type,
+                                                 vector<LogicalType> types_p, idx_t estimated_cardinality)
+    : PhysicalOperator(physical_plan, type, std::move(types_p), estimated_cardinality) {
 
 	caching_supported = true;
 	for (auto &col_type : types) {

package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp

@@ -1,3 +1,5 @@
+#include "duckdb/main/settings.hpp"
+
 #include "duckdb/catalog/catalog_entry/aggregate_function_catalog_entry.hpp"
 #include "duckdb/common/operator/subtract.hpp"
 #include "duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp"
@@ -117,9 +119,6 @@ static bool CanUsePerfectHashAggregate(ClientContext &context, LogicalAggregate
 		return false;
 	}
 	idx_t perfect_hash_bits = 0;
-	if (op.group_stats.empty()) {
-		op.group_stats.resize(op.groups.size());
-	}
 	for (idx_t group_idx = 0; group_idx < op.groups.size(); group_idx++) {
 		auto &group = op.groups[group_idx];
 		auto &stats = op.group_stats[group_idx];
@@ -218,7 +217,7 @@ static bool CanUsePerfectHashAggregate(ClientContext &context, LogicalAggregate
 		bits_per_group.push_back(required_bits);
 		perfect_hash_bits += required_bits;
 		// check if we have exceeded the bits for the hash
-		if (perfect_hash_bits > ClientConfig::GetConfig(context).perfect_ht_threshold) {
+		if (perfect_hash_bits > DBConfig::GetSetting<PerfectHtThresholdSetting>(context)) {
 			// too many bits for perfect hash
 			return false;
 		}
@@ -249,12 +248,25 @@ PhysicalOperator &PhysicalPlanGenerator::CreatePlan(LogicalAggregate &op) {
 		}
 	}
 
+	// Check if all groups are valid
+	if (op.group_stats.empty()) {
+		op.group_stats.resize(op.groups.size());
+	}
+	auto group_validity = TupleDataValidityType::CANNOT_HAVE_NULL_VALUES;
+	for (const auto &stats : op.group_stats) {
+		if (stats && !stats->CanHaveNull()) {
+			continue;
+		}
+		group_validity = TupleDataValidityType::CAN_HAVE_NULL_VALUES;
+		break;
+	}
+
 	if (op.groups.empty() && op.grouping_sets.size() <= 1) {
 		// no groups, check if we can use a simple aggregation
 		// special case: aggregate entire columns together
 		if (can_use_simple_aggregation) {
-			auto &group_by =
-			    Make<PhysicalUngroupedAggregate>(op.types, std::move(op.expressions), op.estimated_cardinality);
+			auto &group_by = Make<PhysicalUngroupedAggregate>(op.types, std::move(op.expressions),
+			                                                  op.estimated_cardinality, op.distinct_validity);
 			group_by.children.push_back(plan);
 			return group_by;
 		}
@@ -286,7 +298,7 @@ PhysicalOperator &PhysicalPlanGenerator::CreatePlan(LogicalAggregate &op) {
 
 	auto &group_by = Make<PhysicalHashAggregate>(context, op.types, std::move(op.expressions), std::move(op.groups),
 	                                             std::move(op.grouping_sets), std::move(op.grouping_functions),
-	                                             op.estimated_cardinality);
+	                                             op.estimated_cardinality, group_validity, op.distinct_validity);
 	group_by.children.push_back(plan);
 	return group_by;
 }
@@ -313,11 +325,11 @@ PhysicalOperator &PhysicalPlanGenerator::ExtractAggregateExpressions(PhysicalOpe
 	}
 	for (auto &aggr : aggregates) {
 		auto &bound_aggr = aggr->Cast<BoundAggregateExpression>();
-		for (auto &child : bound_aggr.children) {
-			auto ref = make_uniq<BoundReferenceExpression>(child->return_type, expressions.size());
-			types.push_back(child->return_type);
-			expressions.push_back(std::move(child));
-			child = std::move(ref);
+		for (auto &child_expr : bound_aggr.children) {
+			auto ref = make_uniq<BoundReferenceExpression>(child_expr->return_type, expressions.size());
+			types.push_back(child_expr->return_type);
+			expressions.push_back(std::move(child_expr));
+			child_expr = std::move(ref);
 		}
 		if (bound_aggr.filter) {
 			auto &filter = bound_aggr.filter;

package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp

@@ -13,6 +13,7 @@
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/planner/expression/bound_window_expression.hpp"
+#include "duckdb/main/settings.hpp"
 
 namespace duckdb {
 
@@ -26,7 +27,7 @@ PhysicalPlanGenerator::PlanAsOfLoopJoin(LogicalComparisonJoin &op, PhysicalOpera
 	//
 	//		 ∏ * \ pk
 	//		 |
-	//		 Γ pk;first(P),arg_xxx(B,inequality)
+	//		 Γ pk;first(P),arg_xxx_null(B,inequality)
 	//		 |
 	//		 ∏ *,inequality
 	//		 |
@@ -87,13 +88,13 @@ PhysicalPlanGenerator::PlanAsOfLoopJoin(LogicalComparisonJoin &op, PhysicalOpera
 		case ExpressionType::COMPARE_GREATERTHAN:
 			D_ASSERT(asof_idx == op.conditions.size());
 			asof_idx = i;
-			arg_min_max = "arg_max";
+			arg_min_max = "arg_max_null";
 			break;
 		case ExpressionType::COMPARE_LESSTHANOREQUALTO:
 		case ExpressionType::COMPARE_LESSTHAN:
 			D_ASSERT(asof_idx == op.conditions.size());
 			asof_idx = i;
-			arg_min_max = "arg_min";
+			arg_min_max = "arg_min_null";
 			break;
 		case ExpressionType::COMPARE_EQUAL:
 		case ExpressionType::COMPARE_NOTEQUAL:
@@ -274,9 +275,10 @@ PhysicalOperator &PhysicalPlanGenerator::PlanAsOfJoin(LogicalComparisonJoin &op)
 	}
 	D_ASSERT(asof_idx < op.conditions.size());
 
-	auto &config = ClientConfig::GetConfig(context);
-	if (!config.force_asof_iejoin) {
-		if (op.children[0]->has_estimated_cardinality && lhs_cardinality < config.asof_loop_join_threshold) {
+	bool force_asof_join = DBConfig::GetSetting<DebugAsofIejoinSetting>(context);
+	if (!force_asof_join) {
+		idx_t asof_join_threshold = DBConfig::GetSetting<AsofLoopJoinThresholdSetting>(context);
+		if (op.children[0]->has_estimated_cardinality && lhs_cardinality < asof_join_threshold) {
 			auto result = PlanAsOfLoopJoin(op, left, right);
 			if (result) {
 				return *result;