duckdb 0.7.2-dev1901.0 → 0.7.2-dev2233.0

package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp

@@ -19,12 +19,8 @@
 
 namespace duckdb {
 
-static bool CanPlanIndexJoin(ClientContext &context, TableScanBindData *bind_data, PhysicalTableScan &scan) {
-	if (!bind_data) {
-		// not a table scan
-		return false;
-	}
-	auto table = bind_data->table;
+static bool CanPlanIndexJoin(ClientContext &context, TableScanBindData &bind_data, PhysicalTableScan &scan) {
+	auto table = bind_data.table;
 	auto &transaction = DuckTransaction::Get(context, *table->catalog);
 	auto &local_storage = LocalStorage::Get(transaction);
 	if (local_storage.Find(table->GetStoragePtr())) {
@@ -139,21 +135,23 @@ void CheckForPerfectJoinOpt(LogicalComparisonJoin &op, PerfectHashJoinStats &joi
 	return;
 }
 
-static void CanUseIndexJoin(TableScanBindData *tbl, Expression &expr, Index **result_index) {
-	tbl->table->GetStorage().info->indexes.Scan([&](Index &index) {
+static optional_ptr<Index> CanUseIndexJoin(TableScanBindData &tbl, Expression &expr) {
+	optional_ptr<Index> result;
+	tbl.table->GetStorage().info->indexes.Scan([&](Index &index) {
 		if (index.unbound_expressions.size() != 1) {
 			return false;
 		}
 		if (expr.alias == index.unbound_expressions[0]->alias) {
-			*result_index = &index;
+			result = &index;
 			return true;
 		}
 		return false;
 	});
+	return result;
 }
 
-Index *CheckIndexJoin(ClientContext &context, LogicalComparisonJoin &op, PhysicalOperator &plan,
-                      Expression &condition) {
+optional_ptr<Index> CheckIndexJoin(ClientContext &context, LogicalComparisonJoin &op, PhysicalOperator &plan,
+                                   Expression &condition) {
 	if (op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN) {
 		return nullptr;
 	}
@@ -169,17 +167,20 @@ Index *CheckIndexJoin(ClientContext &context, LogicalComparisonJoin &op, Physica
 		return nullptr;
 	}
 	auto &tbl_scan = plan.Cast<PhysicalTableScan>();
-	auto tbl = dynamic_cast<TableScanBindData *>(tbl_scan.bind_data.get());
-	Index *result = nullptr;
-	if (CanPlanIndexJoin(context, tbl, tbl_scan)) {
-		CanUseIndexJoin(tbl, condition, &result);
+	auto tbl_data = dynamic_cast<TableScanBindData *>(tbl_scan.bind_data.get());
+	if (!tbl_data) {
+		return nullptr;
+	}
+	optional_ptr<Index> result;
+	if (CanPlanIndexJoin(context, *tbl_data, tbl_scan)) {
+		result = CanUseIndexJoin(*tbl_data, condition);
 	}
 	return result;
 }
 
 static bool PlanIndexJoin(ClientContext &context, LogicalComparisonJoin &op, unique_ptr<PhysicalOperator> &plan,
-                          unique_ptr<PhysicalOperator> &left, unique_ptr<PhysicalOperator> &right, Index *index,
-                          bool swap_condition = false) {
+                          unique_ptr<PhysicalOperator> &left, unique_ptr<PhysicalOperator> &right,
+                          optional_ptr<Index> index, bool swap_condition = false) {
 	if (!index) {
 		return false;
 	}
@@ -205,7 +206,7 @@ static bool PlanIndexJoin(ClientContext &context, LogicalComparisonJoin &op, uni
 		swap(op.left_projection_map, op.right_projection_map);
 	}
 	plan = make_uniq<PhysicalIndexJoin>(op, std::move(left), std::move(right), std::move(op.conditions), op.join_type,
-	                                    op.left_projection_map, op.right_projection_map, tbl_scan.column_ids, index,
+	                                    op.left_projection_map, op.right_projection_map, tbl_scan.column_ids, *index,
 	                                    !swap_condition, op.estimated_cardinality);
 	return true;
 }

package/src/duckdb/src/execution/physical_plan/plan_copy_to_file.cpp

@@ -16,7 +16,8 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCopyToFile
 	auto copy = make_uniq<PhysicalCopyToFile>(op.types, op.function, std::move(op.bind_data), op.estimated_cardinality);
 	copy->file_path = op.file_path;
 	copy->use_tmp_file = op.use_tmp_file;
-	copy->allow_overwrite = op.allow_overwrite;
+	copy->overwrite_or_ignore = op.overwrite_or_ignore;
+	copy->filename_pattern = op.filename_pattern;
 	copy->per_thread_output = op.per_thread_output;
 	copy->partition_output = op.partition_output;
 	copy->partition_columns = op.partition_columns;

package/src/duckdb/src/execution/physical_plan/plan_execute.cpp

@@ -8,13 +8,13 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalExecute &o
 	if (!op.prepared->plan) {
 		D_ASSERT(op.children.size() == 1);
 		auto owned_plan = CreatePlan(*op.children[0]);
-		auto execute = make_uniq<PhysicalExecute>(owned_plan.get());
+		auto execute = make_uniq<PhysicalExecute>(*owned_plan);
 		execute->owned_plan = std::move(owned_plan);
 		execute->prepared = std::move(op.prepared);
 		return std::move(execute);
 	} else {
 		D_ASSERT(op.children.size() == 0);
-		return make_uniq<PhysicalExecute>(op.prepared->plan.get());
+		return make_uniq<PhysicalExecute>(*op.prepared->plan);
 	}
 }
 

package/src/duckdb/src/execution/physical_plan/plan_explain.cpp

@@ -1,11 +1,10 @@
-#include "duckdb/execution/physical_plan_generator.hpp"
-#include "duckdb/planner/operator/logical_explain.hpp"
+#include "duckdb/common/tree_renderer.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
 #include "duckdb/execution/operator/helper/physical_explain_analyze.hpp"
-#include "duckdb/main/client_context.hpp"
-#include "duckdb/common/types/column_data_collection.hpp"
 #include "duckdb/execution/operator/scan/physical_column_data_scan.hpp"
-
-#include "duckdb/common/tree_renderer.hpp"
+#include "duckdb/execution/physical_plan_generator.hpp"
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/planner/operator/logical_explain.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/execution/physical_plan/plan_expression_get.cpp

@@ -1,8 +1,8 @@
+#include "duckdb/common/types/column/column_data_collection.hpp"
+#include "duckdb/execution/operator/scan/physical_column_data_scan.hpp"
 #include "duckdb/execution/operator/scan/physical_expression_scan.hpp"
 #include "duckdb/execution/physical_plan_generator.hpp"
 #include "duckdb/planner/operator/logical_expression_get.hpp"
-#include "duckdb/execution/operator/scan/physical_column_data_scan.hpp"
-#include "duckdb/common/types/column_data_collection.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp

@@ -1,10 +1,10 @@
-#include "duckdb/execution/operator/set/physical_recursive_cte.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
 #include "duckdb/execution/operator/scan/physical_column_data_scan.hpp"
+#include "duckdb/execution/operator/set/physical_recursive_cte.hpp"
 #include "duckdb/execution/physical_plan_generator.hpp"
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
-#include "duckdb/planner/operator/logical_recursive_cte.hpp"
 #include "duckdb/planner/operator/logical_cteref.hpp"
-#include "duckdb/common/types/column_data_collection.hpp"
+#include "duckdb/planner/operator/logical_recursive_cte.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/execution/physical_plan_generator.cpp

@@ -1,7 +1,7 @@
 #include "duckdb/execution/physical_plan_generator.hpp"
 
 #include "duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp"
-#include "duckdb/common/types/column_data_collection.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
 #include "duckdb/execution/column_binding_resolver.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/config.hpp"

package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp

@@ -1,8 +1,9 @@
 #include "duckdb/execution/radix_partitioned_hashtable.hpp"
-#include "duckdb/parallel/task_scheduler.hpp"
+
 #include "duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp"
-#include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/parallel/event.hpp"
+#include "duckdb/parallel/task_scheduler.hpp"
+#include "duckdb/planner/expression/bound_reference_expression.hpp"
 
 namespace duckdb {
 
@@ -334,7 +335,7 @@ public:
 	//! The current position to scan the HT for output tuples
 	idx_t ht_index;
 	//! The set of aggregate scan states
-	unique_ptr<AggregateHTScanState[]> ht_scan_states;
+	unique_ptr<TupleDataParallelScanState[]> ht_scan_states;
 	atomic<bool> initialized;
 	atomic<bool> finished;
 };
@@ -352,8 +353,12 @@ public:
 
 	//! Materialized GROUP BY expressions & aggregates
 	DataChunk scan_chunk;
+	//! HT index
+	idx_t ht_index = DConstants::INVALID_INDEX;
 	//! A reference to the current HT that we are scanning
 	shared_ptr<GroupedAggregateHashTable> ht;
+	//! Scan state for the current HT
+	TupleDataLocalScanState scan_state;
 };
 
 unique_ptr<GlobalSourceState> RadixPartitionedHashTable::GetGlobalSourceState(ClientContext &context) const {
@@ -372,7 +377,7 @@ idx_t RadixPartitionedHashTable::Size(GlobalSinkState &sink_state) const {
 
 	idx_t count = 0;
 	for (const auto &ht : gstate.finalized_hts) {
-		count += ht->Size();
+		count += ht->Count();
 	}
 	return count;
 }
@@ -425,20 +430,32 @@ void RadixPartitionedHashTable::GetData(ExecutionContext &context, DataChunk &ch
 	idx_t elements_found = 0;
 
 	lstate.scan_chunk.Reset();
-	lstate.ht.reset();
 	if (!state.initialized) {
 		lock_guard<mutex> l(state.lock);
-		if (!state.ht_scan_states) {
+		if (!state.initialized) {
+			auto &finalized_hts = gstate.finalized_hts;
 			state.ht_scan_states =
-			    unique_ptr<AggregateHTScanState[]>(new AggregateHTScanState[gstate.finalized_hts.size()]);
-		} else {
-			D_ASSERT(state.initialized);
+			    unique_ptr<TupleDataParallelScanState[]>(new TupleDataParallelScanState[finalized_hts.size()]);
+
+			const auto &layout = gstate.finalized_hts[0]->GetDataCollection().GetLayout();
+			vector<column_t> column_ids;
+			column_ids.reserve(layout.ColumnCount() - 1);
+			for (idx_t col_idx = 0; col_idx < layout.ColumnCount() - 1; col_idx++) {
+				column_ids.emplace_back(col_idx);
+			}
+
+			for (idx_t ht_idx = 0; ht_idx < finalized_hts.size(); ht_idx++) {
+				gstate.finalized_hts[ht_idx]->GetDataCollection().InitializeScan(
+				    state.ht_scan_states.get()[ht_idx].scan_state, column_ids);
+			}
+			state.initialized = true;
 		}
-		state.initialized = true;
 	}
+
+	auto &local_scan_state = lstate.scan_state;
 	while (true) {
+		D_ASSERT(state.ht_scan_states);
 		idx_t ht_index;
-
 		{
 			lock_guard<mutex> l(state.lock);
 			ht_index = state.ht_index;
@@ -446,17 +463,22 @@ void RadixPartitionedHashTable::GetData(ExecutionContext &context, DataChunk &ch
 				state.finished = true;
 				return;
 			}
-			D_ASSERT(ht_index < gstate.finalized_hts.size());
-			lstate.ht = gstate.finalized_hts[ht_index];
-			D_ASSERT(lstate.ht);
 		}
-		D_ASSERT(state.ht_scan_states);
-		auto &scan_state = state.ht_scan_states[ht_index];
+		D_ASSERT(ht_index < gstate.finalized_hts.size());
+		if (lstate.ht_index != DConstants::INVALID_INDEX && ht_index != lstate.ht_index) {
+			lstate.ht->GetDataCollection().FinalizePinState(local_scan_state.pin_state);
+		}
+		lstate.ht_index = ht_index;
+		lstate.ht = gstate.finalized_hts[ht_index];
 		D_ASSERT(lstate.ht);
-		elements_found = lstate.ht->Scan(scan_state, lstate.scan_chunk);
+
+		auto &global_scan_state = state.ht_scan_states[ht_index];
+		elements_found = lstate.ht->Scan(global_scan_state, local_scan_state, lstate.scan_chunk);
 		if (elements_found > 0) {
 			break;
 		}
+		lstate.ht->GetDataCollection().FinalizePinState(local_scan_state.pin_state);
+
 		// move to the next hash table
 		lock_guard<mutex> l(state.lock);
 		ht_index++;

package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp

@@ -1,6 +1,6 @@
-#include "duckdb/function/aggregate_function.hpp"
 #include "duckdb/common/sort/sort.hpp"
-#include "duckdb/common/types/column_data_collection.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
+#include "duckdb/function/aggregate_function.hpp"
 #include "duckdb/function/function_binder.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/planner/expression/bound_aggregate_expression.hpp"

package/src/duckdb/src/function/table/pragma_detailed_profiling_output.cpp

@@ -1,12 +1,12 @@
-#include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp"
-#include "duckdb/planner/constraints/bound_not_null_constraint.hpp"
-#include "duckdb/main/query_profiler.hpp"
+#include "duckdb/common/limits.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
+#include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/client_data.hpp"
-#include "duckdb/common/limits.hpp"
-#include "duckdb/common/types/column_data_collection.hpp"
+#include "duckdb/main/query_profiler.hpp"
+#include "duckdb/planner/constraints/bound_not_null_constraint.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/function/table/pragma_last_profiling_output.cpp

@@ -1,12 +1,12 @@
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp"
 #include "duckdb/common/limits.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/client_data.hpp"
-#include "duckdb/planner/constraints/bound_not_null_constraint.hpp"
 #include "duckdb/main/query_profiler.hpp"
-#include "duckdb/common/types/column_data_collection.hpp"
+#include "duckdb/planner/constraints/bound_not_null_constraint.hpp"
 
 namespace duckdb {