duckdb 0.7.2-dev1898.0 → 0.7.2-dev2144.0

package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp

@@ -1,8 +1,9 @@
 #include "duckdb/execution/radix_partitioned_hashtable.hpp"
-#include "duckdb/parallel/task_scheduler.hpp"
+
 #include "duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp"
-#include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/parallel/event.hpp"
+#include "duckdb/parallel/task_scheduler.hpp"
+#include "duckdb/planner/expression/bound_reference_expression.hpp"
 
 namespace duckdb {
 
@@ -334,7 +335,7 @@ public:
 	//! The current position to scan the HT for output tuples
 	idx_t ht_index;
 	//! The set of aggregate scan states
-	unique_ptr<AggregateHTScanState[]> ht_scan_states;
+	unique_ptr<TupleDataParallelScanState[]> ht_scan_states;
 	atomic<bool> initialized;
 	atomic<bool> finished;
 };
@@ -352,8 +353,12 @@ public:
 
 	//! Materialized GROUP BY expressions & aggregates
 	DataChunk scan_chunk;
+	//! HT index
+	idx_t ht_index = DConstants::INVALID_INDEX;
 	//! A reference to the current HT that we are scanning
 	shared_ptr<GroupedAggregateHashTable> ht;
+	//! Scan state for the current HT
+	TupleDataLocalScanState scan_state;
 };
 
 unique_ptr<GlobalSourceState> RadixPartitionedHashTable::GetGlobalSourceState(ClientContext &context) const {
@@ -372,7 +377,7 @@ idx_t RadixPartitionedHashTable::Size(GlobalSinkState &sink_state) const {
 
 	idx_t count = 0;
 	for (const auto &ht : gstate.finalized_hts) {
-		count += ht->Size();
+		count += ht->Count();
 	}
 	return count;
 }
@@ -425,20 +430,32 @@ void RadixPartitionedHashTable::GetData(ExecutionContext &context, DataChunk &ch
 	idx_t elements_found = 0;
 
 	lstate.scan_chunk.Reset();
-	lstate.ht.reset();
 	if (!state.initialized) {
 		lock_guard<mutex> l(state.lock);
-		if (!state.ht_scan_states) {
+		if (!state.initialized) {
+			auto &finalized_hts = gstate.finalized_hts;
 			state.ht_scan_states =
-			    unique_ptr<AggregateHTScanState[]>(new AggregateHTScanState[gstate.finalized_hts.size()]);
-		} else {
-			D_ASSERT(state.initialized);
+			    unique_ptr<TupleDataParallelScanState[]>(new TupleDataParallelScanState[finalized_hts.size()]);
+
+			const auto &layout = gstate.finalized_hts[0]->GetDataCollection().GetLayout();
+			vector<column_t> column_ids;
+			column_ids.reserve(layout.ColumnCount() - 1);
+			for (idx_t col_idx = 0; col_idx < layout.ColumnCount() - 1; col_idx++) {
+				column_ids.emplace_back(col_idx);
+			}
+
+			for (idx_t ht_idx = 0; ht_idx < finalized_hts.size(); ht_idx++) {
+				gstate.finalized_hts[ht_idx]->GetDataCollection().InitializeScan(
+				    state.ht_scan_states.get()[ht_idx].scan_state, column_ids);
+			}
+			state.initialized = true;
 		}
-		state.initialized = true;
 	}
+
+	auto &local_scan_state = lstate.scan_state;
 	while (true) {
+		D_ASSERT(state.ht_scan_states);
 		idx_t ht_index;
-
 		{
 			lock_guard<mutex> l(state.lock);
 			ht_index = state.ht_index;
@@ -446,17 +463,22 @@ void RadixPartitionedHashTable::GetData(ExecutionContext &context, DataChunk &ch
 				state.finished = true;
 				return;
 			}
-			D_ASSERT(ht_index < gstate.finalized_hts.size());
-			lstate.ht = gstate.finalized_hts[ht_index];
-			D_ASSERT(lstate.ht);
 		}
-		D_ASSERT(state.ht_scan_states);
-		auto &scan_state = state.ht_scan_states[ht_index];
+		D_ASSERT(ht_index < gstate.finalized_hts.size());
+		if (lstate.ht_index != DConstants::INVALID_INDEX && ht_index != lstate.ht_index) {
+			lstate.ht->GetDataCollection().FinalizePinState(local_scan_state.pin_state);
+		}
+		lstate.ht_index = ht_index;
+		lstate.ht = gstate.finalized_hts[ht_index];
 		D_ASSERT(lstate.ht);
-		elements_found = lstate.ht->Scan(scan_state, lstate.scan_chunk);
+
+		auto &global_scan_state = state.ht_scan_states[ht_index];
+		elements_found = lstate.ht->Scan(global_scan_state, local_scan_state, lstate.scan_chunk);
 		if (elements_found > 0) {
 			break;
 		}
+		lstate.ht->GetDataCollection().FinalizePinState(local_scan_state.pin_state);
+
 		// move to the next hash table
 		lock_guard<mutex> l(state.lock);
 		ht_index++;

package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp

@@ -1,6 +1,6 @@
-#include "duckdb/function/aggregate_function.hpp"
 #include "duckdb/common/sort/sort.hpp"
-#include "duckdb/common/types/column_data_collection.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
+#include "duckdb/function/aggregate_function.hpp"
 #include "duckdb/function/function_binder.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/planner/expression/bound_aggregate_expression.hpp"

package/src/duckdb/src/function/table/pragma_detailed_profiling_output.cpp

@@ -1,12 +1,12 @@
-#include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp"
-#include "duckdb/planner/constraints/bound_not_null_constraint.hpp"
-#include "duckdb/main/query_profiler.hpp"
+#include "duckdb/common/limits.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
+#include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/client_data.hpp"
-#include "duckdb/common/limits.hpp"
-#include "duckdb/common/types/column_data_collection.hpp"
+#include "duckdb/main/query_profiler.hpp"
+#include "duckdb/planner/constraints/bound_not_null_constraint.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/function/table/pragma_last_profiling_output.cpp

@@ -1,12 +1,12 @@
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp"
 #include "duckdb/common/limits.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/client_data.hpp"
-#include "duckdb/planner/constraints/bound_not_null_constraint.hpp"
 #include "duckdb/main/query_profiler.hpp"
-#include "duckdb/common/types/column_data_collection.hpp"
+#include "duckdb/planner/constraints/bound_not_null_constraint.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.7.2-dev1898"
+#define DUCKDB_VERSION "0.7.2-dev2144"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "eaf507009f"
+#define DUCKDB_SOURCE_ID "82211fc11b"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/index_catalog_entry.hpp

@@ -27,7 +27,7 @@ public:
 	//! Create an IndexCatalogEntry and initialize storage for it
 	IndexCatalogEntry(Catalog *catalog, SchemaCatalogEntry *schema, CreateIndexInfo *info);
 
-	Index *index;
+	optional_ptr<Index> index;
 	string sql;
 	vector<unique_ptr<ParsedExpression>> expressions;
 	vector<unique_ptr<ParsedExpression>> parsed_expressions;

package/src/duckdb/src/include/duckdb/common/arrow/arrow_appender.hpp

@@ -23,7 +23,7 @@ public:
 	DUCKDB_API ~ArrowAppender();
 
 	//! Append a data chunk to the underlying arrow array
-	DUCKDB_API void Append(DataChunk &input);
+	DUCKDB_API void Append(DataChunk &input, idx_t from, idx_t to, idx_t input_size);
 	//! Returns the underlying arrow array
 	DUCKDB_API ArrowArray Finalize();
 

package/src/duckdb/src/include/duckdb/common/constants.hpp

@@ -110,6 +110,8 @@ struct PhysicalIndex {
 	}
 };
 
+DUCKDB_API bool IsPowerOfTwo(uint64_t v);
 DUCKDB_API uint64_t NextPowerOfTwo(uint64_t v);
+DUCKDB_API uint64_t PreviousPowerOfTwo(uint64_t v);
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/exception.hpp

@@ -121,6 +121,9 @@ public:
 	DUCKDB_API static bool UncaughtException();
 
 	DUCKDB_API static string GetStackTrace(int max_depth = 120);
+	DUCKDB_API static string FormatStackTrace(string message = "") {
+		return (message + "\n" + GetStackTrace());
+	}
 
 private:
 	string exception_message_;