duckdb 0.8.2-dev3458.0 → 0.8.2-dev3949.0

package/src/duckdb/src/include/duckdb/common/types/row/row_layout.hpp

@@ -10,28 +10,22 @@
 
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/types/validity_mask.hpp"
-#include "duckdb/planner/expression.hpp"
 #include "duckdb/execution/operator/aggregate/aggregate_object.hpp"
+#include "duckdb/planner/expression.hpp"
 
 namespace duckdb {
 
 class RowLayout {
 public:
 	friend class TupleDataLayout;
-
-	using Aggregates = vector<AggregateObject>;
 	using ValidityBytes = TemplatedValidityMask<uint8_t>;
 
 	//! Creates an empty RowLayout
 	RowLayout();
 
 public:
-	//! Initializes the RowLayout with the specified types and aggregates to an empty RowLayout
-	void Initialize(vector<LogicalType> types_p, Aggregates aggregates_p, bool align = true);
 	//! Initializes the RowLayout with the specified types to an empty RowLayout
 	void Initialize(vector<LogicalType> types, bool align = true);
-	//! Initializes the RowLayout with the specified aggregates to an empty RowLayout
-	void Initialize(Aggregates aggregates_p, bool align = true);
 	//! Returns the number of data columns
 	inline idx_t ColumnCount() const {
 		return types.size();
@@ -40,14 +34,6 @@ public:
 	inline const vector<LogicalType> &GetTypes() const {
 		return types;
 	}
-	//! Returns the number of aggregates
-	inline idx_t AggregateCount() const {
-		return aggregates.size();
-	}
-	//! Returns a list of the aggregates for this data chunk
-	inline Aggregates &GetAggregates() {
-		return aggregates;
-	}
 	//! Returns the total width required for each row, including padding
 	inline idx_t GetRowWidth() const {
 		return row_width;
@@ -64,10 +50,6 @@ public:
 	inline idx_t GetAggrOffset() const {
 		return flag_width + data_width;
 	}
-	//! Returns the total width required for the aggregates, including padding
-	inline idx_t GetAggrWidth() const {
-		return aggr_width;
-	}
 	//! Returns the column offsets into each row
 	inline const vector<idx_t> &GetOffsets() const {
 		return offsets;
@@ -83,14 +65,10 @@ public:
 private:
 	//! The types of the data columns
 	vector<LogicalType> types;
-	//! The aggregate functions
-	Aggregates aggregates;
 	//! The width of the validity header
 	idx_t flag_width;
 	//! The width of the data portion
 	idx_t data_width;
-	//! The width of the aggregate state portion
-	idx_t aggr_width;
 	//! The width of the entire row
 	idx_t row_width;
 	//! The offsets to the columns and aggregate data in each row

package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_allocator.hpp

@@ -55,6 +55,8 @@ public:
 	TupleDataAllocator(BufferManager &buffer_manager, const TupleDataLayout &layout);
 	TupleDataAllocator(TupleDataAllocator &allocator);
 
+	//! Get the buffer manager
+	BufferManager &GetBufferManager();
 	//! Get the buffer allocator
 	Allocator &GetAllocator();
 	//! Get the layout
@@ -83,16 +85,16 @@ public:
 private:
 	//! Builds out a single part (grabs the lock)
 	TupleDataChunkPart BuildChunkPart(TupleDataPinState &pin_state, TupleDataChunkState &chunk_state,
-	                                  const idx_t append_offset, const idx_t append_count);
+	                                  const idx_t append_offset, const idx_t append_count, TupleDataChunk &chunk);
 	//! Internal function for InitializeChunkState
 	void InitializeChunkStateInternal(TupleDataPinState &pin_state, TupleDataChunkState &chunk_state, idx_t offset,
 	                                  bool recompute, bool init_heap_pointers, bool init_heap_sizes,
-	                                  vector<TupleDataChunkPart *> &parts);
+	                                  unsafe_vector<reference<TupleDataChunkPart>> &parts);
 	//! Internal function for ReleaseOrStoreHandles
-	static void ReleaseOrStoreHandlesInternal(TupleDataSegment &segment, vector<BufferHandle> &pinned_row_handles,
-	                                          unordered_map<uint32_t, BufferHandle> &handles,
-	                                          const unordered_set<uint32_t> &block_ids, vector<TupleDataBlock> &blocks,
-	                                          TupleDataPinProperties properties);
+	static void ReleaseOrStoreHandlesInternal(TupleDataSegment &segment,
+	                                          unsafe_vector<BufferHandle> &pinned_row_handles,
+	                                          perfect_map_t<BufferHandle> &handles, const perfect_set_t &block_ids,
+	                                          unsafe_vector<TupleDataBlock> &blocks, TupleDataPinProperties properties);
 	//! Pins the given row block
 	BufferHandle &PinRowBlock(TupleDataPinState &state, const TupleDataChunkPart &part);
 	//! Pins the given heap block
@@ -108,9 +110,13 @@ private:
 	//! The layout of the data
 	const TupleDataLayout layout;
 	//! Blocks storing the fixed-size rows
-	vector<TupleDataBlock> row_blocks;
+	unsafe_vector<TupleDataBlock> row_blocks;
 	//! Blocks storing the variable-size data of the fixed-size rows (e.g., string, list)
-	vector<TupleDataBlock> heap_blocks;
+	unsafe_vector<TupleDataBlock> heap_blocks;
+
+	//! Re-usable arrays used while building buffer space
+	unsafe_vector<reference<TupleDataChunkPart>> chunk_parts;
+	unsafe_vector<pair<idx_t, idx_t>> chunk_part_indices;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp

@@ -45,6 +45,7 @@ struct TupleDataGatherFunction {
 //! FIXME: rename to RowDataCollection after we phase it out
 class TupleDataCollection {
 	friend class TupleDataChunkIterator;
+	friend class PartitionedTupleData;
 
 public:
 	//! Constructs a TupleDataCollection with the specified layout
@@ -63,8 +64,6 @@ public:
 	idx_t ChunkCount() const;
 	//! The size (in bytes) of the blocks held by this tuple data collection
 	idx_t SizeInBytes() const;
-	//! Get pointers to the pinned blocks
-	void GetBlockPointers(vector<data_ptr_t> &block_pointers) const;
 	//! Unpins all held pins
 	void Unpin();
 
@@ -186,6 +185,8 @@ private:
 	void Initialize();
 	//! Gets all column ids
 	void GetAllColumnIDs(vector<column_t> &column_ids);
+	//! Adds a segment to this TupleDataCollection
+	void AddSegment(TupleDataSegment &&segment);
 
 	//! Computes the heap sizes for the specific Vector that will be appended
 	static void ComputeHeapSizes(Vector &heap_sizes_v, const Vector &source_v, TupleDataVectorFormat &source,
@@ -219,7 +220,7 @@ private:
 	void ScanAtIndex(TupleDataPinState &pin_state, TupleDataChunkState &chunk_state, const vector<column_t> &column_ids,
 	                 idx_t segment_index, idx_t chunk_index, DataChunk &result);
 
-	//! Verify counts of the segments in this collection
+	//! Verify count/data size of this collection
 	void Verify() const;
 
 private:
@@ -229,6 +230,8 @@ private:
 	shared_ptr<TupleDataAllocator> allocator;
 	//! The number of entries stored in the TupleDataCollection
 	idx_t count;
+	//! The size (in bytes) of this TupleDataCollection
+	idx_t data_size;
 	//! The data segments of the TupleDataCollection
 	unsafe_vector<TupleDataSegment> segments;
 	//! The set of scatter functions

package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_layout.hpp

@@ -83,9 +83,14 @@ public:
 	inline bool AllConstant() const {
 		return all_constant;
 	}
+	//! Gets offset to where heap size is stored
 	inline idx_t GetHeapSizeOffset() const {
 		return heap_size_offset;
 	}
+	//! Returns whether any of the aggregates have a destructor
+	inline bool HasDestructor() const {
+		return has_destructor;
+	}
 
 private:
 	//! The types of the data columns
@@ -108,6 +113,8 @@ private:
 	bool all_constant;
 	//! Offset to the heap size of every row
 	idx_t heap_size_offset;
+	//! Whether any of the aggregates have a destructor
+	bool has_destructor;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_segment.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/mutex.hpp"
+#include "duckdb/common/perfect_map_set.hpp"
 #include "duckdb/common/unordered_set.hpp"
 #include "duckdb/common/vector.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
@@ -21,7 +22,7 @@ class TupleDataLayout;
 
 struct TupleDataChunkPart {
 public:
-	TupleDataChunkPart();
+	TupleDataChunkPart(mutex &lock);
 
 	//! Disable copy constructors
 	TupleDataChunkPart(const TupleDataChunkPart &other) = delete;
@@ -45,8 +46,8 @@ public:
 	uint32_t total_heap_size;
 	//! Tuple count for this chunk part
 	uint32_t count;
-	//! Lock for recomputing heap pointers
-	mutex lock;
+	//! Lock for recomputing heap pointers (owned by TupleDataChunk)
+	reference<mutex> lock;
 };
 
 struct TupleDataChunk {
@@ -70,13 +71,15 @@ public:
 
 public:
 	//! The parts of this chunk
-	vector<TupleDataChunkPart> parts;
+	unsafe_vector<TupleDataChunkPart> parts;
 	//! The row block ids referenced by the chunk
-	unordered_set<uint32_t> row_block_ids;
+	perfect_set_t row_block_ids;
 	//! The heap block ids referenced by the chunk
-	unordered_set<uint32_t> heap_block_ids;
+	perfect_set_t heap_block_ids;
 	//! Tuple count for this chunk
 	idx_t count;
+	//! Lock for recomputing heap pointers
+	unsafe_unique_ptr<mutex> lock;
 };
 
 struct TupleDataSegment {
@@ -112,13 +115,15 @@ public:
 	unsafe_vector<TupleDataChunk> chunks;
 	//! The tuple count of this segment
 	idx_t count;
+	//! The data size of this segment
+	idx_t data_size;
 
 	//! Lock for modifying pinned_handles
 	mutex pinned_handles_lock;
 	//! Where handles to row blocks will be stored with TupleDataPinProperties::KEEP_EVERYTHING_PINNED
-	vector<BufferHandle> pinned_row_handles;
+	unsafe_vector<BufferHandle> pinned_row_handles;
 	//! Where handles to heap blocks will be stored with TupleDataPinProperties::KEEP_EVERYTHING_PINNED
-	vector<BufferHandle> pinned_heap_handles;
+	unsafe_vector<BufferHandle> pinned_heap_handles;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/common/mutex.hpp"
+#include "duckdb/common/perfect_map_set.hpp"
 #include "duckdb/common/types.hpp"
 
 namespace duckdb {
@@ -26,8 +27,8 @@ enum class TupleDataPinProperties : uint8_t {
 };
 
 struct TupleDataPinState {
-	unordered_map<uint32_t, BufferHandle> row_handles;
-	unordered_map<uint32_t, BufferHandle> heap_handles;
+	perfect_map_t<BufferHandle> row_handles;
+	perfect_map_t<BufferHandle> heap_handles;
 	TupleDataPinProperties properties = TupleDataPinProperties::INVALID;
 };
 

package/src/duckdb/src/include/duckdb/common/types/vector.hpp

@@ -101,14 +101,14 @@ public:
 	DUCKDB_API void Reference(const Value &value);
 	//! Causes this vector to reference the data held by the other vector.
 	//! The type of the "other" vector should match the type of this vector
-	DUCKDB_API void Reference(Vector &other);
+	DUCKDB_API void Reference(const Vector &other);
 	//! Reinterpret the data of the other vector as the type of this vector
 	//! Note that this takes the data of the other vector as-is and places it in this vector
 	//! Without changing the type of this vector
-	DUCKDB_API void Reinterpret(Vector &other);
+	DUCKDB_API void Reinterpret(const Vector &other);
 
 	//! Causes this vector to reference the data held by the other vector, changes the type if required.
-	DUCKDB_API void ReferenceAndSetType(Vector &other);
+	DUCKDB_API void ReferenceAndSetType(const Vector &other);
 
 	//! Resets a vector from a vector cache.
 	//! This turns the vector back into an empty FlatVector with STANDARD_VECTOR_SIZE entries.

package/src/duckdb/src/include/duckdb/common/vector.hpp

@@ -88,14 +88,14 @@ public:
 	}
 
 	typename original::reference back() {
-		if (original::empty()) {
+		if (MemorySafety<SAFE>::enabled && original::empty()) {
 			throw InternalException("'back' called on an empty vector!");
 		}
 		return get<SAFE>(original::size() - 1);
 	}
 
 	typename original::const_reference back() const {
-		if (original::empty()) {
+		if (MemorySafety<SAFE>::enabled && original::empty()) {
 			throw InternalException("'back' called on an empty vector!");
 		}
 		return get<SAFE>(original::size() - 1);

package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp

@@ -8,12 +8,13 @@
 
 #pragma once
 
-#include "duckdb/common/types/row/tuple_data_collection.hpp"
+#include "duckdb/common/types/row/partitioned_tuple_data.hpp"
 #include "duckdb/execution/base_aggregate_hashtable.hpp"
 #include "duckdb/storage/arena_allocator.hpp"
 #include "duckdb/storage/buffer/buffer_handle.hpp"
 
 namespace duckdb {
+
 class BlockHandle;
 class BufferHandle;
 
@@ -27,91 +28,87 @@ struct FlushMoveState;
    stores them in the HT. It uses linear probing for collision resolution.
 */
 
-// two part hash table
-// hashes and payload
-// hashes layout:
-// [SALT][PAGE_NR][PAGE_OFFSET]
-// [SALT] are the high bits of the hash value, e.g. 16 for 64 bit hashes
-// [PAGE_NR] is the buffer managed payload page index
-// [PAGE_OFFSET] is the logical entry offset into said payload page
-
-// NOTE: PAGE_NR and PAGE_OFFSET are reversed for 64 bit HTs because struct packing
-
-// payload layout
-// [VALIDITY][GROUPS][HASH][PADDING][PAYLOAD]
-// [VALIDITY] is the validity bits of the data columns (including the HASH)
-// [GROUPS] is the group data, could be multiple values, fixed size, strings are elsewhere
-// [HASH] is the hash data of the groups
-// [PADDING] is gunk data to align payload properly
-// [PAYLOAD] is the payload (i.e. the aggregate states)
-struct aggr_ht_entry_64 {
-	uint16_t salt;
-	uint16_t page_offset;
-	uint32_t page_nr; // this has to come last because alignment
-};
+struct aggr_ht_entry_t {
+public:
+	explicit aggr_ht_entry_t(hash_t value_p) : value(value_p) {
+	}
 
-struct aggr_ht_entry_32 {
-	uint8_t salt;
-	uint8_t page_nr;
-	uint16_t page_offset;
-};
+	inline bool IsOccupied() const {
+		return value != 0;
+	}
+
+	inline data_ptr_t GetPointer() const {
+		D_ASSERT(IsOccupied());
+		return reinterpret_cast<data_ptr_t>(value & POINTER_MASK);
+	}
+	inline void SetPointer(const data_ptr_t &pointer) {
+		// Pointer shouldn't use upper bits
+		D_ASSERT((reinterpret_cast<uint64_t>(pointer) & SALT_MASK) == 0);
+		// Value should have all 1's in the pointer area
+		D_ASSERT((value & POINTER_MASK) == POINTER_MASK);
+		// Set upper bits to 1 in pointer so the salt stays intact
+		value &= reinterpret_cast<uint64_t>(pointer) | SALT_MASK;
+	}
 
-enum HtEntryType { HT_WIDTH_32, HT_WIDTH_64 };
+	static inline hash_t ExtractSalt(const hash_t &hash) {
+		// Leaves upper bits intact, sets lower bits to all 1's
+		return hash | POINTER_MASK;
+	}
+	inline hash_t GetSalt() const {
+		return ExtractSalt(value);
+	}
+	inline void SetSalt(const hash_t &salt) {
+		// Shouldn't be occupied when we set this
+		D_ASSERT(!IsOccupied());
+		// Salt should have all 1's in the pointer field
+		D_ASSERT((salt & POINTER_MASK) == POINTER_MASK);
+		// No need to mask, just put the whole thing there
+		value = salt;
+	}
 
-struct AggregateHTScanState {
-	mutex lock;
-	TupleDataScanState scan_state;
-};
+private:
+	//! Upper 16 bits are salt
+	static constexpr const hash_t SALT_MASK = 0xFFFF000000000000;
+	//! Lower 48 bits are the pointer
+	static constexpr const hash_t POINTER_MASK = 0x0000FFFFFFFFFFFF;
 
-struct AggregateHTAppendState {
-	AggregateHTAppendState();
-
-	Vector ht_offsets;
-	Vector hash_salts;
-	SelectionVector group_compare_vector;
-	SelectionVector no_match_vector;
-	SelectionVector empty_vector;
-	SelectionVector new_groups;
-	Vector addresses;
-	unsafe_unique_array<UnifiedVectorFormat> group_data;
-	DataChunk group_chunk;
-
-	TupleDataChunkState chunk_state;
-	bool chunk_state_initialized;
+	hash_t value;
 };
 
 class GroupedAggregateHashTable : public BaseAggregateHashTable {
-public:
-	//! The hash table load factor, when a resize is triggered
-	constexpr static float LOAD_FACTOR = 1.5;
-	constexpr static uint8_t HASH_WIDTH = sizeof(hash_t);
-
 public:
 	GroupedAggregateHashTable(ClientContext &context, Allocator &allocator, vector<LogicalType> group_types,
 	                          vector<LogicalType> payload_types, const vector<BoundAggregateExpression *> &aggregates,
-	                          HtEntryType entry_type = HtEntryType::HT_WIDTH_64,
-	                          idx_t initial_capacity = InitialCapacity());
+	                          idx_t initial_capacity = InitialCapacity(), idx_t radix_bits = 0);
 	GroupedAggregateHashTable(ClientContext &context, Allocator &allocator, vector<LogicalType> group_types,
 	                          vector<LogicalType> payload_types, vector<AggregateObject> aggregates,
-	                          HtEntryType entry_type = HtEntryType::HT_WIDTH_64,
-	                          idx_t initial_capacity = InitialCapacity());
+	                          idx_t initial_capacity = InitialCapacity(), idx_t radix_bits = 0);
 	GroupedAggregateHashTable(ClientContext &context, Allocator &allocator, vector<LogicalType> group_types);
 	~GroupedAggregateHashTable() override;
 
 public:
+	//! The hash table load factor, when a resize is triggered
+	constexpr static float LOAD_FACTOR = 1.5;
+
+	//! Get the layout of this HT
+	const TupleDataLayout &GetLayout() const;
+	//! Number of groups in the HT
+	idx_t Count() const;
+	//! Initial capacity of the HT
+	static idx_t InitialCapacity();
+	//! Capacity that can hold 'count' entries without resizing
+	static idx_t GetCapacityForCount(idx_t count);
+	//! Current capacity of the HT
+	idx_t Capacity() const;
+	//! Threshold at which to resize the HT
+	idx_t ResizeThreshold() const;
+
 	//! Add the given data to the HT, computing the aggregates grouped by the
 	//! data in the group chunk. When resize = true, aggregates will not be
 	//! computed but instead just assigned.
-	idx_t AddChunk(AggregateHTAppendState &state, DataChunk &groups, DataChunk &payload,
-	               const unsafe_vector<idx_t> &filter);
-	idx_t AddChunk(AggregateHTAppendState &state, DataChunk &groups, Vector &group_hashes, DataChunk &payload,
-	               const unsafe_vector<idx_t> &filter);
-	idx_t AddChunk(AggregateHTAppendState &state, DataChunk &groups, DataChunk &payload, AggregateType filter);
-
-	//! Scan the HT starting from the scan_position until the result and group
-	//! chunks are filled. scan_position will be updated by this function.
-	//! Returns the amount of elements found.
-	idx_t Scan(TupleDataParallelScanState &gstate, TupleDataLocalScanState &lstate, DataChunk &result);
+	idx_t AddChunk(DataChunk &groups, DataChunk &payload, const unsafe_vector<idx_t> &filter);
+	idx_t AddChunk(DataChunk &groups, Vector &group_hashes, DataChunk &payload, const unsafe_vector<idx_t> &filter);
+	idx_t AddChunk(DataChunk &groups, DataChunk &payload, AggregateType filter);
 
 	//! Fetch the aggregates for specific groups from the HT and place them in the result
 	void FetchAggregates(DataChunk &groups, DataChunk &result);
@@ -119,108 +116,90 @@ public:
 	//! Finds or creates groups in the hashtable using the specified group keys. The addresses vector will be filled
 	//! with pointers to the groups in the hash table, and the new_groups selection vector will point to the newly
 	//! created groups. The return value is the amount of newly created groups.
-	idx_t FindOrCreateGroups(AggregateHTAppendState &state, DataChunk &groups, Vector &group_hashes,
-	                         Vector &addresses_out, SelectionVector &new_groups_out);
-	idx_t FindOrCreateGroups(AggregateHTAppendState &state, DataChunk &groups, Vector &addresses_out,
+	idx_t FindOrCreateGroups(DataChunk &groups, Vector &group_hashes, Vector &addresses_out,
 	                         SelectionVector &new_groups_out);
-	void FindOrCreateGroups(AggregateHTAppendState &state, DataChunk &groups, Vector &addresses_out);
-
-	//! Executes the filter(if any) and update the aggregates
-	void Combine(GroupedAggregateHashTable &other);
-
-	//! Appends the data in the other HT to this one
-	void Append(GroupedAggregateHashTable &other);
-
-	TupleDataCollection &GetDataCollection() {
-		return *data_collection;
-	}
-
-	idx_t Count() const {
-		return data_collection->Count();
-	}
-
-	idx_t DataSize() const {
-		return data_collection->SizeInBytes();
-	}
-
-	static idx_t InitialCapacity();
-	idx_t Capacity() {
-		return capacity;
-	}
-
-	static idx_t FirstPartSize(idx_t count, HtEntryType entry_type) {
-		idx_t entry_size = entry_type == HT_WIDTH_32 ? sizeof(aggr_ht_entry_32) : sizeof(aggr_ht_entry_64);
-		return NextPowerOfTwo(count * 2L) * entry_size;
-	}
+	idx_t FindOrCreateGroups(DataChunk &groups, Vector &addresses_out, SelectionVector &new_groups_out);
+	void FindOrCreateGroups(DataChunk &groups, Vector &addresses_out);
 
-	idx_t TotalSize() const {
-		return DataSize() + FirstPartSize(Count(), entry_type);
-	}
+	unique_ptr<PartitionedTupleData> &GetPartitionedData();
+	shared_ptr<ArenaAllocator> GetAggregateAllocator();
 
-	idx_t ResizeThreshold();
-	idx_t MaxCapacity();
-	static idx_t GetMaxCapacity(HtEntryType entry_type, idx_t tuple_size);
+	//! Resize the HT to the specified size. Must be larger than the current size.
+	void Resize(idx_t size);
+	//! Resets the pointer table of the HT to all 0's
+	void ClearPointerTable();
+	//! Resets the group count to 0
+	void ResetCount();
+	//! Set the radix bits for this HT
+	void SetRadixBits(idx_t radix_bits);
+	//! Initializes the PartitionedTupleData
+	void InitializePartitionedData();
 
-	void Partition(vector<GroupedAggregateHashTable *> &partition_hts, idx_t radix_bits, bool sink_done);
-	void InitializeFirstPart();
+	//! Executes the filter(if any) and update the aggregates
+	void Combine(GroupedAggregateHashTable &other);
+	void Combine(TupleDataCollection &other_data);
 
-	void Finalize();
+	//! Unpins the data blocks
+	void UnpinData();
 
 private:
-	HtEntryType entry_type;
-
-	//! The capacity of the HT. This can be increased using GroupedAggregateHashTable::Resize
-	idx_t capacity;
-	//! Tuple width
-	idx_t tuple_size;
-	//! Tuples per block
-	idx_t tuples_per_block;
+	//! Append state
+	struct AggregateHTAppendState {
+		AggregateHTAppendState();
+
+		PartitionedTupleDataAppendState append_state;
+
+		Vector ht_offsets;
+		Vector hash_salts;
+		SelectionVector group_compare_vector;
+		SelectionVector no_match_vector;
+		SelectionVector empty_vector;
+		SelectionVector new_groups;
+		Vector addresses;
+		unsafe_unique_array<UnifiedVectorFormat> group_data;
+		DataChunk group_chunk;
+	} state;
+
+	//! The number of radix bits to partition by
+	idx_t radix_bits;
 	//! The data of the HT
-	unique_ptr<TupleDataCollection> data_collection;
-	TupleDataPinState td_pin_state;
-	vector<data_ptr_t> payload_hds_ptrs;
+	unique_ptr<PartitionedTupleData> partitioned_data;
 
-	//! The hashes of the HT
-	AllocatedData hashes_hdl;
-	data_ptr_t hashes_hdl_ptr;
-	idx_t hash_offset; // Offset into the layout of the hash column
-
-	hash_t hash_prefix_shift;
+	//! Predicates for matching groups (always ExpressionType::COMPARE_EQUAL)
+	vector<ExpressionType> predicates;
 
+	//! The number of groups in the HT
+	idx_t count;
+	//! The capacity of the HT. This can be increased using GroupedAggregateHashTable::Resize
+	idx_t capacity;
+	//! The hash map (pointer table) of the HT: allocated data and pointer into it
+	AllocatedData hash_map;
+	aggr_ht_entry_t *entries;
+	//! Offset of the hash column in the rows
+	idx_t hash_offset;
 	//! Bitmask for getting relevant bits from the hashes to determine the position
 	hash_t bitmask;
 
-	bool is_finalized;
-
-	vector<ExpressionType> predicates;
-
 	//! The active arena allocator used by the aggregates for their internal state
 	shared_ptr<ArenaAllocator> aggregate_allocator;
 	//! Owning arena allocators that this HT has data from
 	vector<shared_ptr<ArenaAllocator>> stored_allocators;
 
 private:
+	//! Disabled the copy constructor
 	GroupedAggregateHashTable(const GroupedAggregateHashTable &) = delete;
-
+	//! Destroy the HT
 	void Destroy();
-	void Verify();
-	template <class ENTRY>
-	void VerifyInternal();
-	//! Resize the HT to the specified size. Must be larger than the current size.
-	template <class ENTRY>
-	void Resize(idx_t size);
-	//! Initializes the first part of the HT
-	template <class ENTRY>
-	void InitializeHashes();
+
+	//! Apply bitmask to get the entry in the HT
+	inline idx_t ApplyBitMask(hash_t hash) const;
+
 	//! Does the actual group matching / creation
-	template <class ENTRY>
-	idx_t FindOrCreateGroupsInternal(DataChunk &groups, Vector &group_hashes_v, Vector &addresses_v,
+	idx_t FindOrCreateGroupsInternal(DataChunk &groups, Vector &group_hashes, Vector &addresses,
 	                                 SelectionVector &new_groups);
-	//! Updates payload_hds_ptrs with the new pointers (after appending to data_collection)
-	void UpdateBlockPointers();
-	template <class ENTRY>
-	idx_t FindOrCreateGroupsInternal(AggregateHTAppendState &state, DataChunk &groups, Vector &group_hashes,
-	                                 Vector &addresses, SelectionVector &new_groups);
+
+	//! Verify the pointer table of the HT
+	void Verify();
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp

@@ -8,17 +8,18 @@
 
 #pragma once
 
+#include "duckdb/execution/operator/aggregate/distinct_aggregate_data.hpp"
+#include "duckdb/execution/operator/aggregate/grouped_aggregate_data.hpp"
 #include "duckdb/execution/physical_operator.hpp"
-#include "duckdb/storage/data_table.hpp"
-#include "duckdb/parser/group_by_node.hpp"
 #include "duckdb/execution/radix_partitioned_hashtable.hpp"
-#include "duckdb/execution/operator/aggregate/grouped_aggregate_data.hpp"
-#include "duckdb/execution/operator/aggregate/distinct_aggregate_data.hpp"
+#include "duckdb/parser/group_by_node.hpp"
+#include "duckdb/storage/data_table.hpp"
 
 namespace duckdb {
 
 class ClientContext;
 class BufferManager;
+class PhysicalHashAggregate;
 
 struct HashAggregateGroupingData {
 public: