duckdb 0.7.2-dev1901.0 → 0.7.2-dev2233.0

package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp

@@ -9,52 +9,63 @@
 #pragma once
 
 #include "duckdb/common/fast_mem.hpp"
-#include "duckdb/common/types/partitioned_column_data.hpp"
+#include "duckdb/common/types/column/partitioned_column_data.hpp"
+#include "duckdb/common/types/row/partitioned_tuple_data.hpp"
 
 namespace duckdb {
 
 class BufferManager;
-class RowLayout;
-class RowDataCollection;
 class Vector;
 struct UnifiedVectorFormat;
 struct SelectionVector;
 
-//! Templated radix partitioning constants, can be templated to the number of radix bits
-template <idx_t radix_bits>
-struct RadixPartitioningConstants {
+//! Generic radix partitioning functions
+struct RadixPartitioning {
 public:
-	static constexpr const idx_t NUM_RADIX_BITS = radix_bits;
-	static constexpr const idx_t NUM_PARTITIONS = (idx_t)1 << NUM_RADIX_BITS;
-	static constexpr const idx_t TMP_BUF_SIZE = 8;
+	//! The number of partitions for a given number of radix bits
+	static inline constexpr idx_t NumberOfPartitions(idx_t radix_bits) {
+		return idx_t(1) << radix_bits;
+	}
 
-public:
-	//! Apply bitmask on the highest bits, and right shift to get a number between 0 and NUM_PARTITIONS
-	static inline hash_t ApplyMask(hash_t hash) {
-		return (hash & MASK) >> (sizeof(hash_t) * 8 - NUM_RADIX_BITS);
+	//! Inverse of NumberOfPartitions, given a number of partitions, get the number of radix bits
+	static inline idx_t RadixBits(idx_t n_partitions) {
+		D_ASSERT(IsPowerOfTwo(n_partitions));
+		for (idx_t r = 0; r < sizeof(idx_t) * 8; r++) {
+			if (n_partitions == NumberOfPartitions(r)) {
+				return r;
+			}
+		}
+		throw InternalException("RadixPartitioning::RadixBits unable to find partition count!");
 	}
 
-private:
-	//! Bitmask of the highest bits
-	static constexpr const hash_t MASK = hash_t(-1) ^ ((hash_t(1) << (sizeof(hash_t) * 8 - NUM_RADIX_BITS)) - 1);
-};
+	static inline constexpr idx_t Shift(idx_t radix_bits) {
+		return 48 - radix_bits;
+	}
 
-//! Generic radix partitioning functions
-struct RadixPartitioning {
-public:
-	static idx_t NumberOfPartitions(idx_t radix_bits) {
-		return (idx_t)1 << radix_bits;
+	static inline constexpr hash_t Mask(idx_t radix_bits) {
+		return (hash_t(1 << radix_bits) - 1) << Shift(radix_bits);
 	}
 
 	//! Select using a cutoff on the radix bits of the hash
 	static idx_t Select(Vector &hashes, const SelectionVector *sel, idx_t count, idx_t radix_bits, idx_t cutoff,
 	                    SelectionVector *true_sel, SelectionVector *false_sel);
+};
+
+//! Templated radix partitioning constants, can be templated to the number of radix bits
+template <idx_t radix_bits>
+struct RadixPartitioningConstants {
+public:
+	//! Bitmask of the upper bits of the 5th byte
+	static constexpr const idx_t NUM_PARTITIONS = RadixPartitioning::NumberOfPartitions(radix_bits);
+	static constexpr const idx_t SHIFT = RadixPartitioning::Shift(radix_bits);
+	static constexpr const hash_t MASK = RadixPartitioning::Mask(radix_bits);
 
-	//! Partition the data in block_collection/string_heap to multiple partitions
-	static void PartitionRowData(BufferManager &buffer_manager, const RowLayout &layout, const idx_t hash_offset,
-	                             RowDataCollection &block_collection, RowDataCollection &string_heap,
-	                             vector<unique_ptr<RowDataCollection>> &partition_block_collections,
-	                             vector<unique_ptr<RowDataCollection>> &partition_string_heaps, idx_t radix_bits);
+public:
+	//! Apply bitmask and right shift to get a number between 0 and NUM_PARTITIONS
+	static inline hash_t ApplyMask(hash_t hash) {
+		D_ASSERT((hash & MASK) >> SHIFT < NUM_PARTITIONS);
+		return (hash & MASK) >> SHIFT;
+	}
 };
 
 //! RadixPartitionedColumnData is a PartitionedColumnData that partitions input based on the radix of a hash
@@ -87,6 +98,7 @@ protected:
 			return GetBufferSize(1 << 4);
 		}
 	}
+
 	void InitializeAppendStateInternal(PartitionedColumnDataAppendState &state) const override;
 	void ComputePartitionIndices(PartitionedColumnDataAppendState &state, DataChunk &input) override;
 
@@ -101,4 +113,45 @@ private:
 	const idx_t hash_col_idx;
 };
 
+//! RadixPartitionedTupleData is a PartitionedTupleData that partitions input based on the radix of a hash
+class RadixPartitionedTupleData : public PartitionedTupleData {
+public:
+	RadixPartitionedTupleData(BufferManager &buffer_manager, const TupleDataLayout &layout, idx_t radix_bits_p,
+	                          idx_t hash_col_idx_p);
+	RadixPartitionedTupleData(const RadixPartitionedTupleData &other);
+	~RadixPartitionedTupleData() override;
+
+	idx_t GetRadixBits() const {
+		return radix_bits;
+	}
+
+private:
+	void Initialize();
+
+protected:
+	//===--------------------------------------------------------------------===//
+	// Radix Partitioning interface implementation
+	//===--------------------------------------------------------------------===//
+	void InitializeAppendStateInternal(PartitionedTupleDataAppendState &state,
+	                                   TupleDataPinProperties properties) const override;
+	void ComputePartitionIndices(PartitionedTupleDataAppendState &state, DataChunk &input) override;
+	void ComputePartitionIndices(Vector &row_locations, idx_t count, Vector &partition_indices) const override;
+	idx_t MaxPartitionIndex() const override {
+		return RadixPartitioning::NumberOfPartitions(radix_bits) - 1;
+	}
+
+	bool RepartitionReverseOrder() const override {
+		return true;
+	}
+	void RepartitionFinalizeStates(PartitionedTupleData &old_partitioned_data,
+	                               PartitionedTupleData &new_partitioned_data, PartitionedTupleDataAppendState &state,
+	                               idx_t finished_partition_idx) const override;
+
+private:
+	//! The number of radix bits
+	const idx_t radix_bits;
+	//! The index of the column holding the hashes
+	const idx_t hash_col_idx;
+};
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/reference_map.hpp

@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/reference_map.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/unordered_map.hpp"
+#include "duckdb/common/unordered_set.hpp"
+
+namespace duckdb {
+class Expression;
+
+template <class T>
+struct ReferenceHashFunction {
+	uint64_t operator()(const reference<T> &ref) const {
+		return std::hash<void *>()((void *)&ref.get());
+	}
+};
+
+template <class T>
+struct ReferenceEquality {
+	bool operator()(const reference<T> &a, const reference<T> &b) const {
+		return &a.get() == &b.get();
+	}
+};
+
+template <typename T, typename TGT>
+using reference_map_t = unordered_map<reference<T>, TGT, ReferenceHashFunction<T>, ReferenceEquality<T>>;
+
+template <typename T>
+using reference_set_t = unordered_set<reference<T>, ReferenceHashFunction<T>, ReferenceEquality<T>>;
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp

@@ -18,6 +18,7 @@ struct AggregateObject;
 struct AggregateFilterData;
 class DataChunk;
 class RowLayout;
+class TupleDataLayout;
 class RowDataCollection;
 struct SelectionVector;
 class StringHeap;
@@ -37,9 +38,9 @@ struct RowOperations {
 	// Aggregation Operators
 	//===--------------------------------------------------------------------===//
 	//! initialize - unaligned addresses
-	static void InitializeStates(RowLayout &layout, Vector &addresses, const SelectionVector &sel, idx_t count);
+	static void InitializeStates(TupleDataLayout &layout, Vector &addresses, const SelectionVector &sel, idx_t count);
 	//! destructor - unaligned addresses, updated
-	static void DestroyStates(RowOperationsState &state, RowLayout &layout, Vector &addresses, idx_t count);
+	static void DestroyStates(RowOperationsState &state, TupleDataLayout &layout, Vector &addresses, idx_t count);
 	//! update - aligned addresses
 	static void UpdateStates(RowOperationsState &state, AggregateObject &aggr, Vector &addresses, DataChunk &payload,
 	                         idx_t arg_idx, idx_t count);
@@ -47,10 +48,10 @@ struct RowOperations {
 	static void UpdateFilteredStates(RowOperationsState &state, AggregateFilterData &filter_data, AggregateObject &aggr,
 	                                 Vector &addresses, DataChunk &payload, idx_t arg_idx);
 	//! combine - unaligned addresses, updated
-	static void CombineStates(RowOperationsState &state, RowLayout &layout, Vector &sources, Vector &targets,
+	static void CombineStates(RowOperationsState &state, TupleDataLayout &layout, Vector &sources, Vector &targets,
 	                          idx_t count);
 	//! finalize - unaligned addresses, updated
-	static void FinalizeStates(RowOperationsState &state, RowLayout &layout, Vector &addresses, DataChunk &result,
+	static void FinalizeStates(RowOperationsState &state, TupleDataLayout &layout, Vector &addresses, DataChunk &result,
 	                           idx_t aggr_idx);
 
 	//===--------------------------------------------------------------------===//
@@ -66,7 +67,7 @@ struct RowOperations {
 	                   const idx_t count, const RowLayout &layout, const idx_t col_no, const idx_t build_size = 0,
 	                   data_ptr_t heap_ptr = nullptr);
 	//! Full Scan an entire columns
-	static void FullScanColumn(const RowLayout &layout, Vector &rows, Vector &col, idx_t count, idx_t col_idx);
+	static void FullScanColumn(const TupleDataLayout &layout, Vector &rows, Vector &col, idx_t count, idx_t col_idx);
 
 	//===--------------------------------------------------------------------===//
 	// Comparison Operators
@@ -76,7 +77,7 @@ struct RowOperations {
 	//! Returns the number of matches remaining in the selection.
 	using Predicates = vector<ExpressionType>;
 
-	static idx_t Match(DataChunk &columns, UnifiedVectorFormat col_data[], const RowLayout &layout, Vector &rows,
+	static idx_t Match(DataChunk &columns, UnifiedVectorFormat col_data[], const TupleDataLayout &layout, Vector &rows,
 	                   const Predicates &predicates, SelectionVector &sel, idx_t count, SelectionVector *no_match,
 	                   idx_t &no_match_count);
 

package/src/duckdb/src/include/duckdb/common/sort/comparators.hpp

@@ -9,7 +9,7 @@
 #pragma once
 
 #include "duckdb/common/types.hpp"
-#include "duckdb/common/types/row_layout.hpp"
+#include "duckdb/common/types/row/row_layout.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp

@@ -9,7 +9,7 @@
 #pragma once
 
 #include "duckdb/common/sort/sort.hpp"
-#include "duckdb/common/types/partitioned_column_data.hpp"
+#include "duckdb/common/types/column/partitioned_column_data.hpp"
 #include "duckdb/common/radix_partitioning.hpp"
 #include "duckdb/parallel/base_pipeline_event.hpp"
 

package/src/duckdb/src/include/duckdb/common/sort/sort.hpp

@@ -9,7 +9,7 @@
 #pragma once
 
 #include "duckdb/common/sort/sorted_block.hpp"
-#include "duckdb/common/types/row_data_collection.hpp"
+#include "duckdb/common/types/row/row_data_collection.hpp"
 #include "duckdb/planner/bound_query_node.hpp"
 
 namespace duckdb {

package/src/duckdb/src/include/duckdb/common/sort/sorted_block.hpp

@@ -9,8 +9,8 @@
 
 #include "duckdb/common/fast_mem.hpp"
 #include "duckdb/common/sort/comparators.hpp"
-#include "duckdb/common/types/row_data_collection_scanner.hpp"
-#include "duckdb/common/types/row_layout.hpp"
+#include "duckdb/common/types/row/row_data_collection_scanner.hpp"
+#include "duckdb/common/types/row/row_layout.hpp"
 #include "duckdb/storage/buffer/buffer_handle.hpp"
 
 namespace duckdb {

package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp

@@ -9,7 +9,7 @@
 #pragma once
 
 #include "duckdb/common/map.hpp"
-#include "duckdb/common/types/column_data_collection.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
 
 namespace duckdb {
 class BufferManager;

package/src/duckdb/src/include/duckdb/common/types/{column_data_allocator.hpp → column/column_data_allocator.hpp}

@@ -1,14 +1,14 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/common/types/column_data_allocator.hpp
+// duckdb/common/types/column/column_data_allocator.hpp
 //
 //
 //===----------------------------------------------------------------------===//
 
 #pragma once
 
-#include "duckdb/common/types/column_data_collection.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
 
 namespace duckdb {
 
@@ -28,8 +28,8 @@ struct BlockMetaData {
 
 class ColumnDataAllocator {
 public:
-	ColumnDataAllocator(Allocator &allocator);
-	ColumnDataAllocator(BufferManager &buffer_manager);
+	explicit ColumnDataAllocator(Allocator &allocator);
+	explicit ColumnDataAllocator(BufferManager &buffer_manager);
 	ColumnDataAllocator(ClientContext &context, ColumnDataAllocatorType allocator_type);
 	ColumnDataAllocator(ColumnDataAllocator &allocator);
 

package/src/duckdb/src/include/duckdb/common/types/{column_data_collection.hpp → column/column_data_collection.hpp}

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/common/types/column_data_collection.hpp
+// duckdb/common/types/column/column_data_collection.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -9,7 +9,7 @@
 #pragma once
 
 #include "duckdb/common/pair.hpp"
-#include "duckdb/common/types/column_data_collection_iterators.hpp"
+#include "duckdb/common/types/column/column_data_collection_iterators.hpp"
 
 namespace duckdb {
 class BufferManager;
@@ -131,8 +131,8 @@ public:
 	//! Compare two column data collections to another. If they are equal according to result equality rules,
 	//! return true. That means null values are equal, and approx equality is used for floating point values.
 	//! If they are not equal, return false and fill in the error message.
-	static bool ResultEquals(const ColumnDataCollection &left, const ColumnDataCollection &right,
-	                         string &error_message);
+	static bool ResultEquals(const ColumnDataCollection &left, const ColumnDataCollection &right, string &error_message,
+	                         bool ordered = false);
 
 	//! Obtains the next scan index to scan from
 	bool NextScanIndex(ColumnDataScanState &state, idx_t &chunk_index, idx_t &segment_index, idx_t &row_index) const;

package/src/duckdb/src/include/duckdb/common/types/{column_data_collection_iterators.hpp → column/column_data_collection_iterators.hpp}

@@ -1,14 +1,14 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/common/types/column_data_collection_iterators.hpp
+// duckdb/common/types/column/column_data_collection_iterators.hpp
 //
 //
 //===----------------------------------------------------------------------===//
 
 #pragma once
 
-#include "duckdb/common/types/column_data_scan_states.hpp"
+#include "duckdb/common/types/column/column_data_scan_states.hpp"
 
 namespace duckdb {
 class ColumnDataCollection;

package/src/duckdb/src/include/duckdb/common/types/{column_data_collection_segment.hpp → column/column_data_collection_segment.hpp}

@@ -1,15 +1,15 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/common/types/column_data_collection_segment.hpp
+// duckdb/common/types/column/column_data_collection_segment.hpp
 //
 //
 //===----------------------------------------------------------------------===//
 
 #pragma once
 
-#include "duckdb/common/types/column_data_allocator.hpp"
-#include "duckdb/common/types/column_data_collection.hpp"
+#include "duckdb/common/types/column/column_data_allocator.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/common/types/{column_data_consumer.hpp → column/column_data_consumer.hpp}

@@ -1,16 +1,16 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/common/types/column_data_consumer.hpp
+// duckdb/common/types/column/column_data_consumer.hpp
 //
 //
 //===----------------------------------------------------------------------===//
 
 #pragma once
 
-#include "duckdb/common/types/column_data_collection.hpp"
-#include "duckdb/common/types/column_data_collection_segment.hpp"
-#include "duckdb/common/types/column_data_scan_states.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
+#include "duckdb/common/types/column/column_data_collection_segment.hpp"
+#include "duckdb/common/types/column/column_data_scan_states.hpp"
 
 namespace duckdb {
 
@@ -44,6 +44,10 @@ public:
 public:
 	ColumnDataConsumer(ColumnDataCollection &collection, vector<column_t> column_ids);
 
+	idx_t Count() const {
+		return collection.Count();
+	}
+
 	idx_t ChunkCount() const {
 		return chunk_count;
 	}

package/src/duckdb/src/include/duckdb/common/types/{column_data_scan_states.hpp → column/column_data_scan_states.hpp}

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/common/types/column_data_scan_states.hpp
+// duckdb/common/types/column/column_data_scan_states.hpp
 //
 //
 //===----------------------------------------------------------------------===//

package/src/duckdb/src/include/duckdb/common/types/{partitioned_column_data.hpp → column/partitioned_column_data.hpp}

@@ -1,15 +1,16 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/common/types/partitioned_column_data.hpp
+// duckdb/common/types/column/partitioned_column_data.hpp
 //
 //
 //===----------------------------------------------------------------------===//
 
 #pragma once
 
-#include "duckdb/common/types/column_data_allocator.hpp"
-#include "duckdb/common/types/column_data_collection.hpp"
+#include "duckdb/common/perfect_map_set.hpp"
+#include "duckdb/common/types/column/column_data_allocator.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
 
 namespace duckdb {
 
@@ -22,16 +23,23 @@ public:
 public:
 	Vector partition_indices;
 	SelectionVector partition_sel;
+	perfect_map_t<list_entry_t> partition_entries;
 	DataChunk slice_chunk;
 
 	vector<unique_ptr<DataChunk>> partition_buffers;
 	vector<unique_ptr<ColumnDataAppendState>> partition_append_states;
 };
 
-enum class PartitionedColumnDataType : uint8_t { RADIX, HIVE, INVALID };
+enum class PartitionedColumnDataType : uint8_t {
+	INVALID,
+	//! Radix partitioning on a hash column
+	RADIX,
+	//! Hive-style multi-field partitioning
+	HIVE
+};
 
 //! Shared allocators for parallel partitioning
-struct PartitionAllocators {
+struct PartitionColumnDataAllocators {
 	mutex lock;
 	vector<shared_ptr<ColumnDataAllocator>> allocators;
 };
@@ -80,7 +88,7 @@ protected:
 
 	//! If the buffer is half full, we append to the partition
 	inline idx_t HalfBufferSize() const {
-		D_ASSERT((BufferSize() & (BufferSize() - 1)) == 0); // BufferSize should be a power of two
+		D_ASSERT(IsPowerOfTwo(BufferSize()));
 		return BufferSize() / 2;
 	}
 	//! Create a new shared allocator
@@ -98,7 +106,7 @@ protected:
 	vector<LogicalType> types;
 
 	mutex lock;
-	shared_ptr<PartitionAllocators> allocators;
+	shared_ptr<PartitionColumnDataAllocators> allocators;
 	vector<unique_ptr<ColumnDataCollection>> partitions;
 };
 

package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp

@@ -0,0 +1,140 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/types/row/partitioned_tuple_data.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/perfect_map_set.hpp"
+#include "duckdb/common/types/row/tuple_data_allocator.hpp"
+#include "duckdb/common/types/row/tuple_data_collection.hpp"
+
+namespace duckdb {
+
+//! Local state for parallel partitioning
+struct PartitionedTupleDataAppendState {
+public:
+	PartitionedTupleDataAppendState() : partition_indices(LogicalType::UBIGINT) {
+	}
+
+public:
+	Vector partition_indices;
+	SelectionVector partition_sel;
+
+	static constexpr idx_t MAP_THRESHOLD = 32;
+	perfect_map_t<list_entry_t> partition_entries;
+	list_entry_t partition_entries_arr[MAP_THRESHOLD];
+
+	vector<unique_ptr<TupleDataPinState>> partition_pin_states;
+	TupleDataChunkState chunk_state;
+};
+
+enum class PartitionedTupleDataType : uint8_t {
+	INVALID,
+	//! Radix partitioning on a hash column
+	RADIX
+};
+
+//! Shared allocators for parallel partitioning
+struct PartitionTupleDataAllocators {
+	mutex lock;
+	vector<shared_ptr<TupleDataAllocator>> allocators;
+};
+
+//! PartitionedTupleData represents partitioned row data, which serves as an interface for different types of
+//! partitioning, e.g., radix, hive
+class PartitionedTupleData {
+public:
+	unique_ptr<PartitionedTupleData> CreateShared();
+	virtual ~PartitionedTupleData();
+
+public:
+	//! Get the partitioning type of this PartitionedTupleData
+	PartitionedTupleDataType GetType() const;
+	//! Initializes a local state for parallel partitioning that can be merged into this PartitionedTupleData
+	void InitializeAppendState(PartitionedTupleDataAppendState &state,
+	                           TupleDataPinProperties properties = TupleDataPinProperties::UNPIN_AFTER_DONE) const;
+	//! Appends a DataChunk to this PartitionedTupleData
+	void Append(PartitionedTupleDataAppendState &state, DataChunk &input);
+	//! Appends rows to this PartitionedTupleData
+	void Append(PartitionedTupleDataAppendState &state, TupleDataChunkState &input, idx_t count);
+	//! Flushes any remaining data in the append state into this PartitionedTupleData
+	void FlushAppendState(PartitionedTupleDataAppendState &state);
+	//! Combine another PartitionedTupleData into this PartitionedTupleData
+	void Combine(PartitionedTupleData &other);
+	//! Partition a TupleDataCollection
+	void Partition(TupleDataCollection &source,
+	               TupleDataPinProperties properties = TupleDataPinProperties::UNPIN_AFTER_DONE);
+	//! Repartition this PartitionedTupleData into the new PartitionedTupleData
+	void Repartition(PartitionedTupleData &new_partitioned_data);
+	//! Get the partitions in this PartitionedTupleData
+	vector<unique_ptr<TupleDataCollection>> &GetPartitions();
+	//! Get the count of this PartitionedTupleData
+	idx_t Count() const;
+	//! Get the size (in bytes) of this PartitionedTupleData
+	idx_t SizeInBytes() const;
+
+protected:
+	//===--------------------------------------------------------------------===//
+	// Partitioning type implementation interface
+	//===--------------------------------------------------------------------===//
+	//! Initialize a PartitionedTupleDataAppendState for this type of partitioning (optional)
+	virtual void InitializeAppendStateInternal(PartitionedTupleDataAppendState &state,
+	                                           TupleDataPinProperties properties) const {
+	}
+	//! Compute the partition indices for this type of partitioning for the input DataChunk and store them in the
+	//! `partition_data` of the local state. If this type creates partitions on the fly (for, e.g., hive), this
+	//! function is also in charge of creating new partitions and mapping the input data to a partition index
+	virtual void ComputePartitionIndices(PartitionedTupleDataAppendState &state, DataChunk &input) {
+		throw NotImplementedException("ComputePartitionIndices for this type of PartitionedTupleData");
+	}
+	//! Compute partition indices from rows (similar to function above)
+	virtual void ComputePartitionIndices(Vector &row_locations, idx_t count, Vector &partition_indices) const {
+		throw NotImplementedException("ComputePartitionIndices for this type of PartitionedTupleData");
+	}
+	//! Maximum partition index (optional)
+	virtual idx_t MaxPartitionIndex() const {
+		return DConstants::INVALID_INDEX;
+	}
+
+	//! Whether or not to iterate over the original partitions in reverse order when repartitioning (optional)
+	virtual bool RepartitionReverseOrder() const {
+		return false;
+	}
+	//! Finalize states while repartitioning - useful for unpinning blocks that are no longer needed (optional)
+	virtual void RepartitionFinalizeStates(PartitionedTupleData &old_partitioned_data,
+	                                       PartitionedTupleData &new_partitioned_data,
+	                                       PartitionedTupleDataAppendState &state, idx_t finished_partition_idx) const {
+	}
+
+protected:
+	//! PartitionedTupleData can only be instantiated by derived classes
+	PartitionedTupleData(PartitionedTupleDataType type, BufferManager &buffer_manager, const TupleDataLayout &layout);
+	PartitionedTupleData(const PartitionedTupleData &other);
+
+	//! Create a new shared allocator
+	void CreateAllocator();
+	//! Builds a selection vector in the Append state for the partitions
+	//! - returns true if everything belongs to the same partition - stores partition index in single_partition_idx
+	void BuildPartitionSel(PartitionedTupleDataAppendState &state, idx_t count);
+	//! Builds out the buffer space in the partitions
+	void BuildBufferSpace(PartitionedTupleDataAppendState &state);
+	//! Create a collection for a specific a partition
+	unique_ptr<TupleDataCollection> CreatePartitionCollection(idx_t partition_index) const {
+		return make_uniq<TupleDataCollection>(allocators->allocators[partition_index]);
+	}
+
+protected:
+	PartitionedTupleDataType type;
+	BufferManager &buffer_manager;
+	const TupleDataLayout layout;
+
+	mutex lock;
+	shared_ptr<PartitionTupleDataAllocators> allocators;
+	vector<unique_ptr<TupleDataCollection>> partitions;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/types/{row_data_collection.hpp → row/row_data_collection.hpp}

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/common/types/row_data_collection.hpp
+// duckdb/common/types/row/row_data_collection.hpp
 //
 //
 //===----------------------------------------------------------------------===//

package/src/duckdb/src/include/duckdb/common/types/{row_data_collection_scanner.hpp → row/row_data_collection_scanner.hpp}

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/common/types/row_data_collection_scanner.hpp
+// duckdb/common/types/row/row_data_collection_scanner.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -9,7 +9,7 @@
 #pragma once
 
 #include "duckdb/common/types.hpp"
-#include "duckdb/common/types/row_layout.hpp"
+#include "duckdb/common/types/row/row_layout.hpp"
 
 namespace duckdb {