duckdb 0.8.2-dev37.0 → 0.8.2-dev3989.0

package/src/duckdb/src/include/duckdb/common/type_util.hpp

@@ -1,3 +1,11 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/type_util.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
 #pragma once
 
 #include "duckdb/common/types.hpp"

package/src/duckdb/src/include/duckdb/common/typedefs.hpp

@@ -1,3 +1,11 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/typedefs.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
 #pragma once
 
 #include <cstdint>

package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp

@@ -24,7 +24,7 @@ struct BatchedChunkScanState {
 //! Scans over a BatchedDataCollection are ordered by batch index
 class BatchedDataCollection {
 public:
-	DUCKDB_API BatchedDataCollection(vector<LogicalType> types);
+	DUCKDB_API BatchedDataCollection(ClientContext &context, vector<LogicalType> types, bool buffer_managed = false);
 
 	//! Appends a datachunk with the given batch index to the batched collection
 	DUCKDB_API void Append(DataChunk &input, idx_t batch_index);
@@ -51,7 +51,9 @@ private:
 		ColumnDataAppendState append_state;
 	};
 
+	ClientContext &context;
 	vector<LogicalType> types;
+	bool buffer_managed;
 	//! The data of the batched chunk collection - a set of batch_index -> ColumnDataCollection pointers
 	map<idx_t, unique_ptr<ColumnDataCollection>> data;
 	//! The last batch collection that was inserted into

package/src/duckdb/src/include/duckdb/common/types/bit.hpp

@@ -8,8 +8,12 @@
 
 #pragma once
 
+#include "duckdb/common/assert.hpp"
 #include "duckdb/common/common.hpp"
+#include "duckdb/common/hugeint.hpp"
+#include "duckdb/common/limits.hpp"
 #include "duckdb/common/types.hpp"
+#include "duckdb/common/types/string_type.hpp"
 
 namespace duckdb {
 
@@ -37,7 +41,33 @@ public:
 	//! Convert a string to a bit. This function should ONLY be called after calling GetBitSize, since it does NOT
 	//! perform data validation.
 	DUCKDB_API static void ToBit(string_t str, string_t &output);
+
 	DUCKDB_API static string ToBit(string_t str);
+
+	//! output needs to have enough space allocated before calling this function (blob size + 1)
+	DUCKDB_API static void BlobToBit(string_t blob, string_t &output);
+
+	DUCKDB_API static string BlobToBit(string_t blob);
+
+	//! output_str needs to have enough space allocated before calling this function (sizeof(T) + 1)
+	template <class T>
+	static void NumericToBit(T numeric, string_t &output_str);
+
+	template <class T>
+	static string NumericToBit(T numeric);
+
+	//! bit is expected to fit inside of output num (bit size <= sizeof(T) + 1)
+	template <class T>
+	static void BitToNumeric(string_t bit, T &output_num);
+
+	template <class T>
+	static T BitToNumeric(string_t bit);
+
+	//! bit is expected to fit inside of output_blob (bit size = output_blob + 1)
+	static void BitToBlob(string_t bit, string_t &output_blob);
+
+	static string BitToBlob(string_t bit);
+
 	//! Creates a new bitstring of determined length
 	DUCKDB_API static void BitString(const string_t &input, const idx_t &len, string_t &result);
 	DUCKDB_API static void SetEmptyBitString(string_t &target, string_t &input);
@@ -58,5 +88,56 @@ private:
 	static idx_t GetBitInternal(string_t bit_string, idx_t n);
 	static void SetBitInternal(string_t &bit_string, idx_t n, idx_t new_value);
 	static idx_t GetBitIndex(idx_t n);
+	static uint8_t GetFirstByte(const string_t &str);
 };
+
+//===--------------------------------------------------------------------===//
+// Bit Template definitions
+//===--------------------------------------------------------------------===//
+template <class T>
+void Bit::NumericToBit(T numeric, string_t &output_str) {
+	D_ASSERT(output_str.GetSize() >= sizeof(T) + 1);
+
+	auto output = output_str.GetDataWriteable();
+	auto data = const_data_ptr_cast(&numeric);
+
+	*output = 0; // set padding to 0
+	++output;
+	for (idx_t idx = 0; idx < sizeof(T); ++idx) {
+		output[idx] = data[sizeof(T) - idx - 1];
+	}
+	Bit::Finalize(output_str);
+}
+
+template <class T>
+string Bit::NumericToBit(T numeric) {
+	auto bit_len = sizeof(T) + 1;
+	auto buffer = make_unsafe_uniq_array<char>(bit_len);
+	string_t output_str(buffer.get(), bit_len);
+	Bit::NumericToBit(numeric, output_str);
+	return output_str.GetString();
+}
+
+template <class T>
+T Bit::BitToNumeric(string_t bit) {
+	T output;
+	Bit::BitToNumeric(bit, output);
+	return (output);
+}
+
+template <class T>
+void Bit::BitToNumeric(string_t bit, T &output_num) {
+	D_ASSERT(bit.GetSize() <= sizeof(T) + 1);
+
+	output_num = 0;
+	auto data = const_data_ptr_cast(bit.GetData());
+	auto output = data_ptr_cast(&output_num);
+
+	idx_t padded_byte_idx = sizeof(T) - bit.GetSize() + 1;
+	output[sizeof(T) - 1 - padded_byte_idx] = GetFirstByte(bit);
+	for (idx_t idx = padded_byte_idx + 1; idx < sizeof(T); ++idx) {
+		output[sizeof(T) - 1 - idx] = data[1 + idx - padded_byte_idx];
+	}
+}
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp

@@ -43,9 +43,19 @@ public:
 	void MakeShared() {
 		shared = true;
 	}
+	bool IsShared() const {
+		return shared;
+	}
 	idx_t BlockCount() const {
 		return blocks.size();
 	}
+	idx_t SizeInBytes() const {
+		idx_t total_size = 0;
+		for (const auto &block : blocks) {
+			total_size += block.size;
+		}
+		return total_size;
+	}
 
 public:
 	void AllocateData(idx_t size, uint32_t &block_id, uint32_t &offset, ChunkManagementState *chunk_state);
@@ -53,7 +63,7 @@ public:
 	void Initialize(ColumnDataAllocator &other);
 	void InitializeChunkState(ChunkManagementState &state, ChunkMetaData &meta_data);
 	data_ptr_t GetDataPointer(ChunkManagementState &state, uint32_t block_id, uint32_t offset);
-	void UnswizzlePointers(ChunkManagementState &state, Vector &result, uint16_t v_offset, uint16_t count,
+	void UnswizzlePointers(ChunkManagementState &state, Vector &result, idx_t v_offset, uint16_t count,
 	                       uint32_t block_id, uint32_t offset);
 
 	//! Deletes the block with the given id

package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp

@@ -61,6 +61,9 @@ public:
 		return types.size();
 	}
 
+	//! The size (in bytes) of this ColumnDataCollection
+	idx_t SizeInBytes() const;
+
 	//! Get the allocator
 	DUCKDB_API Allocator &GetAllocator() const;
 
@@ -143,9 +146,17 @@ public:
 	//! Initialize the column data collection
 	void Initialize(vector<LogicalType> types);
 
-	//! Get a vector of references to every chunk (segment, index in segment), and optionally sort by block id
+	//! Get references to the string heaps in this ColumnDataCollection
+	vector<shared_ptr<StringHeap>> GetHeapReferences();
+	//! Get the allocator type of this ColumnDataCollection
+	ColumnDataAllocatorType GetAllocatorType() const;
+
+	//! Get a vector of the segments in this ColumnDataCollection
 	const vector<unique_ptr<ColumnDataCollectionSegment>> &GetSegments() const;
 
+	void FormatSerialize(FormatSerializer &serializer) const;
+	static unique_ptr<ColumnDataCollection> FormatDeserialize(FormatDeserializer &deserializer);
+
 private:
 	//! Creates a new segment within the ColumnDataCollection
 	void CreateSegment();

package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp

@@ -94,7 +94,7 @@ public:
 	//! The set of child indices
 	vector<VectorDataIndex> child_indices;
 	//! The string heap for the column data collection (only used for IN_MEMORY_ALLOCATOR)
-	StringHeap heap;
+	shared_ptr<StringHeap> heap;
 
 public:
 	void AllocateNewChunk();
@@ -126,6 +126,8 @@ public:
 	}
 
 	idx_t ChunkCount() const;
+	idx_t SizeInBytes() const;
+
 	void FetchChunk(idx_t chunk_idx, DataChunk &result);
 	void FetchChunk(idx_t chunk_idx, DataChunk &result, const vector<column_t> &column_ids);
 

package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp

@@ -20,7 +20,9 @@ enum class ColumnDataAllocatorType : uint8_t {
 	BUFFER_MANAGER_ALLOCATOR,
 	//! Use an in-memory allocator, allocating data for every chunk
 	//! This causes the column data collection to allocate blocks that are not tied to a buffer manager
-	IN_MEMORY_ALLOCATOR
+	IN_MEMORY_ALLOCATOR,
+	//! Use a buffer manager to allocate vectors, but use a StringHeap for strings
+	HYBRID
 };
 
 enum class ColumnDataScanProperties : uint8_t {

package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp

@@ -14,13 +14,13 @@
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/common/winapi.hpp"
 
-struct ArrowArray;
-
 namespace duckdb {
 class Allocator;
 class ClientContext;
 class ExecutionContext;
 class VectorCache;
+class FormatSerializer;
+class FormatDeserializer;
 
 //!  A Data Chunk represents a set of vectors.
 /*!
@@ -143,6 +143,9 @@ public:
 	//! Deserializes a blob back into a DataChunk
 	DUCKDB_API void Deserialize(Deserializer &source);
 
+	DUCKDB_API void FormatSerialize(FormatSerializer &serializer) const;
+	DUCKDB_API void FormatDeserialize(FormatDeserializer &source);
+
 	//! Hashes the DataChunk to the target vector
 	DUCKDB_API void Hash(Vector &result);
 	//! Hashes specific vectors of the DataChunk to the target vector
@@ -153,7 +156,7 @@ public:
 
 	//! Converts this DataChunk to a printable string representation
 	DUCKDB_API string ToString() const;
-	DUCKDB_API void Print();
+	DUCKDB_API void Print() const;
 
 	DataChunk(const DataChunk &) = delete;
 

package/src/duckdb/src/include/duckdb/common/types/date.hpp

@@ -72,13 +72,13 @@ struct date_t { // NOLINT
 	};
 
 	// special values
-	static inline date_t infinity() {
+	static inline date_t infinity() { // NOLINT
 		return date_t(NumericLimits<int32_t>::Maximum());
-	} // NOLINT
-	static inline date_t ninfinity() {
+	}                                  // NOLINT
+	static inline date_t ninfinity() { // NOLINT
 		return date_t(-NumericLimits<int32_t>::Maximum());
-	} // NOLINT
-	static inline date_t epoch() {
+	}                              // NOLINT
+	static inline date_t epoch() { // NOLINT
 		return date_t(0);
 	} // NOLINT
 };
@@ -158,6 +158,8 @@ public:
 	DUCKDB_API static int64_t EpochNanoseconds(date_t date);
 	//! Extract the epoch from the date (microseconds since 1970-01-01)
 	DUCKDB_API static int64_t EpochMicroseconds(date_t date);
+	//! Extract the epoch from the date (milliseconds since 1970-01-01)
+	DUCKDB_API static int64_t EpochMilliseconds(date_t date);
 	//! Convert the epoch (seconds since 1970-01-01) to a date_t
 	DUCKDB_API static date_t EpochToDate(int64_t epoch);
 
@@ -179,6 +181,8 @@ public:
 	DUCKDB_API static int32_t ExtractISODayOfTheWeek(date_t date);
 	//! Extract the day of the year
 	DUCKDB_API static int32_t ExtractDayOfTheYear(date_t date);
+	//! Extract the day of the year
+	DUCKDB_API static int64_t ExtractJulianDay(date_t date);
 	//! Extract the ISO week number
 	//! ISO weeks start on Monday and the first week of a year
 	//! contains January 4 of that year.

package/src/duckdb/src/include/duckdb/common/types/datetime.hpp

@@ -81,12 +81,54 @@ struct dtime_t { // NOLINT
 	};
 
 	// special values
-	static inline dtime_t allballs() {
+	static inline dtime_t allballs() { // NOLINT
 		return dtime_t(0);
 	} // NOLINT
 };
 
-struct dtime_tz_t : public dtime_t {};
+struct dtime_tz_t { // NOLINT
+	static constexpr const int TIME_BITS = 40;
+	static constexpr const int OFFSET_BITS = 24;
+	static constexpr const uint64_t OFFSET_MASK = ~uint64_t(0) >> TIME_BITS;
+	static constexpr const int32_t MAX_OFFSET = 1559 * 60 * 60;
+	static constexpr const int32_t MIN_OFFSET = -MAX_OFFSET;
+
+	uint64_t bits;
+
+	dtime_tz_t() = default;
+
+	inline dtime_tz_t(dtime_t t, int32_t offset)
+	    : bits((uint64_t(t.micros) << OFFSET_BITS) | uint64_t(offset + MAX_OFFSET)) {
+	}
+
+	inline dtime_t time() const { // NOLINT
+		return dtime_t(bits >> OFFSET_BITS);
+	}
+
+	inline int32_t offset() const { // NOLINT
+		return int32_t(bits & OFFSET_MASK) - MAX_OFFSET;
+	}
+
+	// comparison operators
+	inline bool operator==(const dtime_tz_t &rhs) const {
+		return bits == rhs.bits;
+	};
+	inline bool operator!=(const dtime_tz_t &rhs) const {
+		return bits != rhs.bits;
+	};
+	inline bool operator<=(const dtime_tz_t &rhs) const {
+		return bits <= rhs.bits;
+	};
+	inline bool operator<(const dtime_tz_t &rhs) const {
+		return bits < rhs.bits;
+	};
+	inline bool operator>(const dtime_tz_t &rhs) const {
+		return bits > rhs.bits;
+	};
+	inline bool operator>=(const dtime_tz_t &rhs) const {
+		return bits >= rhs.bits;
+	};
+};
 
 } // namespace duckdb
 
@@ -100,11 +142,12 @@ struct hash<duckdb::dtime_t> {
 		return hash<int64_t>()((int64_t)k);
 	}
 };
+
 template <>
 struct hash<duckdb::dtime_tz_t> {
 	std::size_t operator()(const duckdb::dtime_tz_t &k) const {
 		using std::hash;
-		return hash<int64_t>()((int64_t)k);
+		return hash<int64_t>()(k.bits);
 	}
 };
 } // namespace std

package/src/duckdb/src/include/duckdb/common/types/hyperloglog.hpp

@@ -18,11 +18,14 @@ struct robj;
 
 namespace duckdb {
 
-enum class HLLStorageType { UNCOMPRESSED = 1 };
+enum class HLLStorageType : uint8_t { UNCOMPRESSED = 1 };
 
 class FieldWriter;
 class FieldReader;
 
+class FormatSerializer;
+class FormatDeserializer;
+
 //! The HyperLogLog class holds a HyperLogLog counter for approximate cardinality counting
 class HyperLogLog {
 public:
@@ -50,6 +53,9 @@ public:
 	void Serialize(FieldWriter &writer) const;
 	static unique_ptr<HyperLogLog> Deserialize(FieldReader &reader);
 
+	void FormatSerialize(FormatSerializer &serializer) const;
+	static unique_ptr<HyperLogLog> FormatDeserialize(FormatDeserializer &deserializer);
+
 public:
 	//! Compute HLL hashes over vdata, and store them in 'hashes'
 	//! Then, compute register indices and prefix lengths, and also store them in 'hashes' as a pair of uint32_t

package/src/duckdb/src/include/duckdb/common/types/interval.hpp

@@ -16,6 +16,9 @@ struct dtime_t;
 struct date_t;
 struct timestamp_t;
 
+class FormatSerializer;
+class FormatDeserializer;
+
 struct interval_t {
 	int32_t months;
 	int32_t days;
@@ -24,6 +27,10 @@ struct interval_t {
 	inline bool operator==(const interval_t &rhs) const {
 		return this->days == rhs.days && this->months == rhs.months && this->micros == rhs.micros;
 	}
+
+	// Serialization
+	void FormatSerialize(FormatSerializer &serializer) const;
+	static interval_t FormatDeserialize(FormatDeserializer &source);
 };
 
 //! The Interval class is a static class that holds helper functions for the Interval

package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp

@@ -22,40 +22,36 @@ struct ListSegment {
 	ListSegment *next;
 };
 struct LinkedList {
-	LinkedList() {};
+	LinkedList() : total_capacity(0), first_segment(nullptr), last_segment(nullptr) {};
 	LinkedList(idx_t total_capacity_p, ListSegment *first_segment_p, ListSegment *last_segment_p)
 	    : total_capacity(total_capacity_p), first_segment(first_segment_p), last_segment(last_segment_p) {
 	}
 
-	idx_t total_capacity = 0;
-	ListSegment *first_segment = nullptr;
-	ListSegment *last_segment = nullptr;
+	idx_t total_capacity;
+	ListSegment *first_segment;
+	ListSegment *last_segment;
 };
 
 // forward declarations
 struct ListSegmentFunctions;
-typedef ListSegment *(*create_segment_t)(const ListSegmentFunctions &functions, Allocator &allocator,
+typedef ListSegment *(*create_segment_t)(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
                                          uint16_t capacity);
-typedef void (*write_data_to_segment_t)(const ListSegmentFunctions &functions, Allocator &allocator,
-                                        ListSegment *segment, Vector &input, idx_t &entry_idx, idx_t &count);
+typedef void (*write_data_to_segment_t)(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
+                                        ListSegment *segment, RecursiveUnifiedVectorFormat &input_data,
+                                        idx_t &entry_idx);
 typedef void (*read_data_from_segment_t)(const ListSegmentFunctions &functions, const ListSegment *segment,
                                          Vector &result, idx_t &total_count);
-typedef ListSegment *(*copy_data_from_segment_t)(const ListSegmentFunctions &functions, const ListSegment *source,
-                                                 Allocator &allocator);
-typedef void (*destroy_segment_t)(const ListSegmentFunctions &functions, ListSegment *segment, Allocator &allocator);
 
 struct ListSegmentFunctions {
 	create_segment_t create_segment;
 	write_data_to_segment_t write_data;
 	read_data_from_segment_t read_data;
-	copy_data_from_segment_t copy_data;
-	destroy_segment_t destroy;
+
 	vector<ListSegmentFunctions> child_functions;
 
-	void AppendRow(Allocator &allocator, LinkedList &linked_list, Vector &input, idx_t &entry_idx, idx_t &count) const;
+	void AppendRow(ArenaAllocator &allocator, LinkedList &linked_list, RecursiveUnifiedVectorFormat &input_data,
+	               idx_t &entry_idx) const;
 	void BuildListVector(const LinkedList &linked_list, Vector &result, idx_t &initial_total_count) const;
-	void CopyLinkedList(const LinkedList &source_list, LinkedList &target_list, Allocator &allocator) const;
-	void Destroy(Allocator &allocator, LinkedList &linked_list) const;
 };
 
 void GetSegmentDataFunctions(ListSegmentFunctions &functions, const LogicalType &type);

package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp

@@ -8,6 +8,7 @@
 
 #pragma once
 
+#include "duckdb/common/fixed_size_map.hpp"
 #include "duckdb/common/perfect_map_set.hpp"
 #include "duckdb/common/types/row/tuple_data_allocator.hpp"
 #include "duckdb/common/types/row/tuple_data_collection.hpp"
@@ -23,10 +24,11 @@ public:
 public:
 	Vector partition_indices;
 	SelectionVector partition_sel;
+	SelectionVector reverse_partition_sel;
 
-	static constexpr idx_t MAP_THRESHOLD = 32;
+	static constexpr idx_t MAP_THRESHOLD = 256;
 	perfect_map_t<list_entry_t> partition_entries;
-	list_entry_t partition_entries_arr[MAP_THRESHOLD];
+	fixed_size_map_t<list_entry_t> fixed_partition_entries;
 
 	vector<unique_ptr<TupleDataPinState>> partition_pin_states;
 	TupleDataChunkState chunk_state;
@@ -48,34 +50,51 @@ struct PartitionTupleDataAllocators {
 //! partitioning, e.g., radix, hive
 class PartitionedTupleData {
 public:
-	unique_ptr<PartitionedTupleData> CreateShared();
 	virtual ~PartitionedTupleData();
 
 public:
+	//! Get the layout of this PartitionedTupleData
+	const TupleDataLayout &GetLayout() const;
 	//! Get the partitioning type of this PartitionedTupleData
 	PartitionedTupleDataType GetType() const;
 	//! Initializes a local state for parallel partitioning that can be merged into this PartitionedTupleData
 	void InitializeAppendState(PartitionedTupleDataAppendState &state,
 	                           TupleDataPinProperties properties = TupleDataPinProperties::UNPIN_AFTER_DONE) const;
 	//! Appends a DataChunk to this PartitionedTupleData
-	void Append(PartitionedTupleDataAppendState &state, DataChunk &input);
+	void Append(PartitionedTupleDataAppendState &state, DataChunk &input,
+	            const SelectionVector &append_sel = *FlatVector::IncrementalSelectionVector(),
+	            const idx_t append_count = DConstants::INVALID_INDEX);
+	//! Appends a DataChunk to this PartitionedTupleData
+	//! - ToUnifiedFormat has already been called
+	void AppendUnified(PartitionedTupleDataAppendState &state, DataChunk &input,
+	                   const SelectionVector &append_sel = *FlatVector::IncrementalSelectionVector(),
+	                   const idx_t append_count = DConstants::INVALID_INDEX);
 	//! Appends rows to this PartitionedTupleData
-	void Append(PartitionedTupleDataAppendState &state, TupleDataChunkState &input, idx_t count);
+	void Append(PartitionedTupleDataAppendState &state, TupleDataChunkState &input, const idx_t count);
 	//! Flushes any remaining data in the append state into this PartitionedTupleData
 	void FlushAppendState(PartitionedTupleDataAppendState &state);
 	//! Combine another PartitionedTupleData into this PartitionedTupleData
 	void Combine(PartitionedTupleData &other);
-	//! Partition a TupleDataCollection
-	void Partition(TupleDataCollection &source,
-	               TupleDataPinProperties properties = TupleDataPinProperties::UNPIN_AFTER_DONE);
+	//! Resets this PartitionedTupleData
+	void Reset();
 	//! Repartition this PartitionedTupleData into the new PartitionedTupleData
 	void Repartition(PartitionedTupleData &new_partitioned_data);
+	//! Unpins the data
+	void Unpin();
 	//! Get the partitions in this PartitionedTupleData
 	vector<unique_ptr<TupleDataCollection>> &GetPartitions();
+	//! Get the data of this PartitionedTupleData as a single unpartitioned TupleDataCollection
+	unique_ptr<TupleDataCollection> GetUnpartitioned();
 	//! Get the count of this PartitionedTupleData
 	idx_t Count() const;
 	//! Get the size (in bytes) of this PartitionedTupleData
 	idx_t SizeInBytes() const;
+	//! Get the number of partitions of this PartitionedTupleData
+	idx_t PartitionCount() const;
+	//! Converts this PartitionedTupleData to a string representation
+	string ToString();
+	//! Prints the string representation of this PartitionedTupleData
+	void Print();
 
 protected:
 	//===--------------------------------------------------------------------===//
@@ -92,7 +111,7 @@ protected:
 		throw NotImplementedException("ComputePartitionIndices for this type of PartitionedTupleData");
 	}
 	//! Compute partition indices from rows (similar to function above)
-	virtual void ComputePartitionIndices(Vector &row_locations, idx_t count, Vector &partition_indices) const {
+	virtual void ComputePartitionIndices(Vector &row_locations, idx_t append_count, Vector &partition_indices) const {
 		throw NotImplementedException("ComputePartitionIndices for this type of PartitionedTupleData");
 	}
 	//! Maximum partition index (optional)
@@ -117,20 +136,36 @@ protected:
 
 	//! Create a new shared allocator
 	void CreateAllocator();
+	//! Whether to use fixed size map or regular marp
+	bool UseFixedSizeMap() const;
 	//! Builds a selection vector in the Append state for the partitions
 	//! - returns true if everything belongs to the same partition - stores partition index in single_partition_idx
-	void BuildPartitionSel(PartitionedTupleDataAppendState &state, idx_t count);
+	void BuildPartitionSel(PartitionedTupleDataAppendState &state, const SelectionVector &append_sel,
+	                       const idx_t append_count);
+	template <class MAP_TYPE, class GETTER>
+	void BuildPartitionSel(PartitionedTupleDataAppendState &state, MAP_TYPE &partition_entries,
+	                       const SelectionVector &append_sel, const idx_t append_count);
 	//! Builds out the buffer space in the partitions
 	void BuildBufferSpace(PartitionedTupleDataAppendState &state);
+	template <class MAP_TYPE, class GETTER>
+	void BuildBufferSpace(PartitionedTupleDataAppendState &state, const MAP_TYPE &partition_entries);
 	//! Create a collection for a specific a partition
 	unique_ptr<TupleDataCollection> CreatePartitionCollection(idx_t partition_index) const {
-		return make_uniq<TupleDataCollection>(allocators->allocators[partition_index]);
+		if (allocators) {
+			return make_uniq<TupleDataCollection>(allocators->allocators[partition_index]);
+		} else {
+			return make_uniq<TupleDataCollection>(buffer_manager, layout);
+		}
 	}
+	//! Verify count/data size of this PartitionedTupleData
+	void Verify() const;
 
 protected:
 	PartitionedTupleDataType type;
 	BufferManager &buffer_manager;
 	const TupleDataLayout layout;
+	idx_t count;
+	idx_t data_size;
 
 	mutex lock;
 	shared_ptr<PartitionTupleDataAllocators> allocators;

package/src/duckdb/src/include/duckdb/common/types/row/row_data_collection_scanner.hpp

@@ -50,6 +50,10 @@ public:
 	RowDataCollectionScanner(RowDataCollection &rows, RowDataCollection &heap, const RowLayout &layout, bool external,
 	                         bool flush = true);
 
+	//	Single block scan
+	RowDataCollectionScanner(RowDataCollection &rows, RowDataCollection &heap, const RowLayout &layout, bool external,
+	                         idx_t block_idx, bool flush);
+
 	//! The type layout of the payload
 	inline const vector<LogicalType> &GetTypes() const {
 		return layout.GetTypes();
@@ -70,6 +74,11 @@ public:
 		return total_count - total_scanned;
 	}
 
+	//! The number of remaining rows
+	inline idx_t BlockIndex() const {
+		return read_state.block_idx;
+	}
+
 	//! Swizzle the blocks for external scanning
 	//! Swizzling is all or nothing, so if we have scanned previously,
 	//! we need to re-swizzle.
@@ -93,7 +102,7 @@ private:
 	//! Read state
 	ScanState read_state;
 	//! The total count of sorted_data
-	const idx_t total_count;
+	idx_t total_count;
 	//! The number of rows scanned so far
 	idx_t total_scanned;
 	//! Addresses used to gather from the sorted data