duckdb 0.8.2-dev33.0 → 0.8.2-dev3300.0

package/src/duckdb/src/common/sort/partition_state.cpp

@@ -87,71 +87,76 @@ PartitionGlobalSinkState::PartitionGlobalSinkState(ClientContext &context,
                                                    const vector<unique_ptr<BaseStatistics>> &partition_stats,
                                                    idx_t estimated_cardinality)
     : context(context), buffer_manager(BufferManager::GetBufferManager(context)), allocator(Allocator::Get(context)),
-      payload_types(payload_types), memory_per_thread(0), count(0) {
+      fixed_bits(0), payload_types(payload_types), memory_per_thread(0), max_bits(1), count(0) {
 
 	GenerateOrderings(partitions, orders, partition_bys, order_bys, partition_stats);
 
 	memory_per_thread = PhysicalOperator::GetMaxThreadMemory(context);
 	external = ClientConfig::GetConfig(context).force_external;
 
+	const auto thread_pages = PreviousPowerOfTwo(memory_per_thread / (4 * idx_t(Storage::BLOCK_ALLOC_SIZE)));
+	while (max_bits < 10 && (thread_pages >> max_bits) > 1) {
+		++max_bits;
+	}
+
 	if (!orders.empty()) {
-		grouping_types = payload_types;
-		grouping_types.push_back(LogicalType::HASH);
+		auto types = payload_types;
+		types.push_back(LogicalType::HASH);
+		grouping_types.Initialize(types);
 
 		ResizeGroupingData(estimated_cardinality);
 	}
 }
 
+void PartitionGlobalSinkState::SyncPartitioning(const PartitionGlobalSinkState &other) {
+	fixed_bits = other.grouping_data ? other.grouping_data->GetRadixBits() : 0;
+
+	const auto old_bits = grouping_data ? grouping_data->GetRadixBits() : 0;
+	if (fixed_bits != old_bits) {
+		const auto hash_col_idx = payload_types.size();
+		grouping_data = make_uniq<RadixPartitionedTupleData>(buffer_manager, grouping_types, fixed_bits, hash_col_idx);
+	}
+}
+
+unique_ptr<RadixPartitionedTupleData> PartitionGlobalSinkState::CreatePartition(idx_t new_bits) const {
+	const auto hash_col_idx = payload_types.size();
+	return make_uniq<RadixPartitionedTupleData>(buffer_manager, grouping_types, new_bits, hash_col_idx);
+}
+
 void PartitionGlobalSinkState::ResizeGroupingData(idx_t cardinality) {
 	//	Have we started to combine? Then just live with it.
-	if (grouping_data && !grouping_data->GetPartitions().empty()) {
+	if (fixed_bits || (grouping_data && !grouping_data->GetPartitions().empty())) {
 		return;
 	}
 	//	Is the average partition size too large?
 	const idx_t partition_size = STANDARD_ROW_GROUPS_SIZE;
 	const auto bits = grouping_data ? grouping_data->GetRadixBits() : 0;
 	auto new_bits = bits ? bits : 4;
-	while (new_bits < 10 && (cardinality / RadixPartitioning::NumberOfPartitions(new_bits)) > partition_size) {
+	while (new_bits < max_bits && (cardinality / RadixPartitioning::NumberOfPartitions(new_bits)) > partition_size) {
 		++new_bits;
 	}
 
 	// Repartition the grouping data
 	if (new_bits != bits) {
-		const auto hash_col_idx = payload_types.size();
-		grouping_data = make_uniq<RadixPartitionedColumnData>(context, grouping_types, new_bits, hash_col_idx);
+		grouping_data = CreatePartition(new_bits);
 	}
 }
 
 void PartitionGlobalSinkState::SyncLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append) {
 	// We are done if the local_partition is right sized.
-	auto &local_radix = local_partition->Cast<RadixPartitionedColumnData>();
-	if (local_radix.GetRadixBits() == grouping_data->GetRadixBits()) {
+	auto &local_radix = local_partition->Cast<RadixPartitionedTupleData>();
+	const auto new_bits = grouping_data->GetRadixBits();
+	if (local_radix.GetRadixBits() == new_bits) {
 		return;
 	}
 
 	// If the local partition is now too small, flush it and reallocate
-	auto new_partition = grouping_data->CreateShared();
-	auto new_append = make_uniq<PartitionedColumnDataAppendState>();
-	new_partition->InitializeAppendState(*new_append);
-
+	auto new_partition = CreatePartition(new_bits);
 	local_partition->FlushAppendState(*local_append);
-	auto &local_groups = local_partition->GetPartitions();
-	for (auto &local_group : local_groups) {
-		ColumnDataScanState scanner;
-		local_group->InitializeScan(scanner);
-
-		DataChunk scan_chunk;
-		local_group->InitializeScanChunk(scan_chunk);
-		for (scan_chunk.Reset(); local_group->Scan(scanner, scan_chunk); scan_chunk.Reset()) {
-			new_partition->Append(*new_append, scan_chunk);
-		}
-	}
-
-	// The append state has stale pointers to the old local partition, so nuke it from orbit.
-	new_partition->FlushAppendState(*new_append);
+	local_partition->Repartition(*new_partition);
 
 	local_partition = std::move(new_partition);
-	local_append = make_uniq<PartitionedColumnDataAppendState>();
+	local_append = make_uniq<PartitionedTupleDataAppendState>();
 	local_partition->InitializeAppendState(*local_append);
 }
 
@@ -160,8 +165,8 @@ void PartitionGlobalSinkState::UpdateLocalPartition(GroupingPartition &local_par
 	lock_guard<mutex> guard(lock);
 
 	if (!local_partition) {
-		local_partition = grouping_data->CreateShared();
-		local_append = make_uniq<PartitionedColumnDataAppendState>();
+		local_partition = CreatePartition(grouping_data->GetRadixBits());
+		local_append = make_uniq<PartitionedTupleDataAppendState>();
 		local_partition->InitializeAppendState(*local_append);
 		return;
 	}
@@ -186,9 +191,7 @@ void PartitionGlobalSinkState::CombineLocalPartition(GroupingPartition &local_pa
 	grouping_data->Combine(*local_partition);
 }
 
-void PartitionGlobalSinkState::BuildSortState(ColumnDataCollection &group_data, PartitionGlobalHashGroup &hash_group) {
-	auto &global_sort = *hash_group.global_sort;
-
+void PartitionGlobalSinkState::BuildSortState(TupleDataCollection &group_data, GlobalSortState &global_sort) const {
 	//	 Set up the sort expression computation.
 	vector<LogicalType> sort_types;
 	ExpressionExecutor executor(context);
@@ -213,16 +216,9 @@ void PartitionGlobalSinkState::BuildSortState(ColumnDataCollection &group_data,
 	for (column_t i = 0; i < payload_types.size(); ++i) {
 		column_ids.emplace_back(i);
 	}
-	ColumnDataConsumer scanner(group_data, column_ids);
-	ColumnDataConsumerScanState chunk_state;
-	chunk_state.current_chunk_state.properties = ColumnDataScanProperties::ALLOW_ZERO_COPY;
-	scanner.InitializeScan();
-	for (auto chunk_idx = scanner.ChunkCount(); chunk_idx-- > 0;) {
-		if (!scanner.AssignChunk(chunk_state)) {
-			break;
-		}
-		scanner.ScanChunk(chunk_state, payload_chunk);
-
+	TupleDataScanState chunk_state;
+	group_data.InitializeScan(chunk_state, column_ids);
+	while (group_data.Scan(chunk_state, payload_chunk)) {
 		sort_chunk.Reset();
 		executor.Execute(payload_chunk, sort_chunk);
 
@@ -230,10 +226,13 @@ void PartitionGlobalSinkState::BuildSortState(ColumnDataCollection &group_data,
 		if (local_sort.SizeInBytes() > memory_per_thread) {
 			local_sort.Sort(global_sort, true);
 		}
-		scanner.FinishChunk(chunk_state);
 	}
 
 	global_sort.AddLocalState(local_sort);
+}
+
+void PartitionGlobalSinkState::BuildSortState(TupleDataCollection &group_data, PartitionGlobalHashGroup &hash_group) {
+	BuildSortState(group_data, *hash_group.global_sort);
 
 	hash_group.count += group_data.Count();
 }
@@ -482,18 +481,29 @@ public:
 	TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override;
 
 private:
+	struct ExecutorCallback : public PartitionGlobalMergeStates::Callback {
+		explicit ExecutorCallback(Executor &executor) : executor(executor) {
+		}
+
+		bool HasError() const override {
+			return executor.HasError();
+		}
+
+		Executor &executor;
+	};
+
 	shared_ptr<Event> event;
 	PartitionLocalMergeState local_state;
 	PartitionGlobalMergeStates &hash_groups;
 };
 
-TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) {
+bool PartitionGlobalMergeStates::ExecuteTask(PartitionLocalMergeState &local_state, Callback &callback) {
 	// Loop until all hash groups are done
 	size_t sorted = 0;
-	while (sorted < hash_groups.states.size()) {
+	while (sorted < states.size()) {
 		// First check if there is an unfinished task for this thread
-		if (executor.HasError()) {
-			return TaskExecutionResult::TASK_ERROR;
+		if (callback.HasError()) {
+			return false;
 		}
 		if (!local_state.TaskFinished()) {
 			local_state.ExecuteTask();
@@ -501,8 +511,8 @@ TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) {
 		}
 
 		// Thread is done with its assigned task, try to fetch new work
-		for (auto group = sorted; group < hash_groups.states.size(); ++group) {
-			auto &global_state = hash_groups.states[group];
+		for (auto group = sorted; group < states.size(); ++group) {
+			auto &global_state = states[group];
 			if (global_state->IsSorted()) {
 				// This hash group is done
 				// Update the high water mark of densely completed groups
@@ -543,6 +553,16 @@ TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) {
 		}
 	}
 
+	return true;
+}
+
+TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) {
+	ExecutorCallback callback(executor);
+
+	if (!hash_groups.ExecuteTask(local_state, callback)) {
+		return TaskExecutionResult::TASK_ERROR;
+	}
+
 	event->FinishTask();
 	return TaskExecutionResult::TASK_FINISHED;
 }

package/src/duckdb/src/common/sort/sort_state.cpp

@@ -315,7 +315,7 @@ void LocalSortState::ReOrder(SortedData &sd, data_ptr_t sorting_ptr, RowDataColl
 		sd.data_blocks.back()->block->SetSwizzling(nullptr);
 		// Create a single heap block to store the ordered heap
 		idx_t total_byte_offset =
-		    std::accumulate(heap.blocks.begin(), heap.blocks.end(), 0,
+		    std::accumulate(heap.blocks.begin(), heap.blocks.end(), (idx_t)0,
 		                    [](idx_t a, const unique_ptr<RowDataBlock> &b) { return a + b->byte_offset; });
 		idx_t heap_block_size = MaxValue(total_byte_offset, (idx_t)Storage::BLOCK_SIZE);
 		auto ordered_heap_block = make_uniq<RowDataBlock>(*buffer_manager, heap_block_size, 1);

package/src/duckdb/src/common/sort/sorted_block.cpp

@@ -85,7 +85,7 @@ SortedBlock::SortedBlock(BufferManager &buffer_manager, GlobalSortState &state)
 }
 
 idx_t SortedBlock::Count() const {
-	idx_t count = std::accumulate(radix_sorting_data.begin(), radix_sorting_data.end(), 0,
+	idx_t count = std::accumulate(radix_sorting_data.begin(), radix_sorting_data.end(), (idx_t)0,
 	                              [](idx_t a, const unique_ptr<RowDataBlock> &b) { return a + b->count; });
 	if (!sort_layout.all_constant) {
 		D_ASSERT(count == blob_sorting_data->Count());

package/src/duckdb/src/common/types/batched_data_collection.cpp

@@ -1,11 +1,14 @@
 #include "duckdb/common/types/batched_data_collection.hpp"
+
+#include "duckdb/common/optional_ptr.hpp"
 #include "duckdb/common/printer.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/common/optional_ptr.hpp"
 
 namespace duckdb {
 
-BatchedDataCollection::BatchedDataCollection(vector<LogicalType> types_p) : types(std::move(types_p)) {
+BatchedDataCollection::BatchedDataCollection(ClientContext &context_p, vector<LogicalType> types_p,
+                                             bool buffer_managed_p)
+    : context(context_p), types(std::move(types_p)), buffer_managed(buffer_managed_p) {
 }
 
 void BatchedDataCollection::Append(DataChunk &input, idx_t batch_index) {
@@ -20,6 +23,8 @@ void BatchedDataCollection::Append(DataChunk &input, idx_t batch_index) {
 		unique_ptr<ColumnDataCollection> new_collection;
 		if (last_collection.collection) {
 			new_collection = make_uniq<ColumnDataCollection>(*last_collection.collection);
+		} else if (buffer_managed) {
+			new_collection = make_uniq<ColumnDataCollection>(BufferManager::GetBufferManager(context), types);
 		} else {
 			new_collection = make_uniq<ColumnDataCollection>(Allocator::DefaultAllocator(), types);
 		}

package/src/duckdb/src/common/types/bit.cpp

@@ -1,4 +1,6 @@
+#include "duckdb/common/assert.hpp"
 #include "duckdb/common/operator/cast_operators.hpp"
+#include "duckdb/common/typedefs.hpp"
 #include "duckdb/common/types/bit.hpp"
 #include "duckdb/common/types/string_type.hpp"
 
@@ -34,6 +36,13 @@ static inline idx_t GetBitSize(const string_t &str) {
 	return str_len;
 }
 
+uint8_t Bit::GetFirstByte(const string_t &str) {
+	D_ASSERT(str.GetSize() > 1);
+
+	auto data = const_data_ptr_cast(str.GetData());
+	return data[1] & ((1 << (8 - data[0])) - 1);
+}
+
 void Bit::Finalize(string_t &str) {
 	// bit strings require all padding bits to be set to 1
 	// this method sets all padding bits to 1
@@ -146,6 +155,48 @@ string Bit::ToBit(string_t str) {
 	return output_str.GetString();
 }
 
+void Bit::BlobToBit(string_t blob, string_t &output_str) {
+	auto data = const_data_ptr_cast(blob.GetData());
+	auto output = output_str.GetDataWriteable();
+	idx_t size = blob.GetSize();
+
+	*output = 0; // No padding
+	memcpy(output + 1, data, size);
+}
+
+string Bit::BlobToBit(string_t blob) {
+	auto buffer = make_unsafe_uniq_array<char>(blob.GetSize() + 1);
+	string_t output_str(buffer.get(), blob.GetSize() + 1);
+	Bit::BlobToBit(blob, output_str);
+	return output_str.GetString();
+}
+
+void Bit::BitToBlob(string_t bit, string_t &output_blob) {
+	D_ASSERT(bit.GetSize() == output_blob.GetSize() + 1);
+
+	auto data = const_data_ptr_cast(bit.GetData());
+	auto output = output_blob.GetDataWriteable();
+	idx_t size = output_blob.GetSize();
+
+	output[0] = GetFirstByte(bit);
+	if (size > 2) {
+		++output;
+		// First byte in bitstring contains amount of padded bits,
+		// second byte in bitstring is the padded byte,
+		// therefore the rest of the data starts at data + 2 (third byte)
+		memcpy(output, data + 2, size - 1);
+	}
+}
+
+string Bit::BitToBlob(string_t bit) {
+	D_ASSERT(bit.GetSize() > 1);
+
+	auto buffer = make_unsafe_uniq_array<char>(bit.GetSize() - 1);
+	string_t output_str(buffer.get(), bit.GetSize() - 1);
+	Bit::BitToBlob(bit, output_str);
+	return output_str.GetString();
+}
+
 // **** scalar functions ****
 void Bit::BitString(const string_t &input, const idx_t &bit_length, string_t &result) {
 	char *res_buf = result.GetDataWriteable();

package/src/duckdb/src/common/types/column/column_data_allocator.cpp

@@ -1,8 +1,8 @@
 #include "duckdb/common/types/column/column_data_allocator.hpp"
 
 #include "duckdb/common/types/column/column_data_collection_segment.hpp"
-#include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/storage/buffer/block_handle.hpp"
+#include "duckdb/storage/buffer_manager.hpp"
 
 namespace duckdb {
 
@@ -19,6 +19,7 @@ ColumnDataAllocator::ColumnDataAllocator(ClientContext &context, ColumnDataAlloc
     : type(allocator_type) {
 	switch (type) {
 	case ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR:
+	case ColumnDataAllocatorType::HYBRID:
 		alloc.buffer_manager = &BufferManager::GetBufferManager(context);
 		break;
 	case ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR:
@@ -33,6 +34,7 @@ ColumnDataAllocator::ColumnDataAllocator(ColumnDataAllocator &other) {
 	type = other.GetType();
 	switch (type) {
 	case ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR:
+	case ColumnDataAllocatorType::HYBRID:
 		alloc.allocator = other.alloc.allocator;
 		break;
 	case ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR:
@@ -44,7 +46,7 @@ ColumnDataAllocator::ColumnDataAllocator(ColumnDataAllocator &other) {
 }
 
 BufferHandle ColumnDataAllocator::Pin(uint32_t block_id) {
-	D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR);
+	D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR || type == ColumnDataAllocatorType::HYBRID);
 	shared_ptr<BlockHandle> handle;
 	if (shared) {
 		// we only need to grab the lock when accessing the vector, because vector access is not thread-safe:
@@ -58,7 +60,7 @@ BufferHandle ColumnDataAllocator::Pin(uint32_t block_id) {
 }
 
 BufferHandle ColumnDataAllocator::AllocateBlock(idx_t size) {
-	D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR);
+	D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR || type == ColumnDataAllocatorType::HYBRID);
 	auto block_size = MaxValue<idx_t>(size, Storage::BLOCK_SIZE);
 	BlockMetaData data;
 	data.size = 0;
@@ -136,6 +138,7 @@ void ColumnDataAllocator::AllocateData(idx_t size, uint32_t &block_id, uint32_t
                                        ChunkManagementState *chunk_state) {
 	switch (type) {
 	case ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR:
+	case ColumnDataAllocatorType::HYBRID:
 		if (shared) {
 			lock_guard<mutex> guard(lock);
 			AllocateBuffer(size, block_id, offset, chunk_state);
@@ -174,8 +177,8 @@ data_ptr_t ColumnDataAllocator::GetDataPointer(ChunkManagementState &state, uint
 	return state.handles[block_id].Ptr() + offset;
 }
 
-void ColumnDataAllocator::UnswizzlePointers(ChunkManagementState &state, Vector &result, uint16_t v_offset,
-                                            uint16_t count, uint32_t block_id, uint32_t offset) {
+void ColumnDataAllocator::UnswizzlePointers(ChunkManagementState &state, Vector &result, idx_t v_offset, uint16_t count,
+                                            uint32_t block_id, uint32_t offset) {
 	D_ASSERT(result.GetType().InternalType() == PhysicalType::VARCHAR);
 	lock_guard<mutex> guard(lock);
 
@@ -225,7 +228,7 @@ Allocator &ColumnDataAllocator::GetAllocator() {
 }
 
 void ColumnDataAllocator::InitializeChunkState(ChunkManagementState &state, ChunkMetaData &chunk) {
-	if (type != ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR) {
+	if (type != ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR && type != ColumnDataAllocatorType::HYBRID) {
 		// nothing to pin
 		return;
 	}

package/src/duckdb/src/common/types/column/column_data_collection.cpp

@@ -6,6 +6,8 @@
 #include "duckdb/common/types/value_map.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
 
 namespace duckdb {
 
@@ -100,6 +102,14 @@ Allocator &ColumnDataCollection::GetAllocator() const {
 	return allocator->GetAllocator();
 }
 
+idx_t ColumnDataCollection::SizeInBytes() const {
+	idx_t total_size = 0;
+	for (const auto &segment : segments) {
+		total_size += segment->SizeInBytes();
+	}
+	return total_size;
+}
+
 //===--------------------------------------------------------------------===//
 // ColumnDataRow
 //===--------------------------------------------------------------------===//
@@ -333,7 +343,7 @@ struct StandardValueCopy : public BaseValueCopy<T> {
 
 struct StringValueCopy : public BaseValueCopy<string_t> {
 	static string_t Operation(ColumnDataMetaData &meta_data, string_t input) {
-		return input.IsInlined() ? input : meta_data.segment.heap.AddBlob(input);
+		return input.IsInlined() ? input : meta_data.segment.heap->AddBlob(input);
 	}
 };
 
@@ -423,7 +433,8 @@ void ColumnDataCopy<string_t>(ColumnDataMetaData &meta_data, const UnifiedVector
                               idx_t offset, idx_t copy_count) {
 
 	const auto &allocator_type = meta_data.segment.allocator->GetType();
-	if (allocator_type == ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR) {
+	if (allocator_type == ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR ||
+	    allocator_type == ColumnDataAllocatorType::HYBRID) {
 		// strings cannot be spilled to disk - use StringHeap
 		TemplatedColumnDataCopy<StringValueCopy>(meta_data, source_data, source, offset, copy_count);
 		return;
@@ -930,6 +941,7 @@ void ColumnDataCollection::Verify() {
 #endif
 }
 
+// LCOV_EXCL_START
 string ColumnDataCollection::ToString() const {
 	DataChunk chunk;
 	InitializeScanChunk(chunk);
@@ -950,6 +962,7 @@ string ColumnDataCollection::ToString() const {
 
 	return result;
 }
+// LCOV_EXCL_STOP
 
 void ColumnDataCollection::Print() const {
 	Printer::Print(ToString());
@@ -1030,8 +1043,61 @@ bool ColumnDataCollection::ResultEquals(const ColumnDataCollection &left, const
 	return true;
 }
 
+vector<shared_ptr<StringHeap>> ColumnDataCollection::GetHeapReferences() {
+	vector<shared_ptr<StringHeap>> result(segments.size(), nullptr);
+	for (idx_t segment_idx = 0; segment_idx < segments.size(); segment_idx++) {
+		result[segment_idx] = segments[segment_idx]->heap;
+	}
+	return result;
+}
+
+ColumnDataAllocatorType ColumnDataCollection::GetAllocatorType() const {
+	return allocator->GetType();
+}
+
 const vector<unique_ptr<ColumnDataCollectionSegment>> &ColumnDataCollection::GetSegments() const {
 	return segments;
 }
 
+void ColumnDataCollection::FormatSerialize(FormatSerializer &serializer) const {
+	vector<vector<Value>> values;
+	values.resize(ColumnCount());
+	for (auto &chunk : Chunks()) {
+		for (idx_t c = 0; c < chunk.ColumnCount(); c++) {
+			for (idx_t r = 0; r < chunk.size(); r++) {
+				values[c].push_back(chunk.GetValue(c, r));
+			}
+		}
+	}
+	serializer.WriteProperty(100, "types", types);
+	serializer.WriteProperty(101, "values", values);
+}
+
+unique_ptr<ColumnDataCollection> ColumnDataCollection::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto types = deserializer.ReadProperty<vector<LogicalType>>(100, "types");
+	auto values = deserializer.ReadProperty<vector<vector<Value>>>(101, "values");
+
+	auto collection = make_uniq<ColumnDataCollection>(Allocator::DefaultAllocator(), types);
+	if (values.empty()) {
+		return collection;
+	}
+	DataChunk chunk;
+	chunk.Initialize(Allocator::DefaultAllocator(), types);
+
+	for (idx_t r = 0; r < values[0].size(); r++) {
+		for (idx_t c = 0; c < types.size(); c++) {
+			chunk.SetValue(c, chunk.size(), values[c][r]);
+		}
+		chunk.SetCardinality(chunk.size() + 1);
+		if (chunk.size() == STANDARD_VECTOR_SIZE) {
+			collection->Append(chunk);
+			chunk.Reset();
+		}
+	}
+	if (chunk.size() > 0) {
+		collection->Append(chunk);
+	}
+	return collection;
+}
+
 } // namespace duckdb

package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp

@@ -6,7 +6,8 @@ namespace duckdb {
 
 ColumnDataCollectionSegment::ColumnDataCollectionSegment(shared_ptr<ColumnDataAllocator> allocator_p,
                                                          vector<LogicalType> types_p)
-    : allocator(std::move(allocator_p)), types(std::move(types_p)), count(0), heap(allocator->GetAllocator()) {
+    : allocator(std::move(allocator_p)), types(std::move(types_p)), count(0),
+      heap(make_shared<StringHeap>(allocator->GetAllocator())) {
 }
 
 idx_t ColumnDataCollectionSegment::GetDataSize(idx_t type_size) {
@@ -26,7 +27,8 @@ VectorDataIndex ColumnDataCollectionSegment::AllocateVectorInternal(const Logica
 	auto type_size = internal_type == PhysicalType::STRUCT ? 0 : GetTypeIdSize(internal_type);
 	allocator->AllocateData(GetDataSize(type_size) + ValidityMask::STANDARD_MASK_SIZE, meta_data.block_id,
 	                        meta_data.offset, chunk_state);
-	if (allocator->GetType() == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR) {
+	if (allocator->GetType() == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR ||
+	    allocator->GetType() == ColumnDataAllocatorType::HYBRID) {
 		chunk_meta.block_ids.insert(meta_data.block_id);
 	}
 
@@ -203,10 +205,17 @@ idx_t ColumnDataCollectionSegment::ReadVector(ChunkManagementState &state, Vecto
 		}
 	} else if (internal_type == PhysicalType::VARCHAR) {
 		if (allocator->GetType() == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR) {
-			for (auto &swizzle_segment : vdata.swizzle_data) {
-				auto &string_heap_segment = GetVectorData(swizzle_segment.child_index);
-				allocator->UnswizzlePointers(state, result, swizzle_segment.offset, swizzle_segment.count,
-				                             string_heap_segment.block_id, string_heap_segment.offset);
+			auto next_index = vector_index;
+			idx_t offset = 0;
+			while (next_index.IsValid()) {
+				auto &current_vdata = GetVectorData(next_index);
+				for (auto &swizzle_segment : current_vdata.swizzle_data) {
+					auto &string_heap_segment = GetVectorData(swizzle_segment.child_index);
+					allocator->UnswizzlePointers(state, result, offset + swizzle_segment.offset, swizzle_segment.count,
+					                             string_heap_segment.block_id, string_heap_segment.offset);
+				}
+				offset += current_vdata.count;
+				next_index = current_vdata.next_data;
 			}
 		}
 		if (state.properties == ColumnDataScanProperties::DISALLOW_ZERO_COPY) {
@@ -234,6 +243,11 @@ idx_t ColumnDataCollectionSegment::ChunkCount() const {
 	return chunk_data.size();
 }
 
+idx_t ColumnDataCollectionSegment::SizeInBytes() const {
+	D_ASSERT(!allocator->IsShared());
+	return allocator->SizeInBytes() + heap->SizeInBytes();
+}
+
 void ColumnDataCollectionSegment::FetchChunk(idx_t chunk_idx, DataChunk &result) {
 	vector<column_t> column_ids;
 	column_ids.reserve(types.size());

package/src/duckdb/src/common/types/column/partitioned_column_data.cpp

@@ -32,13 +32,13 @@ PartitionedColumnData::~PartitionedColumnData() {
 
 void PartitionedColumnData::InitializeAppendState(PartitionedColumnDataAppendState &state) const {
 	state.partition_sel.Initialize();
-	state.slice_chunk.Initialize(context, types);
+	state.slice_chunk.Initialize(BufferAllocator::Get(context), types);
 	InitializeAppendStateInternal(state);
 }
 
 unique_ptr<DataChunk> PartitionedColumnData::CreatePartitionBuffer() const {
 	auto result = make_uniq<DataChunk>();
-	result->Initialize(BufferManager::GetBufferManager(context).GetBufferAllocator(), types, BufferSize());
+	result->Initialize(BufferAllocator::Get(context), types, BufferSize());
 	return result;
 }
 

package/src/duckdb/src/common/types/data_chunk.cpp

@@ -309,7 +309,7 @@ void DataChunk::Hash(Vector &result) {
 
 void DataChunk::Hash(vector<idx_t> &column_ids, Vector &result) {
 	D_ASSERT(result.GetType().id() == LogicalType::HASH);
-	D_ASSERT(column_ids.size() > 0);
+	D_ASSERT(!column_ids.empty());
 
 	VectorOperations::Hash(data[column_ids[0]], result, size());
 	for (idx_t i = 1; i < column_ids.size(); i++) {
@@ -327,7 +327,7 @@ void DataChunk::Verify() {
 #endif
 }
 
-void DataChunk::Print() {
+void DataChunk::Print() const {
 	Printer::Print(ToString());
 }
 

package/src/duckdb/src/common/types/date.cpp

@@ -441,6 +441,15 @@ int64_t Date::EpochMicroseconds(date_t date) {
 	return result;
 }
 
+int64_t Date::EpochMilliseconds(date_t date) {
+	int64_t result;
+	const auto MILLIS_PER_DAY = Interval::MICROS_PER_DAY / Interval::MICROS_PER_MSEC;
+	if (!TryMultiplyOperator::Operation<int64_t, int64_t, int64_t>(date.days, MILLIS_PER_DAY, result)) {
+		throw ConversionException("Could not convert DATE (%s) to milliseconds", Date::ToString(date));
+	}
+	return result;
+}
+
 int32_t Date::ExtractYear(date_t d, int32_t *last_year) {
 	auto n = d.days;
 	// cached look up: check if year of this date is the same as the last one we looked up
@@ -481,6 +490,12 @@ int32_t Date::ExtractDayOfTheYear(date_t date) {
 	return date.days - Date::CUMULATIVE_YEAR_DAYS[year_offset] + 1;
 }
 
+int64_t Date::ExtractJulianDay(date_t date) {
+	// Julian Day 0 is (-4713, 11, 24) in the proleptic Gregorian calendar.
+	static const auto JULIAN_EPOCH = -2440588;
+	return date.days - JULIAN_EPOCH;
+}
+
 int32_t Date::ExtractISODayOfTheWeek(date_t date) {
 	// date of 0 is 1970-01-01, which was a Thursday (4)
 	// -7 = 4