duckdb 0.8.2-dev4203.0 → 0.8.2-dev4376.0

package/src/duckdb/src/execution/index/fixed_size_buffer.cpp

@@ -5,70 +5,288 @@
 
 namespace duckdb {
 
+//===--------------------------------------------------------------------===//
+// PartialBlockForIndex
+//===--------------------------------------------------------------------===//
+
+PartialBlockForIndex::PartialBlockForIndex(PartialBlockState state, BlockManager &block_manager,
+                                           const shared_ptr<BlockHandle> &block_handle)
+    : PartialBlock(state, block_manager, block_handle) {
+}
+
+void PartialBlockForIndex::Flush(const idx_t free_space_left) {
+	FlushInternal(free_space_left);
+	block_handle = block_manager.ConvertToPersistent(state.block_id, std::move(block_handle));
+	Clear();
+}
+
+void PartialBlockForIndex::Merge(PartialBlock &other, idx_t offset, idx_t other_size) {
+	throw InternalException("no merge for PartialBlockForIndex");
+}
+
+void PartialBlockForIndex::Clear() {
+	block_handle.reset();
+}
+
+//===--------------------------------------------------------------------===//
+// FixedSizeBuffer
+//===--------------------------------------------------------------------===//
+
+constexpr idx_t FixedSizeBuffer::BASE[];
+constexpr uint8_t FixedSizeBuffer::SHIFT[];
+
 FixedSizeBuffer::FixedSizeBuffer(BlockManager &block_manager)
-    : block_manager(block_manager), segment_count(0), dirty(false), vacuum(false), block_handle(nullptr) {
+    : block_manager(block_manager), segment_count(0), allocation_size(0), dirty(false), vacuum(false), block_pointer(),
+      block_handle(nullptr) {
 
 	auto &buffer_manager = block_manager.buffer_manager;
 	buffer_handle = buffer_manager.Allocate(Storage::BLOCK_SIZE, false, &block_handle);
 }
 
-FixedSizeBuffer::FixedSizeBuffer(BlockManager &block_manager, const idx_t segment_count, const block_id_t &block_id)
-    : block_manager(block_manager), segment_count(segment_count), dirty(false), vacuum(false) {
+FixedSizeBuffer::FixedSizeBuffer(BlockManager &block_manager, const idx_t segment_count, const idx_t allocation_size,
+                                 const BlockPointer &block_pointer)
+    : block_manager(block_manager), segment_count(segment_count), allocation_size(allocation_size), dirty(false),
+      vacuum(false), block_pointer(block_pointer) {
 
-	D_ASSERT(block_id < MAXIMUM_BLOCK);
-	block_handle = block_manager.RegisterBlock(block_id);
-	D_ASSERT(BlockId() < MAXIMUM_BLOCK);
+	D_ASSERT(block_pointer.IsValid());
+	block_handle = block_manager.RegisterBlock(block_pointer.block_id);
+	D_ASSERT(block_handle->BlockId() < MAXIMUM_BLOCK);
 }
 
 void FixedSizeBuffer::Destroy() {
 	if (InMemory()) {
+		// we can have multiple readers on a pinned block, and unpinning the buffer handle
+		// decrements the reader count on the underlying block handle (Destroy() unpins)
 		buffer_handle.Destroy();
 	}
 	if (OnDisk()) {
-		block_manager.MarkBlockAsFree(BlockId());
+		// marking a block as modified decreases the reference count of multi-use blocks
+		block_manager.MarkBlockAsModified(block_pointer.block_id);
 	}
 }
 
-void FixedSizeBuffer::Serialize() {
+void FixedSizeBuffer::Serialize(PartialBlockManager &partial_block_manager, const idx_t available_segments,
+                                const idx_t segment_size, const idx_t bitmask_offset) {
 
+	// we do not serialize a block that is already on disk and not in memory
 	if (!InMemory()) {
 		if (!OnDisk() || dirty) {
-			throw InternalException("invalid/missing buffer in FixedSizeAllocator");
+			throw InternalException("invalid or missing buffer in FixedSizeAllocator");
 		}
 		return;
 	}
+
+	// we do not serialize a block that is already on disk and not dirty
 	if (!dirty && OnDisk()) {
 		return;
 	}
 
-	// the buffer is in memory
-	D_ASSERT(InMemory());
-	// the buffer never was on disk, or there were changes to it after loading it from disk
-	D_ASSERT(!OnDisk() || dirty);
+	if (dirty) {
+		// the allocation possibly changed
+		auto max_offset = GetMaxOffset(available_segments);
+		allocation_size = max_offset * segment_size + bitmask_offset;
+	}
 
-	// we persist any changes, so the buffer is no longer dirty
-	dirty = false;
+	// the buffer is in memory, so we copied it onto a new buffer when pinning
+	D_ASSERT(InMemory() && !OnDisk());
 
-	if (!OnDisk()) {
-		// temporary block - convert to persistent
-		auto block_id = block_manager.GetFreeBlockId();
-		D_ASSERT(block_id < MAXIMUM_BLOCK);
-		block_handle = block_manager.ConvertToPersistent(block_id, std::move(block_handle));
-		buffer_handle.Destroy();
+	// now we write the changes, first get a partial block allocation
+	PartialBlockAllocation allocation = partial_block_manager.GetBlockAllocation(allocation_size);
+	block_pointer.block_id = allocation.state.block_id;
+	block_pointer.offset = allocation.state.offset;
+
+	auto &buffer_manager = block_manager.buffer_manager;
+
+	if (allocation.partial_block) {
+		// copy to an existing partial block
+		D_ASSERT(block_pointer.offset > 0);
+		auto &p_block_for_index = allocation.partial_block->Cast<PartialBlockForIndex>();
+		auto dst_handle = buffer_manager.Pin(p_block_for_index.block_handle);
+		memcpy(dst_handle.Ptr() + block_pointer.offset, buffer_handle.Ptr(), allocation_size);
+		SetUninitializedRegions(p_block_for_index, segment_size, block_pointer.offset, bitmask_offset);
 
 	} else {
-		// already a persistent block - only need to write it
-		auto block_id = block_handle->BlockId();
-		D_ASSERT(block_id < MAXIMUM_BLOCK);
-		block_manager.Write(buffer_handle.GetFileBuffer(), block_id);
+		// create a new block that can potentially be used as a partial block
+		D_ASSERT(block_handle);
+		D_ASSERT(!block_pointer.offset);
+		auto p_block_for_index = make_uniq<PartialBlockForIndex>(allocation.state, block_manager, block_handle);
+		SetUninitializedRegions(*p_block_for_index, segment_size, block_pointer.offset, bitmask_offset);
+		allocation.partial_block = std::move(p_block_for_index);
 	}
+
+	partial_block_manager.RegisterPartialBlock(std::move(allocation));
+
+	// resetting this buffer
+	buffer_handle.Destroy();
+	block_handle = block_manager.RegisterBlock(block_pointer.block_id);
+	D_ASSERT(block_handle->BlockId() < MAXIMUM_BLOCK);
+
+	// we persist any changes, so the buffer is no longer dirty
+	dirty = false;
 }
 
 void FixedSizeBuffer::Pin() {
 
 	auto &buffer_manager = block_manager.buffer_manager;
-	D_ASSERT(BlockId() < MAXIMUM_BLOCK);
-	buffer_handle = BufferHandle(buffer_manager.Pin(block_handle));
+	D_ASSERT(block_pointer.IsValid());
+	D_ASSERT(block_handle && block_handle->BlockId() < MAXIMUM_BLOCK);
+	D_ASSERT(!dirty);
+
+	buffer_handle = buffer_manager.Pin(block_handle);
+
+	// we need to copy the (partial) data into a new (not yet disk-backed) buffer handle
+	shared_ptr<BlockHandle> new_block_handle;
+	auto new_buffer_handle = buffer_manager.Allocate(Storage::BLOCK_SIZE, false, &new_block_handle);
+
+	memcpy(new_buffer_handle.Ptr(), buffer_handle.Ptr() + block_pointer.offset, allocation_size);
+
+	Destroy();
+	buffer_handle = std::move(new_buffer_handle);
+	block_handle = new_block_handle;
+	block_pointer = BlockPointer();
+}
+
+uint32_t FixedSizeBuffer::GetOffset(const idx_t bitmask_count) {
+
+	// this function calls Get() on the buffer, so the buffer must already be in memory
+	D_ASSERT(InMemory());
+
+	// get the bitmask data
+	auto bitmask_ptr = reinterpret_cast<validity_t *>(Get());
+	ValidityMask mask(bitmask_ptr);
+	auto data = mask.GetData();
+
+	// fills up a buffer sequentially before searching for free bits
+	if (mask.RowIsValid(segment_count)) {
+		mask.SetInvalid(segment_count);
+		return segment_count;
+	}
+
+	for (idx_t entry_idx = 0; entry_idx < bitmask_count; entry_idx++) {
+		// get an entry with free bits
+		if (data[entry_idx] == 0) {
+			continue;
+		}
+
+		// find the position of the free bit
+		auto entry = data[entry_idx];
+		idx_t first_valid_bit = 0;
+
+		// this loop finds the position of the rightmost set bit in entry and stores it
+		// in first_valid_bit
+		for (idx_t i = 0; i < 6; i++) {
+			// set the left half of the bits of this level to zero and test if the entry is still not zero
+			if (entry & BASE[i]) {
+				// first valid bit is in the rightmost s[i] bits
+				// permanently set the left half of the bits to zero
+				entry &= BASE[i];
+			} else {
+				// first valid bit is in the leftmost s[i] bits
+				// shift by s[i] for the next iteration and add s[i] to the position of the rightmost set bit
+				entry >>= SHIFT[i];
+				first_valid_bit += SHIFT[i];
+			}
+		}
+		D_ASSERT(entry);
+
+		auto prev_bits = entry_idx * sizeof(validity_t) * 8;
+		D_ASSERT(mask.RowIsValid(prev_bits + first_valid_bit));
+		mask.SetInvalid(prev_bits + first_valid_bit);
+		return (prev_bits + first_valid_bit);
+	}
+
+	throw InternalException("Invalid bitmask for FixedSizeAllocator");
+}
+
+uint32_t FixedSizeBuffer::GetMaxOffset(const idx_t available_segments) {
+
+	// this function calls Get() on the buffer, so the buffer must already be in memory
+	D_ASSERT(InMemory());
+
+	// finds the maximum zero bit in a bitmask, and adds one to it,
+	// so that max_offset * segment_size = allocated_size of this bitmask's buffer
+	idx_t entry_size = sizeof(validity_t) * 8;
+	idx_t bitmask_count = available_segments / entry_size;
+	if (available_segments % entry_size != 0) {
+		bitmask_count++;
+	}
+	uint32_t max_offset = bitmask_count * sizeof(validity_t) * 8;
+	auto bits_in_last_entry = available_segments % (sizeof(validity_t) * 8);
+
+	// get the bitmask data
+	auto bitmask_ptr = reinterpret_cast<validity_t *>(Get());
+	const ValidityMask mask(bitmask_ptr);
+	const auto data = mask.GetData();
+
+	D_ASSERT(bitmask_count > 0);
+	for (idx_t i = bitmask_count; i > 0; i--) {
+
+		auto entry = data[i - 1];
+
+		// set all bits after bits_in_last_entry
+		if (i == bitmask_count) {
+			entry |= ~idx_t(0) << bits_in_last_entry;
+		}
+
+		if (entry == ~idx_t(0)) {
+			max_offset -= sizeof(validity_t) * 8;
+			continue;
+		}
+
+		// invert data[entry_idx]
+		auto entry_inv = ~entry;
+		idx_t first_valid_bit = 0;
+
+		// then find the position of the LEFTMOST set bit
+		for (idx_t level = 0; level < 6; level++) {
+
+			// set the right half of the bits of this level to zero and test if the entry is still not zero
+			if (entry_inv & ~BASE[level]) {
+				// first valid bit is in the leftmost s[level] bits
+				// shift by s[level] for the next iteration and add s[level] to the position of the leftmost set bit
+				entry_inv >>= SHIFT[level];
+				first_valid_bit += SHIFT[level];
+			} else {
+				// first valid bit is in the rightmost s[level] bits
+				// permanently set the left half of the bits to zero
+				entry_inv &= BASE[level];
+			}
+		}
+		D_ASSERT(entry_inv);
+		max_offset -= sizeof(validity_t) * 8 - first_valid_bit;
+		D_ASSERT(!mask.RowIsValid(max_offset));
+		return max_offset + 1;
+	}
+
+	// there are no allocations in this buffer
+	// FIXME: put this line back in and then fix the missing vacuum bug in
+	// FIXME: test_index_large_aborted_append.test with force_restart
+	// FIXME: test if we still have non-dirty buffer to serialize after fixing this
+	//	throw InternalException("tried to serialize empty buffer");
+	return 0;
+}
+
+void FixedSizeBuffer::SetUninitializedRegions(PartialBlockForIndex &p_block_for_index, const idx_t segment_size,
+                                              const idx_t offset, const idx_t bitmask_offset) {
+
+	// this function calls Get() on the buffer, so the buffer must already be in memory
+	D_ASSERT(InMemory());
+
+	auto bitmask_ptr = reinterpret_cast<validity_t *>(Get());
+	ValidityMask mask(bitmask_ptr);
+
+	idx_t i = 0;
+	idx_t max_offset = offset + allocation_size;
+	idx_t current_offset = offset + bitmask_offset;
+	while (current_offset < max_offset) {
+
+		if (mask.RowIsValid(i)) {
+			D_ASSERT(current_offset + segment_size <= max_offset);
+			p_block_for_index.AddUninitializedRegion(current_offset, current_offset + segment_size);
+		}
+		current_offset += segment_size;
+		i++;
+	}
 }
 
 } // namespace duckdb

package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp

@@ -156,8 +156,7 @@ SinkFinalizeType PhysicalWindow::Finalize(Pipeline &pipeline, Event &event, Clie
 	}
 
 	// Find the first group to sort
-	auto &groups = state.global_partition->grouping_data->GetPartitions();
-	if (groups.empty()) {
+	if (!state.global_partition->HasMergeTasks()) {
 		// Empty input!
 		return SinkFinalizeType::NO_OUTPUT_POSSIBLE;
 	}
@@ -538,7 +537,7 @@ WindowGlobalSourceState::Task WindowGlobalSourceState::NextTask(idx_t hash_bin)
 	if (hash_bin < bin_count) {
 		//	Find a non-empty hash group.
 		for (; hash_bin < hash_groups.size(); hash_bin = next_build++) {
-			if (hash_groups[hash_bin]) {
+			if (hash_groups[hash_bin] && hash_groups[hash_bin]->count) {
 				auto result = CreateTask(hash_bin);
 				if (result.second) {
 					return result;

package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp

@@ -163,8 +163,7 @@ SinkFinalizeType PhysicalAsOfJoin::Finalize(Pipeline &pipeline, Event &event, Cl
 	gstate.lhs_sink->SyncPartitioning(gstate.rhs_sink);
 
 	// Find the first group to sort
-	auto &groups = gstate.rhs_sink.grouping_data->GetPartitions();
-	if (groups.empty() && EmptyResultIfRHSIsEmpty()) {
+	if (!gstate.rhs_sink.HasMergeTasks() && EmptyResultIfRHSIsEmpty()) {
 		// Empty input!
 		return SinkFinalizeType::NO_OUTPUT_POSSIBLE;
 	}
@@ -433,6 +432,9 @@ void AsOfProbeBuffer::BeginLeftScan(hash_t scan_bin) {
 
 	left_hash = lhs_sink.hash_groups[left_group].get();
 	auto &left_sort = *(left_hash->global_sort);
+	if (left_sort.sorted_blocks.empty()) {
+		return;
+	}
 	lhs_scanner = make_uniq<PayloadScanner>(left_sort, false);
 	left_itr = make_uniq<SBIterator>(left_sort, iterator_comp);
 
@@ -681,14 +683,16 @@ class AsOfLocalSourceState : public LocalSourceState {
 public:
 	using HashGroupPtr = unique_ptr<PartitionGlobalHashGroup>;
 
-	AsOfLocalSourceState(AsOfGlobalSourceState &gsource, const PhysicalAsOfJoin &op);
+	AsOfLocalSourceState(AsOfGlobalSourceState &gsource, const PhysicalAsOfJoin &op, ClientContext &client_p);
 
-	void CombineLeftPartitions();
-	void MergeLeftPartitions();
+	//	Return true if we were not interrupted (another thread died)
+	bool CombineLeftPartitions();
+	bool MergeLeftPartitions();
 
 	idx_t BeginRightScan(const idx_t hash_bin);
 
 	AsOfGlobalSourceState &gsource;
+	ClientContext &client;
 
 	//! The left side partition being probed
 	AsOfProbeBuffer probe_buffer;
@@ -702,38 +706,45 @@ public:
 	const bool *found_match = {};
 };
 
-AsOfLocalSourceState::AsOfLocalSourceState(AsOfGlobalSourceState &gsource, const PhysicalAsOfJoin &op)
-    : gsource(gsource), probe_buffer(gsource.gsink.lhs_sink->context, op) {
+AsOfLocalSourceState::AsOfLocalSourceState(AsOfGlobalSourceState &gsource, const PhysicalAsOfJoin &op,
+                                           ClientContext &client_p)
+    : gsource(gsource), client(client_p), probe_buffer(gsource.gsink.lhs_sink->context, op) {
 	gsource.mergers++;
 }
 
-void AsOfLocalSourceState::CombineLeftPartitions() {
+bool AsOfLocalSourceState::CombineLeftPartitions() {
 	const auto buffer_count = gsource.gsink.lhs_buffers.size();
-	while (gsource.combined < buffer_count) {
+	while (gsource.combined < buffer_count && !client.interrupted) {
 		const auto next_combine = gsource.next_combine++;
 		if (next_combine < buffer_count) {
 			gsource.gsink.lhs_buffers[next_combine]->Combine();
 			++gsource.combined;
 		} else {
-			std::this_thread::yield();
+			TaskScheduler::GetScheduler(client).YieldThread();
 		}
 	}
+
+	return !client.interrupted;
 }
 
-void AsOfLocalSourceState::MergeLeftPartitions() {
+bool AsOfLocalSourceState::MergeLeftPartitions() {
 	PartitionGlobalMergeStates::Callback local_callback;
 	PartitionLocalMergeState local_merge(*gsource.gsink.lhs_sink);
 	gsource.GetMergeStates().ExecuteTask(local_merge, local_callback);
 	gsource.merged++;
-	while (gsource.merged < gsource.mergers) {
-		std::this_thread::yield();
+	while (gsource.merged < gsource.mergers && !client.interrupted) {
+		TaskScheduler::GetScheduler(client).YieldThread();
 	}
+	return !client.interrupted;
 }
 
 idx_t AsOfLocalSourceState::BeginRightScan(const idx_t hash_bin_p) {
 	hash_bin = hash_bin_p;
 
 	hash_group = std::move(gsource.gsink.rhs_sink.hash_groups[hash_bin]);
+	if (hash_group->global_sort->sorted_blocks.empty()) {
+		return 0;
+	}
 	scanner = make_uniq<PayloadScanner>(*hash_group->global_sort);
 	found_match = gsource.gsink.right_outers[hash_bin].GetMatches();
 
@@ -743,7 +754,7 @@ idx_t AsOfLocalSourceState::BeginRightScan(const idx_t hash_bin_p) {
 unique_ptr<LocalSourceState> PhysicalAsOfJoin::GetLocalSourceState(ExecutionContext &context,
                                                                    GlobalSourceState &gstate) const {
 	auto &gsource = gstate.Cast<AsOfGlobalSourceState>();
-	return make_uniq<AsOfLocalSourceState>(gsource, *this);
+	return make_uniq<AsOfLocalSourceState>(gsource, *this, context.client);
 }
 
 SourceResultType PhysicalAsOfJoin::GetData(ExecutionContext &context, DataChunk &chunk,
@@ -751,17 +762,21 @@ SourceResultType PhysicalAsOfJoin::GetData(ExecutionContext &context, DataChunk
 	auto &gsource = input.global_state.Cast<AsOfGlobalSourceState>();
 	auto &lsource = input.local_state.Cast<AsOfLocalSourceState>();
 	auto &rhs_sink = gsource.gsink.rhs_sink;
+	auto &client = context.client;
 
 	//	Step 1: Combine the partitions
-	lsource.CombineLeftPartitions();
+	if (!lsource.CombineLeftPartitions()) {
+		return SourceResultType::FINISHED;
+	}
 
 	//	Step 2: Sort on all threads
-	lsource.MergeLeftPartitions();
+	if (!lsource.MergeLeftPartitions()) {
+		return SourceResultType::FINISHED;
+	}
 
 	//	Step 3: Join the partitions
 	auto &lhs_sink = *gsource.gsink.lhs_sink;
-	auto &partitions = lhs_sink.grouping_data->GetPartitions();
-	const auto left_bins = partitions.size();
+	const auto left_bins = lhs_sink.grouping_data ? lhs_sink.grouping_data->GetPartitions().size() : 1;
 	while (gsource.flushed < left_bins) {
 		//	Make sure we have something to flush
 		if (!lsource.probe_buffer.Scanning()) {
@@ -769,13 +784,13 @@ SourceResultType PhysicalAsOfJoin::GetData(ExecutionContext &context, DataChunk
 			if (left_bin < left_bins) {
 				//	More to flush
 				lsource.probe_buffer.BeginLeftScan(left_bin);
-			} else if (!IsRightOuterJoin(join_type)) {
+			} else if (!IsRightOuterJoin(join_type) || client.interrupted) {
 				return SourceResultType::FINISHED;
 			} else {
 				//	Wait for all threads to finish
 				//	TODO: How to implement a spin wait correctly?
 				//	Returning BLOCKED seems to hang the system.
-				std::this_thread::yield();
+				TaskScheduler::GetScheduler(client).YieldThread();
 				continue;
 			}
 		}

package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp

@@ -194,7 +194,7 @@ public:
 		}
 		auto new_count = current_collection->GetTotalRows();
 		auto batch_type =
-		    new_count < RowGroup::ROW_GROUP_SIZE ? RowGroupBatchType::NOT_FLUSHED : RowGroupBatchType::FLUSHED;
+		    new_count < Storage::ROW_GROUP_SIZE ? RowGroupBatchType::NOT_FLUSHED : RowGroupBatchType::FLUSHED;
 		if (batch_type == RowGroupBatchType::FLUSHED && writer) {
 			writer->WriteLastRowGroup(*current_collection);
 		}

package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp

@@ -482,7 +482,7 @@ SinkCombineResultType PhysicalInsert::Combine(ExecutionContext &context, Operato
 
 	lock_guard<mutex> lock(gstate.lock);
 	gstate.insert_count += append_count;
-	if (append_count < RowGroup::ROW_GROUP_SIZE) {
+	if (append_count < Storage::ROW_GROUP_SIZE) {
 		// we have few rows - append to the local storage directly
 		auto &table = gstate.table;
 		auto &storage = table.GetStorage();

package/src/duckdb/src/function/table/arrow_conversion.cpp

@@ -837,7 +837,15 @@ void ArrowTableFunction::ArrowToDuckDB(ArrowScanLocalState &scan_state, const ar
 			throw InvalidInputException("arrow_scan: array length mismatch");
 		}
 		// Make sure this Vector keeps the Arrow chunk alive in case we can zero-copy the data
-		output.data[idx].GetBuffer()->SetAuxiliaryData(make_uniq<ArrowAuxiliaryData>(scan_state.chunk));
+		if (scan_state.arrow_owned_data.find(idx) == scan_state.arrow_owned_data.end()) {
+			auto arrow_data = make_shared<ArrowArrayWrapper>();
+			arrow_data->arrow_array = scan_state.chunk->arrow_array;
+			scan_state.chunk->arrow_array.release = nullptr;
+			scan_state.arrow_owned_data[idx] = arrow_data;
+		}
+
+		output.data[idx].GetBuffer()->SetAuxiliaryData(make_uniq<ArrowAuxiliaryData>(scan_state.arrow_owned_data[idx]));
+
 		D_ASSERT(arrow_convert_data.find(col_idx) != arrow_convert_data.end());
 		auto &arrow_type = *arrow_convert_data.at(col_idx);
 		if (array.dictionary) {

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.2-dev4203"
+#define DUCKDB_VERSION "0.8.2-dev4376"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "7208022cac"
+#define DUCKDB_SOURCE_ID "312b995450"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/constants.hpp

@@ -58,21 +58,6 @@ struct DConstants {
 	static constexpr const idx_t INVALID_INDEX = idx_t(-1);
 };
 
-struct Storage {
-	//! The size of a hard disk sector, only really needed for Direct IO
-	constexpr static int SECTOR_SIZE = 4096;
-	//! Block header size for blocks written to the storage
-	constexpr static int BLOCK_HEADER_SIZE = sizeof(uint64_t);
-	// Size of a memory slot managed by the StorageManager. This is the quantum of allocation for Blocks on DuckDB. We
-	// default to 256KB. (1 << 18)
-	constexpr static int BLOCK_ALLOC_SIZE = 262144;
-	//! The actual memory space that is available within the blocks
-	constexpr static int BLOCK_SIZE = BLOCK_ALLOC_SIZE - BLOCK_HEADER_SIZE;
-	//! The size of the headers. This should be small and written more or less atomically by the hard disk. We default
-	//! to the page size, which is 4KB. (1 << 12)
-	constexpr static int FILE_HEADER_SIZE = 4096;
-};
-
 struct LogicalIndex {
 	explicit LogicalIndex(idx_t index) : index(index) {
 	}

package/src/duckdb/src/include/duckdb/common/serializer/memory_stream.hpp

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/common/serializer/buffer_stream.hpp
+// duckdb/common/serializer/memory_stream.hpp
 //
 //
 //===----------------------------------------------------------------------===//

package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp

@@ -18,7 +18,6 @@ namespace duckdb {
 class PartitionGlobalHashGroup {
 public:
 	using GlobalSortStatePtr = unique_ptr<GlobalSortState>;
-	using LocalSortStatePtr = unique_ptr<LocalSortState>;
 	using Orders = vector<BoundOrderByNode>;
 	using Types = vector<LogicalType>;
 
@@ -54,6 +53,8 @@ public:
 	                         const vector<BoundOrderByNode> &order_bys, const Types &payload_types,
 	                         const vector<unique_ptr<BaseStatistics>> &partitions_stats, idx_t estimated_cardinality);
 
+	bool HasMergeTasks() const;
+
 	unique_ptr<RadixPartitionedTupleData> CreatePartition(idx_t new_bits) const;
 	void SyncPartitioning(const PartitionGlobalSinkState &other);
 
@@ -97,21 +98,26 @@ private:
 
 class PartitionLocalSinkState {
 public:
+	using LocalSortStatePtr = unique_ptr<LocalSortState>;
+
 	PartitionLocalSinkState(ClientContext &context, PartitionGlobalSinkState &gstate_p);
 
 	// Global state
 	PartitionGlobalSinkState &gstate;
 	Allocator &allocator;
 
-	// OVER(PARTITION BY...) (hash grouping)
+	//	Shared expression evaluation
 	ExpressionExecutor executor;
 	DataChunk group_chunk;
 	DataChunk payload_chunk;
+	size_t sort_cols;
+
+	// OVER(PARTITION BY...) (hash grouping)
 	unique_ptr<PartitionedTupleData> local_partition;
 	unique_ptr<PartitionedTupleDataAppendState> local_append;
 
-	// OVER(...) (sorting)
-	size_t sort_cols;
+	// OVER(ORDER BY...) (only sorting)
+	LocalSortStatePtr local_sort;
 
 	// OVER() (no sorting)
 	RowLayout payload_layout;
@@ -134,8 +140,12 @@ class PartitionGlobalMergeState {
 public:
 	using GroupDataPtr = unique_ptr<TupleDataCollection>;
 
+	//	OVER(PARTITION BY...)
 	PartitionGlobalMergeState(PartitionGlobalSinkState &sink, GroupDataPtr group_data, hash_t hash_bin);
 
+	//	OVER(ORDER BY...)
+	explicit PartitionGlobalMergeState(PartitionGlobalSinkState &sink);
+
 	bool IsSorted() const {
 		lock_guard<mutex> guard(lock);
 		return stage == PartitionSortStage::SORTED;

package/src/duckdb/src/include/duckdb/common/types/validity_mask.hpp

@@ -332,6 +332,9 @@ public:
 	DUCKDB_API string ToString(idx_t count) const;
 
 	DUCKDB_API static bool IsAligned(idx_t count);
+
+	void Write(WriteStream &writer, idx_t count);
+	void Read(ReadStream &reader, idx_t count);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp

@@ -45,6 +45,8 @@ public:
 	//! Get a new chain of leaf nodes, might cause new buffer allocations,
 	//! with the node parameter holding the tail of the chain
 	static void New(ART &art, reference<Node> &node, const row_t *row_ids, idx_t count);
+	//! Get a new leaf node without any data
+	static Leaf &New(ART &art, Node &node);
 	//! Free the leaf (chain)
 	static void Free(ART &art, Node &node);
 

package/src/duckdb/src/include/duckdb/execution/index/fixed_size_allocator.hpp

@@ -29,10 +29,6 @@ public:
 	//! We can vacuum 10% or more of the total in-memory footprint
 	static constexpr uint8_t VACUUM_THRESHOLD = 10;
 
-	//! Constants for fast offset calculations in the bitmask
-	static constexpr idx_t BASE[] = {0x00000000FFFFFFFF, 0x0000FFFF, 0x00FF, 0x0F, 0x3, 0x1};
-	static constexpr uint8_t SHIFT[] = {32, 16, 8, 4, 2, 1};
-
 public:
 	FixedSizeAllocator(const idx_t segment_size, BlockManager &block_manager);
 
@@ -80,7 +76,7 @@ public:
 	IndexPointer VacuumPointer(const IndexPointer ptr);
 
 	//! Serializes all in-memory buffers and the metadata
-	BlockPointer Serialize(MetadataWriter &writer);
+	BlockPointer Serialize(PartialBlockManager &partial_block_manager, MetadataWriter &writer);
 	//! Deserializes all metadata
 	void Deserialize(const BlockPointer &block_pointer);
 
@@ -117,8 +113,6 @@ private:
 		auto buffer_ptr = buffer.Get(dirty);
 		return buffer_ptr + ptr.GetOffset() * segment_size + bitmask_offset;
 	}
-	//! Returns the first free offset in a bitmask
-	uint32_t GetOffset(ValidityMask &mask, const idx_t segment_count);
 	//! Returns an available buffer id
 	idx_t GetAvailableBufferId() const;
 };