duckdb 0.8.2-dev4142.0 → 0.8.2-dev4314.0

package/src/duckdb/src/include/duckdb/common/serializer/memory_stream.hpp

@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/serializer/buffer_stream.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/serializer/write_stream.hpp"
+#include "duckdb/common/serializer/read_stream.hpp"
+#include "duckdb/common/typedefs.hpp"
+
+namespace duckdb {
+
+class MemoryStream : public WriteStream, public ReadStream {
+private:
+	idx_t position;
+	idx_t capacity;
+	bool owns_data;
+	data_ptr_t data;
+
+public:
+	// Create a new owning MemoryStream with an internal  backing buffer with the specified capacity. The stream will
+	// own the backing buffer, resize it when needed and free its memory when the stream is destroyed
+	explicit MemoryStream(idx_t capacity = 512);
+
+	// Create a new non-owning MemoryStream over the specified external buffer and capacity. The stream will not take
+	// ownership of the backing buffer, will not attempt to resize it and will not free the memory when the stream
+	// is destroyed
+	explicit MemoryStream(data_ptr_t buffer, idx_t capacity);
+
+	~MemoryStream() override;
+
+	// Write data to the stream.
+	// Throws if the write would exceed the capacity of the stream and the backing buffer is not owned by the stream
+	void WriteData(const_data_ptr_t buffer, idx_t write_size) override;
+
+	// Read data from the stream.
+	// Throws if the read would exceed the capacity of the stream
+	void ReadData(data_ptr_t buffer, idx_t read_size) override;
+
+	// Rewind the stream to the start, keeping the capacity and the backing buffer intact
+	void Rewind();
+
+	// Release ownership of the backing buffer and turn a owning stream into a non-owning one.
+	// The stream will no longer be responsible for freeing the data.
+	// The stream will also no longer attempt to automatically resize the buffer when the capacity is reached.
+	void Release();
+
+	// Get a pointer to the underlying backing buffer
+	data_ptr_t GetData() const;
+
+	// Get the current position in the stream
+	idx_t GetPosition() const;
+
+	// Get the capacity of the stream
+	idx_t GetCapacity() const;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/serializer/read_stream.hpp

@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/serializer/read_stream.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/catalog/catalog.hpp"
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/exception.hpp"
+#include "duckdb/common/vector.hpp"
+#include <type_traits>
+
+namespace duckdb {
+
+class ReadStream {
+public:
+	// Reads a set amount of data from the stream into the specified buffer and moves the stream forward accordingly
+	virtual void ReadData(data_ptr_t buffer, idx_t read_size) = 0;
+
+	// Reads a type from the stream and moves the stream forward sizeof(T) bytes
+	// The type must be a standard layout type
+	template <class T>
+	T Read() {
+		static_assert(std::is_standard_layout<T>(), "Read element must be a standard layout data type");
+		T value;
+		ReadData(data_ptr_cast(&value), sizeof(T));
+		return value;
+	}
+
+	virtual ~ReadStream() {
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/serializer/serialization_traits.hpp

@@ -7,10 +7,12 @@
 #include "duckdb/common/unordered_set.hpp"
 #include "duckdb/common/set.hpp"
 #include "duckdb/common/shared_ptr.hpp"
+#include "duckdb/common/unique_ptr.hpp"
+
 namespace duckdb {
 
-class FormatSerializer;   // Forward declare
-class FormatDeserializer; // Forward declare
+class Serializer;   // Forward declare
+class Deserializer; // Forward declare
 
 typedef uint16_t field_id_t;
 const field_id_t MESSAGE_TERMINATOR_FIELD_ID = 0xFFFF;
@@ -19,34 +21,34 @@ const field_id_t MESSAGE_TERMINATOR_FIELD_ID = 0xFFFF;
 template <class...>
 using void_t = void;
 
-// Check for anything implementing a `void FormatSerialize(FormatSerializer &FormatSerializer)` method
+// Check for anything implementing a `void Serialize(Serializer &Serializer)` method
 template <typename T, typename = T>
 struct has_serialize : std::false_type {};
 template <typename T>
 struct has_serialize<
     T, typename std::enable_if<
-           std::is_same<decltype(std::declval<T>().FormatSerialize(std::declval<FormatSerializer &>())), void>::value,
-           T>::type> : std::true_type {};
+           std::is_same<decltype(std::declval<T>().Serialize(std::declval<Serializer &>())), void>::value, T>::type>
+    : std::true_type {};
 
 template <typename T, typename = T>
 struct has_deserialize : std::false_type {};
 
-// Accept `static unique_ptr<T> FormatDeserialize(FormatDeserializer& deserializer)`
+// Accept `static unique_ptr<T> Deserialize(Deserializer& deserializer)`
 template <typename T>
 struct has_deserialize<
-    T, typename std::enable_if<std::is_same<decltype(T::FormatDeserialize), unique_ptr<T>(FormatDeserializer &)>::value,
-                               T>::type> : std::true_type {};
+    T, typename std::enable_if<std::is_same<decltype(T::Deserialize), unique_ptr<T>(Deserializer &)>::value, T>::type>
+    : std::true_type {};
 
-// Accept `static shared_ptr<T> FormatDeserialize(FormatDeserializer& deserializer)`
+// Accept `static shared_ptr<T> Deserialize(Deserializer& deserializer)`
 template <typename T>
 struct has_deserialize<
-    T, typename std::enable_if<std::is_same<decltype(T::FormatDeserialize), shared_ptr<T>(FormatDeserializer &)>::value,
-                               T>::type> : std::true_type {};
+    T, typename std::enable_if<std::is_same<decltype(T::Deserialize), shared_ptr<T>(Deserializer &)>::value, T>::type>
+    : std::true_type {};
 
-// Accept `static T FormatDeserialize(FormatDeserializer& deserializer)`
+// Accept `static T Deserialize(Deserializer& deserializer)`
 template <typename T>
 struct has_deserialize<
-    T, typename std::enable_if<std::is_same<decltype(T::FormatDeserialize), T(FormatDeserializer &)>::value, T>::type>
+    T, typename std::enable_if<std::is_same<decltype(T::Deserialize), T(Deserializer &)>::value, T>::type>
     : std::true_type {};
 
 // Check if T is a vector, and provide access to the inner type

package/src/duckdb/src/include/duckdb/common/serializer/{format_serializer.hpp → serializer.hpp}

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/common/serializer/format_serializer.hpp
+// duckdb/common/serializer/serializer.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -9,8 +9,6 @@
 #pragma once
 
 #include "duckdb/common/enum_util.hpp"
-#include "duckdb/common/field_writer.hpp"
-#include "duckdb/common/serializer.hpp"
 #include "duckdb/common/serializer/serialization_traits.hpp"
 #include "duckdb/common/types/interval.hpp"
 #include "duckdb/common/types/string_type.hpp"
@@ -19,18 +17,21 @@
 
 namespace duckdb {
 
-class FormatSerializer {
+class Serializer {
 protected:
 	bool serialize_enum_as_string = false;
 	bool serialize_default_values = false;
 
 public:
+	virtual ~Serializer() {
+	}
+
 	class List {
-		friend FormatSerializer;
+		friend Serializer;
 
 	private:
-		FormatSerializer &serializer;
-		explicit List(FormatSerializer &serializer) : serializer(serializer) {
+		Serializer &serializer;
+		explicit List(Serializer &serializer) : serializer(serializer) {
 		}
 
 	public:
@@ -223,11 +224,11 @@ protected:
 		OnListEnd();
 	}
 
-	// class or struct implementing `FormatSerialize(FormatSerializer& FormatSerializer)`;
+	// class or struct implementing `Serialize(Serializer& Serializer)`;
 	template <typename T>
 	typename std::enable_if<has_serialize<T>::value>::type WriteValue(const T &value) {
 		OnObjectBegin();
-		value.FormatSerialize(*this);
+		value.Serialize(*this);
 		OnObjectEnd();
 	}
 
@@ -275,18 +276,18 @@ protected:
 
 // We need to special case vector<bool> because elements of vector<bool> cannot be referenced
 template <>
-void FormatSerializer::WriteValue(const vector<bool> &vec);
+void Serializer::WriteValue(const vector<bool> &vec);
 
 // List Impl
 template <class FUNC>
-void FormatSerializer::List::WriteObject(FUNC f) {
+void Serializer::List::WriteObject(FUNC f) {
 	serializer.OnObjectBegin();
 	f(serializer);
 	serializer.OnObjectEnd();
 }
 
 template <class T>
-void FormatSerializer::List::WriteElement(const T &value) {
+void Serializer::List::WriteElement(const T &value) {
 	serializer.WriteValue(value);
 }
 

package/src/duckdb/src/include/duckdb/common/serializer/write_stream.hpp

@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/serializer/write_stream.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/catalog/catalog.hpp"
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/exception.hpp"
+#include "duckdb/common/vector.hpp"
+#include <type_traits>
+
+namespace duckdb {
+
+class WriteStream {
+public:
+	// Writes a set amount of data from the specified buffer into the stream and moves the stream forward accordingly
+	virtual void WriteData(const_data_ptr_t buffer, idx_t write_size) = 0;
+
+	// Writes a type into the stream and moves the stream forward sizeof(T) bytes
+	// The type must be a standard layout type
+	template <class T>
+	void Write(T element) {
+		static_assert(std::is_standard_layout<T>(), "Write element must be a standard layout data type");
+		WriteData(const_data_ptr_cast(&element), sizeof(T));
+	}
+
+	virtual ~WriteStream() {
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp

@@ -18,7 +18,6 @@ namespace duckdb {
 class PartitionGlobalHashGroup {
 public:
 	using GlobalSortStatePtr = unique_ptr<GlobalSortState>;
-	using LocalSortStatePtr = unique_ptr<LocalSortState>;
 	using Orders = vector<BoundOrderByNode>;
 	using Types = vector<LogicalType>;
 
@@ -54,6 +53,8 @@ public:
 	                         const vector<BoundOrderByNode> &order_bys, const Types &payload_types,
 	                         const vector<unique_ptr<BaseStatistics>> &partitions_stats, idx_t estimated_cardinality);
 
+	bool HasMergeTasks() const;
+
 	unique_ptr<RadixPartitionedTupleData> CreatePartition(idx_t new_bits) const;
 	void SyncPartitioning(const PartitionGlobalSinkState &other);
 
@@ -97,21 +98,26 @@ private:
 
 class PartitionLocalSinkState {
 public:
+	using LocalSortStatePtr = unique_ptr<LocalSortState>;
+
 	PartitionLocalSinkState(ClientContext &context, PartitionGlobalSinkState &gstate_p);
 
 	// Global state
 	PartitionGlobalSinkState &gstate;
 	Allocator &allocator;
 
-	// OVER(PARTITION BY...) (hash grouping)
+	//	Shared expression evaluation
 	ExpressionExecutor executor;
 	DataChunk group_chunk;
 	DataChunk payload_chunk;
+	size_t sort_cols;
+
+	// OVER(PARTITION BY...) (hash grouping)
 	unique_ptr<PartitionedTupleData> local_partition;
 	unique_ptr<PartitionedTupleDataAppendState> local_append;
 
-	// OVER(...) (sorting)
-	size_t sort_cols;
+	// OVER(ORDER BY...) (only sorting)
+	LocalSortStatePtr local_sort;
 
 	// OVER() (no sorting)
 	RowLayout payload_layout;
@@ -134,8 +140,12 @@ class PartitionGlobalMergeState {
 public:
 	using GroupDataPtr = unique_ptr<TupleDataCollection>;
 
+	//	OVER(PARTITION BY...)
 	PartitionGlobalMergeState(PartitionGlobalSinkState &sink, GroupDataPtr group_data, hash_t hash_bin);
 
+	//	OVER(ORDER BY...)
+	explicit PartitionGlobalMergeState(PartitionGlobalSinkState &sink);
+
 	bool IsSorted() const {
 		lock_guard<mutex> guard(lock);
 		return stage == PartitionSortStage::SORTED;

package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp

@@ -154,8 +154,8 @@ public:
 	//! Get a vector of the segments in this ColumnDataCollection
 	const vector<unique_ptr<ColumnDataCollectionSegment>> &GetSegments() const;
 
-	void FormatSerialize(FormatSerializer &serializer) const;
-	static unique_ptr<ColumnDataCollection> FormatDeserialize(FormatDeserializer &deserializer);
+	void Serialize(Serializer &serializer) const;
+	static unique_ptr<ColumnDataCollection> Deserialize(Deserializer &deserializer);
 
 private:
 	//! Creates a new segment within the ColumnDataCollection

package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp

@@ -19,8 +19,8 @@ class Allocator;
 class ClientContext;
 class ExecutionContext;
 class VectorCache;
-class FormatSerializer;
-class FormatDeserializer;
+class Serializer;
+class Deserializer;
 
 //!  A Data Chunk represents a set of vectors.
 /*!
@@ -138,14 +138,9 @@ public:
 	//! Vector to point back to the data owned by this DataChunk.
 	DUCKDB_API void Reset();
 
-	//! Serializes a DataChunk to a stand-alone binary blob
-	DUCKDB_API void Serialize(Serializer &serializer);
-	//! Deserializes a blob back into a DataChunk
+	DUCKDB_API void Serialize(Serializer &serializer) const;
 	DUCKDB_API void Deserialize(Deserializer &source);
 
-	DUCKDB_API void FormatSerialize(FormatSerializer &serializer) const;
-	DUCKDB_API void FormatDeserialize(FormatDeserializer &source);
-
 	//! Hashes the DataChunk to the target vector
 	DUCKDB_API void Hash(Vector &result);
 	//! Hashes specific vectors of the DataChunk to the target vector

package/src/duckdb/src/include/duckdb/common/types/hyperloglog.hpp

@@ -20,11 +20,8 @@ namespace duckdb {
 
 enum class HLLStorageType : uint8_t { UNCOMPRESSED = 1 };
 
-class FieldWriter;
-class FieldReader;
-
-class FormatSerializer;
-class FormatDeserializer;
+class Serializer;
+class Deserializer;
 
 //! The HyperLogLog class holds a HyperLogLog counter for approximate cardinality counting
 class HyperLogLog {
@@ -49,12 +46,9 @@ public:
 	data_ptr_t GetPtr() const;
 	//! Get copy of the HLL
 	unique_ptr<HyperLogLog> Copy();
-	//! (De)Serialize the HLL
-	void Serialize(FieldWriter &writer) const;
-	static unique_ptr<HyperLogLog> Deserialize(FieldReader &reader);
 
-	void FormatSerialize(FormatSerializer &serializer) const;
-	static unique_ptr<HyperLogLog> FormatDeserialize(FormatDeserializer &deserializer);
+	void Serialize(Serializer &serializer) const;
+	static unique_ptr<HyperLogLog> Deserialize(Deserializer &deserializer);
 
 public:
 	//! Compute HLL hashes over vdata, and store them in 'hashes'

package/src/duckdb/src/include/duckdb/common/types/interval.hpp

@@ -16,8 +16,8 @@ struct dtime_t;
 struct date_t;
 struct timestamp_t;
 
-class FormatSerializer;
-class FormatDeserializer;
+class Serializer;
+class Deserializer;
 
 struct interval_t {
 	int32_t months;
@@ -29,8 +29,8 @@ struct interval_t {
 	}
 
 	// Serialization
-	void FormatSerialize(FormatSerializer &serializer) const;
-	static interval_t FormatDeserialize(FormatDeserializer &source);
+	void Serialize(Serializer &serializer) const;
+	static interval_t Deserialize(Deserializer &source);
 };
 
 //! The Interval class is a static class that holds helper functions for the Interval

package/src/duckdb/src/include/duckdb/common/types/value.hpp

@@ -20,8 +20,6 @@
 namespace duckdb {
 
 class CastFunctionSet;
-class Deserializer;
-class Serializer;
 struct GetCastFunctionInput;
 struct ExtraValueInfo;
 
@@ -225,10 +223,8 @@ public:
 
 	//! Serializes a Value to a stand-alone binary blob
 	DUCKDB_API void Serialize(Serializer &serializer) const;
-	DUCKDB_API void FormatSerialize(FormatSerializer &serializer) const;
-	DUCKDB_API static Value FormatDeserialize(FormatDeserializer &deserializer);
 	//! Deserializes a Value from a blob
-	DUCKDB_API static Value Deserialize(Deserializer &source);
+	DUCKDB_API static Value Deserialize(Deserializer &deserializer);
 
 	//===--------------------------------------------------------------------===//
 	// Comparison Operators

package/src/duckdb/src/include/duckdb/common/types/vector.hpp

@@ -173,13 +173,8 @@ public:
 	//! This functions resizes the vector
 	DUCKDB_API void Resize(idx_t cur_size, idx_t new_size);
 
-	//! Serializes a Vector to a stand-alone binary blob
-	DUCKDB_API void Serialize(idx_t count, Serializer &serializer);
-	//! Deserializes a blob back into a Vector
-	DUCKDB_API void Deserialize(idx_t count, Deserializer &source);
-
-	DUCKDB_API void FormatSerialize(FormatSerializer &serializer, idx_t count);
-	DUCKDB_API void FormatDeserialize(FormatDeserializer &deserializer, idx_t count);
+	DUCKDB_API void Serialize(Serializer &serializer, idx_t count);
+	DUCKDB_API void Deserialize(Deserializer &deserializer, idx_t count);
 
 	// Getters
 	inline VectorType GetVectorType() const {

package/src/duckdb/src/include/duckdb/common/types.hpp

@@ -17,15 +17,12 @@
 
 namespace duckdb {
 
-class FormatSerializer;
-class FormatDeserializer;
 class Serializer;
 class Deserializer;
 class Value;
 class TypeCatalogEntry;
 class Vector;
 class ClientContext;
-class FieldWriter;
 
 struct string_t;
 
@@ -283,14 +280,8 @@ struct LogicalType {
 		return !(*this == rhs);
 	}
 
-	//! Serializes a LogicalType to a stand-alone binary blob
 	DUCKDB_API void Serialize(Serializer &serializer) const;
-
-	//! Deserializes a blob back into an LogicalType
-	DUCKDB_API static LogicalType Deserialize(Deserializer &source);
-
-	DUCKDB_API void FormatSerialize(FormatSerializer &serializer) const;
-	DUCKDB_API static LogicalType FormatDeserialize(FormatDeserializer &deserializer);
+	DUCKDB_API static LogicalType Deserialize(Deserializer &deserializer);
 
 
 	static bool TypeIsTimestamp(LogicalTypeId id) {

package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp

@@ -45,6 +45,8 @@ public:
 	//! Get a new chain of leaf nodes, might cause new buffer allocations,
 	//! with the node parameter holding the tail of the chain
 	static void New(ART &art, reference<Node> &node, const row_t *row_ids, idx_t count);
+	//! Get a new leaf node without any data
+	static Leaf &New(ART &art, Node &node);
 	//! Free the leaf (chain)
 	static void Free(ART &art, Node &node);
 

package/src/duckdb/src/include/duckdb/execution/index/fixed_size_allocator.hpp

@@ -29,10 +29,6 @@ public:
 	//! We can vacuum 10% or more of the total in-memory footprint
 	static constexpr uint8_t VACUUM_THRESHOLD = 10;
 
-	//! Constants for fast offset calculations in the bitmask
-	static constexpr idx_t BASE[] = {0x00000000FFFFFFFF, 0x0000FFFF, 0x00FF, 0x0F, 0x3, 0x1};
-	static constexpr uint8_t SHIFT[] = {32, 16, 8, 4, 2, 1};
-
 public:
 	FixedSizeAllocator(const idx_t segment_size, BlockManager &block_manager);
 
@@ -80,7 +76,7 @@ public:
 	IndexPointer VacuumPointer(const IndexPointer ptr);
 
 	//! Serializes all in-memory buffers and the metadata
-	BlockPointer Serialize(MetadataWriter &writer);
+	BlockPointer Serialize(PartialBlockManager &partial_block_manager, MetadataWriter &writer);
 	//! Deserializes all metadata
 	void Deserialize(const BlockPointer &block_pointer);
 
@@ -117,8 +113,6 @@ private:
 		auto buffer_ptr = buffer.Get(dirty);
 		return buffer_ptr + ptr.GetOffset() * segment_size + bitmask_offset;
 	}
-	//! Returns the first free offset in a bitmask
-	uint32_t GetOffset(ValidityMask &mask, const idx_t segment_count);
 	//! Returns an available buffer id
 	idx_t GetAvailableBufferId() const;
 };

package/src/duckdb/src/include/duckdb/execution/index/fixed_size_buffer.hpp

@@ -8,36 +8,61 @@
 
 #pragma once
 
+#include "duckdb/storage/partial_block_manager.hpp"
 #include "duckdb/common/typedefs.hpp"
 #include "duckdb/storage/buffer/block_handle.hpp"
 #include "duckdb/storage/buffer/buffer_handle.hpp"
+#include "duckdb/storage/block_manager.hpp"
 
 namespace duckdb {
 
 class FixedSizeAllocator;
 class MetadataWriter;
 
+struct PartialBlockForIndex : public PartialBlock {
+public:
+	PartialBlockForIndex(PartialBlockState state, BlockManager &block_manager,
+	                     const shared_ptr<BlockHandle> &block_handle);
+	~PartialBlockForIndex() override {};
+
+public:
+	void Flush(const idx_t free_space_left) override;
+	void Clear() override;
+	void Merge(PartialBlock &other, idx_t offset, idx_t other_size) override;
+};
+
 //! A fixed-size buffer holds fixed-size segments of data. It lazily deserializes a buffer, if on-disk and not
 //! yet in memory, and it only serializes dirty and non-written buffers to disk during
 //! serialization.
 class FixedSizeBuffer {
+public:
+	//! Constants for fast offset calculations in the bitmask
+	static constexpr idx_t BASE[] = {0x00000000FFFFFFFF, 0x0000FFFF, 0x00FF, 0x0F, 0x3, 0x1};
+	static constexpr uint8_t SHIFT[] = {32, 16, 8, 4, 2, 1};
+
 public:
 	//! Constructor for a new in-memory buffer
 	explicit FixedSizeBuffer(BlockManager &block_manager);
 	//! Constructor for deserializing buffer metadata from disk
-	FixedSizeBuffer(BlockManager &block_manager, const idx_t segment_count, const block_id_t &block_id);
+	FixedSizeBuffer(BlockManager &block_manager, const idx_t segment_count, const idx_t allocation_size,
+	                const BlockPointer &block_pointer);
 
 	//! Block manager of the database instance
 	BlockManager &block_manager;
 
 	//! The number of allocated segments
 	idx_t segment_count;
+	//! The size of allocated memory in this buffer (necessary for copying while pinning)
+	idx_t allocation_size;
 
 	//! True: the in-memory buffer is no longer consistent with a (possibly existing) copy on disk
 	bool dirty;
 	//! True: can be vacuumed after the vacuum operation
 	bool vacuum;
 
+	//! Partial block id and offset
+	BlockPointer block_pointer;
+
 public:
 	//! Returns true, if the buffer is in-memory
 	inline bool InMemory() const {
@@ -45,12 +70,7 @@ public:
 	}
 	//! Returns true, if the block is on-disk
 	inline bool OnDisk() const {
-		return (block_handle != nullptr) && (block_handle->BlockId() < MAXIMUM_BLOCK);
-	}
-	//! Returns the block ID
-	inline block_id_t BlockId() const {
-		D_ASSERT(OnDisk());
-		return block_handle->BlockId();
+		return block_pointer.IsValid();
 	}
 	//! Returns a pointer to the buffer in memory, and calls Deserialize, if the buffer is not in memory
 	inline data_ptr_t Get(const bool dirty_p = true) {
@@ -65,15 +85,25 @@ public:
 	//! Destroys the in-memory buffer and the on-disk block
 	void Destroy();
 	//! Serializes a buffer (if dirty or not on disk)
-	void Serialize();
+	void Serialize(PartialBlockManager &partial_block_manager, const idx_t available_segments, const idx_t segment_size,
+	               const idx_t bitmask_offset);
 	//! Pin a buffer (if not in-memory)
 	void Pin();
+	//! Returns the first free offset in a bitmask
+	uint32_t GetOffset(const idx_t bitmask_count);
 
 private:
 	//! The buffer handle of the in-memory buffer
 	BufferHandle buffer_handle;
 	//! The block handle of the on-disk buffer
 	shared_ptr<BlockHandle> block_handle;
+
+private:
+	//! Returns the maximum non-free offset in a bitmask
+	uint32_t GetMaxOffset(const idx_t available_segments_per_buffer);
+	//! Sets all uninitialized regions of a buffer in the respective partial block allocation
+	void SetUninitializedRegions(PartialBlockForIndex &p_block_for_index, const idx_t segment_size, const idx_t offset,
+	                             const idx_t bitmask_offset);
 };
 
 } // namespace duckdb