duckdb 0.8.2-dev4514.0 → 0.8.2-dev4572.0

package/src/duckdb/src/include/duckdb/function/compression_function.hpp

@@ -14,6 +14,7 @@
 #include "duckdb/common/map.hpp"
 #include "duckdb/storage/storage_info.hpp"
 #include "duckdb/common/mutex.hpp"
+#include "duckdb/storage/data_pointer.hpp"
 
 namespace duckdb {
 class DatabaseInstance;
@@ -21,6 +22,7 @@ class ColumnData;
 class ColumnDataCheckpointer;
 class ColumnSegment;
 class SegmentStatistics;
+struct ColumnSegmentState;
 
 struct ColumnFetchState;
 struct ColumnScanState;
@@ -62,6 +64,11 @@ struct CompressedSegmentState {
 	virtual ~CompressedSegmentState() {
 	}
 
+	//! Display info for PRAGMA storage_info
+	virtual string GetSegmentInfo() const { // LCOV_EXCL_START
+		return "";
+	} // LCOV_EXCL_STOP
+
 	template <class TARGET>
 	TARGET &Cast() {
 		D_ASSERT(dynamic_cast<TARGET *>(this));
@@ -75,7 +82,7 @@ struct CompressedSegmentState {
 };
 
 struct CompressionAppendState {
-	CompressionAppendState(BufferHandle handle_p) : handle(std::move(handle_p)) {
+	explicit CompressionAppendState(BufferHandle handle_p) : handle(std::move(handle_p)) {
 	}
 	virtual ~CompressionAppendState() {
 	}
@@ -139,13 +146,24 @@ typedef void (*compression_skip_t)(ColumnSegment &segment, ColumnScanState &stat
 //===--------------------------------------------------------------------===//
 // Append (optional)
 //===--------------------------------------------------------------------===//
-typedef unique_ptr<CompressedSegmentState> (*compression_init_segment_t)(ColumnSegment &segment, block_id_t block_id);
+typedef unique_ptr<CompressedSegmentState> (*compression_init_segment_t)(
+    ColumnSegment &segment, block_id_t block_id, optional_ptr<ColumnSegmentState> segment_state);
 typedef unique_ptr<CompressionAppendState> (*compression_init_append_t)(ColumnSegment &segment);
 typedef idx_t (*compression_append_t)(CompressionAppendState &append_state, ColumnSegment &segment,
                                       SegmentStatistics &stats, UnifiedVectorFormat &data, idx_t offset, idx_t count);
 typedef idx_t (*compression_finalize_append_t)(ColumnSegment &segment, SegmentStatistics &stats);
 typedef void (*compression_revert_append_t)(ColumnSegment &segment, idx_t start_row);
 
+//===--------------------------------------------------------------------===//
+// Serialization (optional)
+//===--------------------------------------------------------------------===//
+//! Function prototype for serializing the segment state
+typedef unique_ptr<ColumnSegmentState> (*compression_serialize_state_t)(ColumnSegment &segment);
+//! Function prototype for deserializing the segment state
+typedef unique_ptr<ColumnSegmentState> (*compression_deserialize_state_t)(Deserializer &deserializer);
+//! Function prototype for cleaning up the segment state when the column data is dropped
+typedef void (*compression_cleanup_state_t)(ColumnSegment &segment);
+
 class CompressionFunction {
 public:
 	CompressionFunction(CompressionType type, PhysicalType data_type, compression_init_analyze_t init_analyze,
@@ -157,12 +175,16 @@ public:
 	                    compression_init_segment_t init_segment = nullptr,
 	                    compression_init_append_t init_append = nullptr, compression_append_t append = nullptr,
 	                    compression_finalize_append_t finalize_append = nullptr,
-	                    compression_revert_append_t revert_append = nullptr)
+	                    compression_revert_append_t revert_append = nullptr,
+	                    compression_serialize_state_t serialize_state = nullptr,
+	                    compression_deserialize_state_t deserialize_state = nullptr,
+	                    compression_cleanup_state_t cleanup_state = nullptr)
 	    : type(type), data_type(data_type), init_analyze(init_analyze), analyze(analyze), final_analyze(final_analyze),
 	      init_compression(init_compression), compress(compress), compress_finalize(compress_finalize),
 	      init_scan(init_scan), scan_vector(scan_vector), scan_partial(scan_partial), fetch_row(fetch_row), skip(skip),
 	      init_segment(init_segment), init_append(init_append), append(append), finalize_append(finalize_append),
-	      revert_append(revert_append) {
+	      revert_append(revert_append), serialize_state(serialize_state), deserialize_state(deserialize_state),
+	      cleanup_state(cleanup_state) {
 	}
 
 	//! Compression type
@@ -218,6 +240,16 @@ public:
 	compression_finalize_append_t finalize_append;
 	//! Revert append (optional)
 	compression_revert_append_t revert_append;
+
+	// State serialize functions
+	//! This is only necessary if the segment state has information that must be written to disk in the metadata
+
+	//! Serialize the segment state to the metadata (optional)
+	compression_serialize_state_t serialize_state;
+	//! Deserialize the segment state to the metadata (optional)
+	compression_deserialize_state_t deserialize_state;
+	//! Cleanup the segment state (optional)
+	compression_cleanup_state_t cleanup_state;
 };
 
 //! The set of compression functions

package/src/duckdb/src/include/duckdb/function/table/arrow.hpp

@@ -129,6 +129,8 @@ public:
 
 	//! Scan Function
 	static void ArrowScanFunction(ClientContext &context, TableFunctionInput &data, DataChunk &output);
+	static void PopulateArrowTableType(ArrowTableType &arrow_table, ArrowSchemaWrapper &schema_p, vector<string> &names,
+	                                   vector<LogicalType> &return_types);
 
 protected:
 	//! Defines Maximum Number of Threads

package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp

@@ -25,6 +25,10 @@ struct PragmaStorageInfo {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 
+struct PragmaMetadataInfo {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
 struct PragmaLastProfilingOutput {
 	static void RegisterFunction(BuiltinFunctions &set);
 };

package/src/duckdb/src/include/duckdb/storage/checkpoint/string_checkpoint_state.hpp

@@ -14,13 +14,16 @@
 #include "duckdb/function/compression_function.hpp"
 
 namespace duckdb {
+struct UncompressedStringSegmentState;
 
 class OverflowStringWriter {
 public:
 	virtual ~OverflowStringWriter() {
 	}
 
-	virtual void WriteString(string_t string, block_id_t &result_block, int32_t &result_offset) = 0;
+	virtual void WriteString(UncompressedStringSegmentState &state, string_t string, block_id_t &result_block,
+	                         int32_t &result_offset) = 0;
+	virtual void Flush() = 0;
 };
 
 struct StringBlock {
@@ -43,15 +46,35 @@ struct string_location_t {
 };
 
 struct UncompressedStringSegmentState : public CompressedSegmentState {
-	~UncompressedStringSegmentState();
+	~UncompressedStringSegmentState() override;
 
 	//! The string block holding strings that do not fit in the main block
 	//! FIXME: this should be replaced by a heap that also allows freeing of unused strings
 	unique_ptr<StringBlock> head;
+	//! Map of block id to string block
+	unordered_map<block_id_t, reference<StringBlock>> overflow_blocks;
 	//! Overflow string writer (if any), if not set overflow strings will be written to memory blocks
 	unique_ptr<OverflowStringWriter> overflow_writer;
-	//! Map of block id to string block
-	unordered_map<block_id_t, StringBlock *> overflow_blocks;
+	//! The set of overflow blocks written to disk (if any)
+	vector<block_id_t> on_disk_blocks;
+
+public:
+	shared_ptr<BlockHandle> GetHandle(BlockManager &manager, block_id_t block_id);
+
+	void RegisterBlock(BlockManager &manager, block_id_t block_id);
+
+	string GetSegmentInfo() const override {
+		if (on_disk_blocks.empty()) {
+			return "";
+		}
+		string result = StringUtil::Join(on_disk_blocks, on_disk_blocks.size(), ", ",
+		                                 [&](block_id_t block) { return to_string(block); });
+		return "Overflow String Block Ids: " + result;
+	}
+
+private:
+	mutex block_lock;
+	unordered_map<block_id_t, shared_ptr<BlockHandle>> handles;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/checkpoint/write_overflow_strings_to_disk.hpp

@@ -30,10 +30,12 @@ public:
 	static constexpr idx_t STRING_SPACE = Storage::BLOCK_SIZE - sizeof(block_id_t);
 
 public:
-	void WriteString(string_t string, block_id_t &result_block, int32_t &result_offset) override;
+	void WriteString(UncompressedStringSegmentState &state, string_t string, block_id_t &result_block,
+	                 int32_t &result_offset) override;
+	void Flush() override;
 
 private:
-	void AllocateNewBlock(block_id_t new_block_id);
+	void AllocateNewBlock(UncompressedStringSegmentState &state, block_id_t new_block_id);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp

@@ -20,8 +20,27 @@ namespace duckdb {
 class Serializer;
 class Deserializer;
 
+struct ColumnSegmentState {
+	virtual ~ColumnSegmentState() {
+	}
+
+	virtual void Serialize(Serializer &serializer) const = 0;
+	static unique_ptr<ColumnSegmentState> Deserialize(Deserializer &deserializer);
+
+	template <class TARGET>
+	TARGET &Cast() {
+		D_ASSERT(dynamic_cast<TARGET *>(this));
+		return reinterpret_cast<TARGET &>(*this);
+	}
+	template <class TARGET>
+	const TARGET &Cast() const {
+		D_ASSERT(dynamic_cast<const TARGET *>(this));
+		return reinterpret_cast<const TARGET &>(*this);
+	}
+};
+
 struct DataPointer {
-	DataPointer(BaseStatistics stats) : statistics(std::move(stats)) {
+	explicit DataPointer(BaseStatistics stats) : statistics(std::move(stats)) {
 	}
 
 	uint64_t row_start;
@@ -30,6 +49,8 @@ struct DataPointer {
 	CompressionType compression_type;
 	//! Type-specific statistics of the segment
 	BaseStatistics statistics;
+	//! Serialized segment state
+	unique_ptr<ColumnSegmentState> segment_state;
 
 	void Serialize(Serializer &serializer) const;
 	static DataPointer Deserialize(Deserializer &source);

package/src/duckdb/src/include/duckdb/storage/database_size.hpp

@@ -21,4 +21,10 @@ struct DatabaseSize {
 	idx_t wal_size = 0;
 };
 
+struct MetadataBlockInfo {
+	block_id_t block_id;
+	idx_t total_blocks;
+	vector<idx_t> free_list;
+};
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp

@@ -16,6 +16,7 @@
 
 namespace duckdb {
 class DatabaseInstance;
+struct MetadataBlockInfo;
 
 struct MetadataBlock {
 	shared_ptr<BlockHandle> block;
@@ -66,6 +67,7 @@ public:
 	void MarkBlocksAsModified();
 	void ClearModifiedBlocks(const vector<MetaBlockPointer> &pointers);
 
+	vector<MetadataBlockInfo> GetMetadataInfo() const;
 	idx_t BlockCount();
 
 	void Write(WriteStream &sink);

package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp

@@ -68,6 +68,7 @@ public:
 	virtual bool IsCheckpointClean(MetaBlockPointer checkpoint_id) = 0;
 	virtual void CreateCheckpoint(bool delete_wal = false, bool force_checkpoint = false) = 0;
 	virtual DatabaseSize GetDatabaseSize() = 0;
+	virtual vector<MetadataBlockInfo> GetMetadataInfo() = 0;
 	virtual shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) = 0;
 
 protected:
@@ -112,6 +113,7 @@ public:
 	bool IsCheckpointClean(MetaBlockPointer checkpoint_id) override;
 	void CreateCheckpoint(bool delete_wal, bool force_checkpoint) override;
 	DatabaseSize GetDatabaseSize() override;
+	vector<MetadataBlockInfo> GetMetadataInfo() override;
 	shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) override;
 
 protected:

package/src/duckdb/src/include/duckdb/storage/string_uncompressed.hpp

@@ -56,7 +56,8 @@ public:
 	static void StringScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result);
 	static void StringFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id, Vector &result,
 	                           idx_t result_idx);
-	static unique_ptr<CompressedSegmentState> StringInitSegment(ColumnSegment &segment, block_id_t block_id);
+	static unique_ptr<CompressedSegmentState> StringInitSegment(ColumnSegment &segment, block_id_t block_id,
+	                                                            optional_ptr<ColumnSegmentState> segment_state);
 
 	static unique_ptr<CompressionAppendState> StringInitAppend(ColumnSegment &segment) {
 		auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
@@ -194,5 +195,9 @@ public:
 	                                    data_ptr_t baseptr, int32_t dict_offset, uint32_t string_length);
 	static string_t FetchString(ColumnSegment &segment, StringDictionaryContainer dict, Vector &result,
 	                            data_ptr_t baseptr, string_location_t location, uint32_t string_length);
+
+	static unique_ptr<ColumnSegmentState> SerializeState(ColumnSegment &segment);
+	static unique_ptr<ColumnSegmentState> DeserializeState(Deserializer &deserializer);
+	static void CleanupState(ColumnSegment &segment);
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp

@@ -57,7 +57,8 @@ public:
 	static unique_ptr<ColumnSegment> CreatePersistentSegment(DatabaseInstance &db, BlockManager &block_manager,
 	                                                         block_id_t id, idx_t offset, const LogicalType &type_p,
 	                                                         idx_t start, idx_t count, CompressionType compression_type,
-	                                                         BaseStatistics statistics);
+	                                                         BaseStatistics statistics,
+	                                                         unique_ptr<ColumnSegmentState> segment_state);
 	static unique_ptr<ColumnSegment> CreateTransientSegment(DatabaseInstance &db, const LogicalType &type, idx_t start,
 	                                                        idx_t segment_size = Storage::BLOCK_SIZE);
 	static unique_ptr<ColumnSegment> CreateSegment(ColumnSegment &other, idx_t start);
@@ -118,14 +119,17 @@ public:
 		return row_index - this->start;
 	}
 
-	CompressedSegmentState *GetSegmentState() {
+	optional_ptr<CompressedSegmentState> GetSegmentState() {
 		return segment_state.get();
 	}
 
+	void CommitDropSegment();
+
 public:
 	ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, LogicalType type, ColumnSegmentType segment_type,
 	              idx_t start, idx_t count, CompressionFunction &function, BaseStatistics statistics,
-	              block_id_t block_id, idx_t offset, idx_t segment_size);
+	              block_id_t block_id, idx_t offset, idx_t segment_size,
+	              unique_ptr<ColumnSegmentState> segment_state = nullptr);
 	ColumnSegment(ColumnSegment &other, idx_t start);
 
 private:

package/src/duckdb/src/include/duckdb/storage/table_storage_info.hpp

@@ -28,6 +28,7 @@ struct ColumnSegmentInfo {
 	bool persistent;
 	block_id_t block_id;
 	idx_t block_offset;
+	string segment_info;
 };
 
 struct IndexInfo {

package/src/duckdb/src/planner/binder/expression/bind_aggregate_expression.cpp

@@ -86,7 +86,7 @@ BindResult BaseSelectBinder::BindAggregate(FunctionExpression &aggr, AggregateFu
 	this->bound_aggregate = true;
 	unique_ptr<Expression> bound_filter;
 	AggregateBinder aggregate_binder(binder, context);
-	string error, filter_error;
+	string error;
 
 	// Now we bind the filter (if any)
 	if (aggr.filter) {
@@ -167,9 +167,6 @@ BindResult BaseSelectBinder::BindAggregate(FunctionExpression &aggr, AggregateFu
 	} else if (depth > 0 && !aggregate_binder.HasBoundColumns()) {
 		return BindResult("Aggregate with only constant parameters has to be bound in the root subquery");
 	}
-	if (!filter_error.empty()) {
-		return BindResult(filter_error);
-	}
 
 	if (aggr.filter) {
 		auto &child = BoundExpression::GetExpression(*aggr.filter);

package/src/duckdb/src/storage/checkpoint/row_group_writer.cpp

@@ -20,10 +20,7 @@ PartialBlockAllocation RowGroupWriter::GetBlockAllocation(uint32_t segment_size)
 void SingleFileRowGroupWriter::WriteColumnDataPointers(ColumnCheckpointState &column_checkpoint_state,
                                                        Serializer &serializer) {
 	const auto &data_pointers = column_checkpoint_state.data_pointers;
-	serializer.WriteList(100, "data_pointers", data_pointers.size(), [&](Serializer::List &list, idx_t i) {
-		auto &data_pointer = data_pointers[i];
-		list.WriteElement(data_pointer);
-	});
+	serializer.WriteProperty(100, "data_pointers", data_pointers);
 }
 
 MetadataWriter &SingleFileRowGroupWriter::GetPayloadWriter() {

package/src/duckdb/src/storage/checkpoint/write_overflow_strings_to_disk.cpp

@@ -10,19 +10,42 @@ WriteOverflowStringsToDisk::WriteOverflowStringsToDisk(BlockManager &block_manag
 }
 
 WriteOverflowStringsToDisk::~WriteOverflowStringsToDisk() {
-	if (offset > 0) {
-		block_manager.Write(handle.GetFileBuffer(), block_id);
+	// verify that the overflow writer has been flushed
+	D_ASSERT(Exception::UncaughtException() || offset == 0);
+}
+
+shared_ptr<BlockHandle> UncompressedStringSegmentState::GetHandle(BlockManager &manager, block_id_t block_id) {
+	lock_guard<mutex> lock(block_lock);
+	auto entry = handles.find(block_id);
+	if (entry != handles.end()) {
+		return entry->second;
 	}
+	auto result = manager.RegisterBlock(block_id);
+	handles.insert(make_pair(block_id, result));
+	return result;
 }
 
-void WriteOverflowStringsToDisk::WriteString(string_t string, block_id_t &result_block, int32_t &result_offset) {
+void UncompressedStringSegmentState::RegisterBlock(BlockManager &manager, block_id_t block_id) {
+	lock_guard<mutex> lock(block_lock);
+	auto entry = handles.find(block_id);
+	if (entry != handles.end()) {
+		throw InternalException("UncompressedStringSegmentState::RegisterBlock - block id %llu already exists",
+		                        block_id);
+	}
+	auto result = manager.RegisterBlock(block_id);
+	handles.insert(make_pair(block_id, std::move(result)));
+	on_disk_blocks.push_back(block_id);
+}
+
+void WriteOverflowStringsToDisk::WriteString(UncompressedStringSegmentState &state, string_t string,
+                                             block_id_t &result_block, int32_t &result_offset) {
 	auto &buffer_manager = block_manager.buffer_manager;
 	if (!handle.IsValid()) {
 		handle = buffer_manager.Allocate(Storage::BLOCK_SIZE);
 	}
 	// first write the length of the string
 	if (block_id == INVALID_BLOCK || offset + 2 * sizeof(uint32_t) >= STRING_SPACE) {
-		AllocateNewBlock(block_manager.GetFreeBlockId());
+		AllocateNewBlock(state, block_manager.GetFreeBlockId());
 	}
 	result_block = block_id;
 	result_offset = offset;
@@ -55,23 +78,37 @@ void WriteOverflowStringsToDisk::WriteString(string_t string, block_id_t &result
 			strptr += to_write;
 		}
 		if (remaining > 0) {
+			D_ASSERT(offset == WriteOverflowStringsToDisk::STRING_SPACE);
 			// there is still remaining stuff to write
-			// first get the new block id and write it to the end of the previous block
-			auto new_block_id = block_manager.GetFreeBlockId();
-			Store<block_id_t>(new_block_id, data_ptr + offset);
 			// now write the current block to disk and allocate a new block
-			AllocateNewBlock(new_block_id);
+			AllocateNewBlock(state, block_manager.GetFreeBlockId());
 		}
 	}
 }
 
-void WriteOverflowStringsToDisk::AllocateNewBlock(block_id_t new_block_id) {
+void WriteOverflowStringsToDisk::Flush() {
+	if (block_id != INVALID_BLOCK && offset > 0) {
+		// zero-initialize the empty part of the overflow string buffer (if any)
+		if (offset < STRING_SPACE) {
+			memset(handle.Ptr() + offset, 0, STRING_SPACE - offset);
+		}
+		// write to disk
+		block_manager.Write(handle.GetFileBuffer(), block_id);
+	}
+	block_id = INVALID_BLOCK;
+	offset = 0;
+}
+
+void WriteOverflowStringsToDisk::AllocateNewBlock(UncompressedStringSegmentState &state, block_id_t new_block_id) {
 	if (block_id != INVALID_BLOCK) {
 		// there is an old block, write it first
-		block_manager.Write(handle.GetFileBuffer(), block_id);
+		// write the new block id at the end of the previous block
+		Store<block_id_t>(new_block_id, handle.Ptr() + WriteOverflowStringsToDisk::STRING_SPACE);
+		Flush();
 	}
 	offset = 0;
 	block_id = new_block_id;
+	state.RegisterBlock(block_manager, new_block_id);
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -136,8 +136,6 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
 
 	// truncate the file
 	block_manager.Truncate();
-
-	metadata_manager.MarkBlocksAsModified();
 }
 
 void CheckpointReader::LoadCheckpoint(ClientContext &context, MetadataReader &reader) {

package/src/duckdb/src/storage/compression/fixed_size_uncompressed.cpp

@@ -65,7 +65,7 @@ void UncompressedCompressState::CreateEmptySegment(idx_t row_start) {
 	auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
 	if (type.InternalType() == PhysicalType::VARCHAR) {
 		auto &state = compressed_segment->GetSegmentState()->Cast<UncompressedStringSegmentState>();
-		state.overflow_writer = make_uniq<WriteOverflowStringsToDisk>(checkpointer.GetColumnData().GetBlockManager());
+		state.overflow_writer = make_uniq<WriteOverflowStringsToDisk>(checkpointer.GetRowGroup().GetBlockManager());
 	}
 	current_segment = std::move(compressed_segment);
 	current_segment->InitializeAppend(append_state);
@@ -73,6 +73,11 @@ void UncompressedCompressState::CreateEmptySegment(idx_t row_start) {
 
 void UncompressedCompressState::FlushSegment(idx_t segment_size) {
 	auto &state = checkpointer.GetCheckpointState();
+	if (current_segment->type.InternalType() == PhysicalType::VARCHAR) {
+		auto &segment_state = current_segment->GetSegmentState()->Cast<UncompressedStringSegmentState>();
+		segment_state.overflow_writer->Flush();
+		segment_state.overflow_writer.reset();
+	}
 	state.FlushSegment(std::move(current_segment), segment_size);
 }
 

package/src/duckdb/src/storage/compression/string_uncompressed.cpp

@@ -3,6 +3,9 @@
 #include "duckdb/common/pair.hpp"
 #include "duckdb/storage/checkpoint/write_overflow_strings_to_disk.hpp"
 #include "miniz_wrapper.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/storage/table/column_data.hpp"
 
 namespace duckdb {
 
@@ -141,9 +144,22 @@ void UncompressedStringStorage::StringFetchRow(ColumnSegment &segment, ColumnFet
 //===--------------------------------------------------------------------===//
 // Append
 //===--------------------------------------------------------------------===//
+struct SerializedStringSegmentState : public ColumnSegmentState {
+	SerializedStringSegmentState() {
+	}
+	explicit SerializedStringSegmentState(vector<block_id_t> blocks_p) : blocks(std::move(blocks_p)) {
+	}
+
+	vector<block_id_t> blocks;
+
+	void Serialize(Serializer &serializer) const override {
+		serializer.WriteProperty(1, "overflow_blocks", blocks);
+	}
+};
 
-unique_ptr<CompressedSegmentState> UncompressedStringStorage::StringInitSegment(ColumnSegment &segment,
-                                                                                block_id_t block_id) {
+unique_ptr<CompressedSegmentState>
+UncompressedStringStorage::StringInitSegment(ColumnSegment &segment, block_id_t block_id,
+                                             optional_ptr<ColumnSegmentState> segment_state) {
 	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 	if (block_id == INVALID_BLOCK) {
 		auto handle = buffer_manager.Pin(segment.block);
@@ -152,7 +168,12 @@ unique_ptr<CompressedSegmentState> UncompressedStringStorage::StringInitSegment(
 		dictionary.end = segment.SegmentSize();
 		SetDictionary(segment, handle, dictionary);
 	}
-	return make_uniq<UncompressedStringSegmentState>();
+	auto result = make_uniq<UncompressedStringSegmentState>();
+	if (segment_state) {
+		auto &serialized_state = segment_state->Cast<SerializedStringSegmentState>();
+		result->on_disk_blocks = std::move(serialized_state.blocks);
+	}
+	return std::move(result);
 }
 
 idx_t UncompressedStringStorage::FinalizeAppend(ColumnSegment &segment, SegmentStatistics &stats) {
@@ -179,6 +200,32 @@ idx_t UncompressedStringStorage::FinalizeAppend(ColumnSegment &segment, SegmentS
 	return total_size;
 }
 
+//===--------------------------------------------------------------------===//
+// Serialization & Cleanup
+//===--------------------------------------------------------------------===//
+unique_ptr<ColumnSegmentState> UncompressedStringStorage::SerializeState(ColumnSegment &segment) {
+	auto &state = segment.GetSegmentState()->Cast<UncompressedStringSegmentState>();
+	if (state.on_disk_blocks.empty()) {
+		// no on-disk blocks - nothing to write
+		return nullptr;
+	}
+	return make_uniq<SerializedStringSegmentState>(state.on_disk_blocks);
+}
+
+unique_ptr<ColumnSegmentState> UncompressedStringStorage::DeserializeState(Deserializer &deserializer) {
+	auto result = make_uniq<SerializedStringSegmentState>();
+	deserializer.ReadProperty(1, "overflow_blocks", result->blocks);
+	return std::move(result);
+}
+
+void UncompressedStringStorage::CleanupState(ColumnSegment &segment) {
+	auto &state = segment.GetSegmentState()->Cast<UncompressedStringSegmentState>();
+	auto &block_manager = segment.GetBlockManager();
+	for (auto &block_id : state.on_disk_blocks) {
+		block_manager.MarkBlockAsModified(block_id);
+	}
+}
+
 //===--------------------------------------------------------------------===//
 // Get Function
 //===--------------------------------------------------------------------===//
@@ -192,7 +239,9 @@ CompressionFunction StringUncompressed::GetFunction(PhysicalType data_type) {
 	                           UncompressedStringStorage::StringScanPartial, UncompressedStringStorage::StringFetchRow,
 	                           UncompressedFunctions::EmptySkip, UncompressedStringStorage::StringInitSegment,
 	                           UncompressedStringStorage::StringInitAppend, UncompressedStringStorage::StringAppend,
-	                           UncompressedStringStorage::FinalizeAppend);
+	                           UncompressedStringStorage::FinalizeAppend, nullptr,
+	                           UncompressedStringStorage::SerializeState, UncompressedStringStorage::DeserializeState,
+	                           UncompressedStringStorage::CleanupState);
 }
 
 //===--------------------------------------------------------------------===//
@@ -226,7 +275,7 @@ void UncompressedStringStorage::WriteString(ColumnSegment &segment, string_t str
 	auto &state = segment.GetSegmentState()->Cast<UncompressedStringSegmentState>();
 	if (state.overflow_writer) {
 		// overflow writer is set: write string there
-		state.overflow_writer->WriteString(string, result_block, result_offset);
+		state.overflow_writer->WriteString(state, string, result_block, result_offset);
 	} else {
 		// default overflow behavior: use in-memory buffer to store the overflow string
 		WriteStringMemory(segment, string, result_block, result_offset);
@@ -251,7 +300,7 @@ void UncompressedStringStorage::WriteStringMemory(ColumnSegment &segment, string
 		new_block->size = alloc_size;
 		// allocate an in-memory buffer for it
 		handle = buffer_manager.Allocate(alloc_size, false, &block);
-		state.overflow_blocks[block->BlockId()] = new_block.get();
+		state.overflow_blocks.insert(make_pair(block->BlockId(), reference<StringBlock>(*new_block)));
 		new_block->block = std::move(block);
 		new_block->next = std::move(state.head);
 		state.head = std::move(new_block);
@@ -282,7 +331,7 @@ string_t UncompressedStringStorage::ReadOverflowString(ColumnSegment &segment, V
 	if (block < MAXIMUM_BLOCK) {
 		// read the overflow string from disk
 		// pin the initial handle and read the length
-		auto block_handle = block_manager.RegisterBlock(block);
+		auto block_handle = state.GetHandle(block_manager, block);
 		auto handle = buffer_manager.Pin(block_handle);
 
 		// read header
@@ -295,7 +344,7 @@ string_t UncompressedStringStorage::ReadOverflowString(ColumnSegment &segment, V
 		unsafe_unique_array<data_t> decompression_buffer;
 
 		// If string is in single block we decompress straight from it, else we copy first
-		if (remaining <= Storage::BLOCK_SIZE - sizeof(block_id_t) - offset) {
+		if (remaining <= WriteOverflowStringsToDisk::STRING_SPACE - offset) {
 			decompression_ptr = handle.Ptr() + offset;
 		} else {
 			decompression_buffer = make_unsafe_uniq_array<data_t>(compressed_size);
@@ -303,7 +352,7 @@ string_t UncompressedStringStorage::ReadOverflowString(ColumnSegment &segment, V
 
 			// now append the string to the single buffer
 			while (remaining > 0) {
-				idx_t to_write = MinValue<idx_t>(remaining, Storage::BLOCK_SIZE - sizeof(block_id_t) - offset);
+				idx_t to_write = MinValue<idx_t>(remaining, WriteOverflowStringsToDisk::STRING_SPACE - offset);
 				memcpy(target_ptr, handle.Ptr() + offset, to_write);
 
 				remaining -= to_write;
@@ -311,8 +360,9 @@ string_t UncompressedStringStorage::ReadOverflowString(ColumnSegment &segment, V
 				target_ptr += to_write;
 				if (remaining > 0) {
 					// read the next block
-					block_id_t next_block = Load<block_id_t>(handle.Ptr() + offset);
-					block_handle = block_manager.RegisterBlock(next_block);
+					D_ASSERT(offset == WriteOverflowStringsToDisk::STRING_SPACE);
+					block_id_t next_block = Load<block_id_t>(handle.Ptr() + WriteOverflowStringsToDisk::STRING_SPACE);
+					block_handle = state.GetHandle(block_manager, next_block);
 					handle = buffer_manager.Pin(block_handle);
 					offset = 0;
 				}
@@ -336,7 +386,7 @@ string_t UncompressedStringStorage::ReadOverflowString(ColumnSegment &segment, V
 		// first pin the handle, if it is not pinned yet
 		auto entry = state.overflow_blocks.find(block);
 		D_ASSERT(entry != state.overflow_blocks.end());
-		auto handle = buffer_manager.Pin(entry->second->block);
+		auto handle = buffer_manager.Pin(entry->second.get().block);
 		auto final_buffer = handle.Ptr();
 		StringVector::AddHandle(result, std::move(handle));
 		return ReadStringWithLength(final_buffer, offset);

package/src/duckdb/src/storage/compression/validity_uncompressed.cpp

@@ -398,7 +398,8 @@ static unique_ptr<CompressionAppendState> ValidityInitAppend(ColumnSegment &segm
 	return make_uniq<CompressionAppendState>(std::move(handle));
 }
 
-unique_ptr<CompressedSegmentState> ValidityInitSegment(ColumnSegment &segment, block_id_t block_id) {
+unique_ptr<CompressedSegmentState> ValidityInitSegment(ColumnSegment &segment, block_id_t block_id,
+                                                       optional_ptr<ColumnSegmentState> segment_state) {
 	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 	if (block_id == INVALID_BLOCK) {
 		auto handle = buffer_manager.Pin(segment.block);