duckdb 0.8.2-dev4474.0 → 0.8.2-dev4572.0

package/src/duckdb/src/include/duckdb/function/compression_function.hpp

@@ -14,6 +14,7 @@
 #include "duckdb/common/map.hpp"
 #include "duckdb/storage/storage_info.hpp"
 #include "duckdb/common/mutex.hpp"
+#include "duckdb/storage/data_pointer.hpp"
 
 namespace duckdb {
 class DatabaseInstance;
@@ -21,6 +22,7 @@ class ColumnData;
 class ColumnDataCheckpointer;
 class ColumnSegment;
 class SegmentStatistics;
+struct ColumnSegmentState;
 
 struct ColumnFetchState;
 struct ColumnScanState;
@@ -62,6 +64,11 @@ struct CompressedSegmentState {
 	virtual ~CompressedSegmentState() {
 	}
 
+	//! Display info for PRAGMA storage_info
+	virtual string GetSegmentInfo() const { // LCOV_EXCL_START
+		return "";
+	} // LCOV_EXCL_STOP
+
 	template <class TARGET>
 	TARGET &Cast() {
 		D_ASSERT(dynamic_cast<TARGET *>(this));
@@ -75,7 +82,7 @@ struct CompressedSegmentState {
 };
 
 struct CompressionAppendState {
-	CompressionAppendState(BufferHandle handle_p) : handle(std::move(handle_p)) {
+	explicit CompressionAppendState(BufferHandle handle_p) : handle(std::move(handle_p)) {
 	}
 	virtual ~CompressionAppendState() {
 	}
@@ -139,13 +146,24 @@ typedef void (*compression_skip_t)(ColumnSegment &segment, ColumnScanState &stat
 //===--------------------------------------------------------------------===//
 // Append (optional)
 //===--------------------------------------------------------------------===//
-typedef unique_ptr<CompressedSegmentState> (*compression_init_segment_t)(ColumnSegment &segment, block_id_t block_id);
+typedef unique_ptr<CompressedSegmentState> (*compression_init_segment_t)(
+    ColumnSegment &segment, block_id_t block_id, optional_ptr<ColumnSegmentState> segment_state);
 typedef unique_ptr<CompressionAppendState> (*compression_init_append_t)(ColumnSegment &segment);
 typedef idx_t (*compression_append_t)(CompressionAppendState &append_state, ColumnSegment &segment,
                                       SegmentStatistics &stats, UnifiedVectorFormat &data, idx_t offset, idx_t count);
 typedef idx_t (*compression_finalize_append_t)(ColumnSegment &segment, SegmentStatistics &stats);
 typedef void (*compression_revert_append_t)(ColumnSegment &segment, idx_t start_row);
 
+//===--------------------------------------------------------------------===//
+// Serialization (optional)
+//===--------------------------------------------------------------------===//
+//! Function prototype for serializing the segment state
+typedef unique_ptr<ColumnSegmentState> (*compression_serialize_state_t)(ColumnSegment &segment);
+//! Function prototype for deserializing the segment state
+typedef unique_ptr<ColumnSegmentState> (*compression_deserialize_state_t)(Deserializer &deserializer);
+//! Function prototype for cleaning up the segment state when the column data is dropped
+typedef void (*compression_cleanup_state_t)(ColumnSegment &segment);
+
 class CompressionFunction {
 public:
 	CompressionFunction(CompressionType type, PhysicalType data_type, compression_init_analyze_t init_analyze,
@@ -157,12 +175,16 @@ public:
 	                    compression_init_segment_t init_segment = nullptr,
 	                    compression_init_append_t init_append = nullptr, compression_append_t append = nullptr,
 	                    compression_finalize_append_t finalize_append = nullptr,
-	                    compression_revert_append_t revert_append = nullptr)
+	                    compression_revert_append_t revert_append = nullptr,
+	                    compression_serialize_state_t serialize_state = nullptr,
+	                    compression_deserialize_state_t deserialize_state = nullptr,
+	                    compression_cleanup_state_t cleanup_state = nullptr)
 	    : type(type), data_type(data_type), init_analyze(init_analyze), analyze(analyze), final_analyze(final_analyze),
 	      init_compression(init_compression), compress(compress), compress_finalize(compress_finalize),
 	      init_scan(init_scan), scan_vector(scan_vector), scan_partial(scan_partial), fetch_row(fetch_row), skip(skip),
 	      init_segment(init_segment), init_append(init_append), append(append), finalize_append(finalize_append),
-	      revert_append(revert_append) {
+	      revert_append(revert_append), serialize_state(serialize_state), deserialize_state(deserialize_state),
+	      cleanup_state(cleanup_state) {
 	}
 
 	//! Compression type
@@ -218,6 +240,16 @@ public:
 	compression_finalize_append_t finalize_append;
 	//! Revert append (optional)
 	compression_revert_append_t revert_append;
+
+	// State serialize functions
+	//! This is only necessary if the segment state has information that must be written to disk in the metadata
+
+	//! Serialize the segment state to the metadata (optional)
+	compression_serialize_state_t serialize_state;
+	//! Deserialize the segment state to the metadata (optional)
+	compression_deserialize_state_t deserialize_state;
+	//! Cleanup the segment state (optional)
+	compression_cleanup_state_t cleanup_state;
 };
 
 //! The set of compression functions

package/src/duckdb/src/include/duckdb/function/table/arrow.hpp

@@ -129,6 +129,8 @@ public:
 
 	//! Scan Function
 	static void ArrowScanFunction(ClientContext &context, TableFunctionInput &data, DataChunk &output);
+	static void PopulateArrowTableType(ArrowTableType &arrow_table, ArrowSchemaWrapper &schema_p, vector<string> &names,
+	                                   vector<LogicalType> &return_types);
 
 protected:
 	//! Defines Maximum Number of Threads

package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp

@@ -25,6 +25,10 @@ struct PragmaStorageInfo {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 
+struct PragmaMetadataInfo {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
 struct PragmaLastProfilingOutput {
 	static void RegisterFunction(BuiltinFunctions &set);
 };

package/src/duckdb/src/include/duckdb/optimizer/rule/empty_needle_removal.hpp

@@ -13,7 +13,7 @@
 namespace duckdb {
 
 // The Empty_needle_removal Optimization rule folds some foldable ConstantExpression
-//(e.g.: PREFIX('xyz', '') is TRUE, PREFIX(NULL, '') is NULL, so rewrite PREFIX(x, '') to (CASE WHEN x IS NOT NULL THEN)
+//(e.g.: PREFIX('xyz', '') is TRUE, PREFIX(NULL, '') is NULL, so rewrite PREFIX(x, '') to TRUE_OR_NULL(x)
 class EmptyNeedleRemovalRule : public Rule {
 public:
 	explicit EmptyNeedleRemovalRule(ExpressionRewriter &rewriter);

package/src/duckdb/src/include/duckdb/storage/checkpoint/string_checkpoint_state.hpp

@@ -14,13 +14,16 @@
 #include "duckdb/function/compression_function.hpp"
 
 namespace duckdb {
+struct UncompressedStringSegmentState;
 
 class OverflowStringWriter {
 public:
 	virtual ~OverflowStringWriter() {
 	}
 
-	virtual void WriteString(string_t string, block_id_t &result_block, int32_t &result_offset) = 0;
+	virtual void WriteString(UncompressedStringSegmentState &state, string_t string, block_id_t &result_block,
+	                         int32_t &result_offset) = 0;
+	virtual void Flush() = 0;
 };
 
 struct StringBlock {
@@ -43,15 +46,35 @@ struct string_location_t {
 };
 
 struct UncompressedStringSegmentState : public CompressedSegmentState {
-	~UncompressedStringSegmentState();
+	~UncompressedStringSegmentState() override;
 
 	//! The string block holding strings that do not fit in the main block
 	//! FIXME: this should be replaced by a heap that also allows freeing of unused strings
 	unique_ptr<StringBlock> head;
+	//! Map of block id to string block
+	unordered_map<block_id_t, reference<StringBlock>> overflow_blocks;
 	//! Overflow string writer (if any), if not set overflow strings will be written to memory blocks
 	unique_ptr<OverflowStringWriter> overflow_writer;
-	//! Map of block id to string block
-	unordered_map<block_id_t, StringBlock *> overflow_blocks;
+	//! The set of overflow blocks written to disk (if any)
+	vector<block_id_t> on_disk_blocks;
+
+public:
+	shared_ptr<BlockHandle> GetHandle(BlockManager &manager, block_id_t block_id);
+
+	void RegisterBlock(BlockManager &manager, block_id_t block_id);
+
+	string GetSegmentInfo() const override {
+		if (on_disk_blocks.empty()) {
+			return "";
+		}
+		string result = StringUtil::Join(on_disk_blocks, on_disk_blocks.size(), ", ",
+		                                 [&](block_id_t block) { return to_string(block); });
+		return "Overflow String Block Ids: " + result;
+	}
+
+private:
+	mutex block_lock;
+	unordered_map<block_id_t, shared_ptr<BlockHandle>> handles;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/checkpoint/write_overflow_strings_to_disk.hpp

@@ -30,10 +30,12 @@ public:
 	static constexpr idx_t STRING_SPACE = Storage::BLOCK_SIZE - sizeof(block_id_t);
 
 public:
-	void WriteString(string_t string, block_id_t &result_block, int32_t &result_offset) override;
+	void WriteString(UncompressedStringSegmentState &state, string_t string, block_id_t &result_block,
+	                 int32_t &result_offset) override;
+	void Flush() override;
 
 private:
-	void AllocateNewBlock(block_id_t new_block_id);
+	void AllocateNewBlock(UncompressedStringSegmentState &state, block_id_t new_block_id);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp

@@ -20,8 +20,27 @@ namespace duckdb {
 class Serializer;
 class Deserializer;
 
+struct ColumnSegmentState {
+	virtual ~ColumnSegmentState() {
+	}
+
+	virtual void Serialize(Serializer &serializer) const = 0;
+	static unique_ptr<ColumnSegmentState> Deserialize(Deserializer &deserializer);
+
+	template <class TARGET>
+	TARGET &Cast() {
+		D_ASSERT(dynamic_cast<TARGET *>(this));
+		return reinterpret_cast<TARGET &>(*this);
+	}
+	template <class TARGET>
+	const TARGET &Cast() const {
+		D_ASSERT(dynamic_cast<const TARGET *>(this));
+		return reinterpret_cast<const TARGET &>(*this);
+	}
+};
+
 struct DataPointer {
-	DataPointer(BaseStatistics stats) : statistics(std::move(stats)) {
+	explicit DataPointer(BaseStatistics stats) : statistics(std::move(stats)) {
 	}
 
 	uint64_t row_start;
@@ -30,6 +49,8 @@ struct DataPointer {
 	CompressionType compression_type;
 	//! Type-specific statistics of the segment
 	BaseStatistics statistics;
+	//! Serialized segment state
+	unique_ptr<ColumnSegmentState> segment_state;
 
 	void Serialize(Serializer &serializer) const;
 	static DataPointer Deserialize(Deserializer &source);

package/src/duckdb/src/include/duckdb/storage/database_size.hpp

@@ -21,4 +21,10 @@ struct DatabaseSize {
 	idx_t wal_size = 0;
 };
 
+struct MetadataBlockInfo {
+	block_id_t block_id;
+	idx_t total_blocks;
+	vector<idx_t> free_list;
+};
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp

@@ -16,6 +16,7 @@
 
 namespace duckdb {
 class DatabaseInstance;
+struct MetadataBlockInfo;
 
 struct MetadataBlock {
 	shared_ptr<BlockHandle> block;
@@ -66,6 +67,7 @@ public:
 	void MarkBlocksAsModified();
 	void ClearModifiedBlocks(const vector<MetaBlockPointer> &pointers);
 
+	vector<MetadataBlockInfo> GetMetadataInfo() const;
 	idx_t BlockCount();
 
 	void Write(WriteStream &sink);

package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp

@@ -68,6 +68,7 @@ public:
 	virtual bool IsCheckpointClean(MetaBlockPointer checkpoint_id) = 0;
 	virtual void CreateCheckpoint(bool delete_wal = false, bool force_checkpoint = false) = 0;
 	virtual DatabaseSize GetDatabaseSize() = 0;
+	virtual vector<MetadataBlockInfo> GetMetadataInfo() = 0;
 	virtual shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) = 0;
 
 protected:
@@ -112,6 +113,7 @@ public:
 	bool IsCheckpointClean(MetaBlockPointer checkpoint_id) override;
 	void CreateCheckpoint(bool delete_wal, bool force_checkpoint) override;
 	DatabaseSize GetDatabaseSize() override;
+	vector<MetadataBlockInfo> GetMetadataInfo() override;
 	shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) override;
 
 protected:

package/src/duckdb/src/include/duckdb/storage/string_uncompressed.hpp

@@ -56,7 +56,8 @@ public:
 	static void StringScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result);
 	static void StringFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id, Vector &result,
 	                           idx_t result_idx);
-	static unique_ptr<CompressedSegmentState> StringInitSegment(ColumnSegment &segment, block_id_t block_id);
+	static unique_ptr<CompressedSegmentState> StringInitSegment(ColumnSegment &segment, block_id_t block_id,
+	                                                            optional_ptr<ColumnSegmentState> segment_state);
 
 	static unique_ptr<CompressionAppendState> StringInitAppend(ColumnSegment &segment) {
 		auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
@@ -194,5 +195,9 @@ public:
 	                                    data_ptr_t baseptr, int32_t dict_offset, uint32_t string_length);
 	static string_t FetchString(ColumnSegment &segment, StringDictionaryContainer dict, Vector &result,
 	                            data_ptr_t baseptr, string_location_t location, uint32_t string_length);
+
+	static unique_ptr<ColumnSegmentState> SerializeState(ColumnSegment &segment);
+	static unique_ptr<ColumnSegmentState> DeserializeState(Deserializer &deserializer);
+	static void CleanupState(ColumnSegment &segment);
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp

@@ -57,7 +57,8 @@ public:
 	static unique_ptr<ColumnSegment> CreatePersistentSegment(DatabaseInstance &db, BlockManager &block_manager,
 	                                                         block_id_t id, idx_t offset, const LogicalType &type_p,
 	                                                         idx_t start, idx_t count, CompressionType compression_type,
-	                                                         BaseStatistics statistics);
+	                                                         BaseStatistics statistics,
+	                                                         unique_ptr<ColumnSegmentState> segment_state);
 	static unique_ptr<ColumnSegment> CreateTransientSegment(DatabaseInstance &db, const LogicalType &type, idx_t start,
 	                                                        idx_t segment_size = Storage::BLOCK_SIZE);
 	static unique_ptr<ColumnSegment> CreateSegment(ColumnSegment &other, idx_t start);
@@ -118,14 +119,17 @@ public:
 		return row_index - this->start;
 	}
 
-	CompressedSegmentState *GetSegmentState() {
+	optional_ptr<CompressedSegmentState> GetSegmentState() {
 		return segment_state.get();
 	}
 
+	void CommitDropSegment();
+
 public:
 	ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, LogicalType type, ColumnSegmentType segment_type,
 	              idx_t start, idx_t count, CompressionFunction &function, BaseStatistics statistics,
-	              block_id_t block_id, idx_t offset, idx_t segment_size);
+	              block_id_t block_id, idx_t offset, idx_t segment_size,
+	              unique_ptr<ColumnSegmentState> segment_state = nullptr);
 	ColumnSegment(ColumnSegment &other, idx_t start);
 
 private:

package/src/duckdb/src/include/duckdb/storage/table_storage_info.hpp

@@ -28,6 +28,7 @@ struct ColumnSegmentInfo {
 	bool persistent;
 	block_id_t block_id;
 	idx_t block_offset;
+	string segment_info;
 };
 
 struct IndexInfo {

package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp

@@ -72,17 +72,21 @@ void ColumnLifetimeAnalyzer::VisitOperator(LogicalOperator &op) {
 		for (auto &cond : comp_join.conditions) {
 			if (cond.comparison == ExpressionType::COMPARE_EQUAL) {
 				has_equality = true;
+				break;
 			}
 		}
 		if (!has_equality) {
 			break;
 		}
-		// now, for each of the columns of the RHS, check which columns need to be projected
+		// visit current operator expressions so they are added to the referenced_columns
+		LogicalOperatorVisitor::VisitOperatorExpressions(op);
+
 		column_binding_set_t unused_bindings;
+		auto old_op_bindings = op.GetColumnBindings();
 		ExtractUnusedColumnBindings(op.children[1]->GetColumnBindings(), unused_bindings);
 
 		// now recurse into the filter and its children
-		StandardVisitOperator(op);
+		LogicalOperatorVisitor::VisitOperatorChildren(op);
 
 		// then generate the projection map
 		GenerateProjectionMap(op.children[1]->GetColumnBindings(), unused_bindings, comp_join.right_projection_map);
@@ -118,12 +122,14 @@ void ColumnLifetimeAnalyzer::VisitOperator(LogicalOperator &op) {
 		if (everything_referenced) {
 			break;
 		}
+		// first visit operator expressions to populate referenced columns
+		LogicalOperatorVisitor::VisitOperatorExpressions(op);
 		// filter, figure out which columns are not needed after the filter
 		column_binding_set_t unused_bindings;
 		ExtractUnusedColumnBindings(op.children[0]->GetColumnBindings(), unused_bindings);
 
 		// now recurse into the filter and its children
-		StandardVisitOperator(op);
+		LogicalOperatorVisitor::VisitOperatorChildren(op);
 
 		// then generate the projection map
 		GenerateProjectionMap(op.children[0]->GetColumnBindings(), unused_bindings, filter.projection_map);

package/src/duckdb/src/optimizer/statistics/operator/propagate_projection.cpp

@@ -11,7 +11,6 @@ unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalProj
 		ReplaceWithEmptyResult(*node_ptr);
 		return std::move(node_stats);
 	}
-
 	// then propagate to each of the expressions
 	for (idx_t i = 0; i < proj.expressions.size(); i++) {
 		auto stats = PropagateExpression(proj.expressions[i]);

package/src/duckdb/src/planner/binder/expression/bind_aggregate_expression.cpp

@@ -86,7 +86,7 @@ BindResult BaseSelectBinder::BindAggregate(FunctionExpression &aggr, AggregateFu
 	this->bound_aggregate = true;
 	unique_ptr<Expression> bound_filter;
 	AggregateBinder aggregate_binder(binder, context);
-	string error, filter_error;
+	string error;
 
 	// Now we bind the filter (if any)
 	if (aggr.filter) {
@@ -167,9 +167,6 @@ BindResult BaseSelectBinder::BindAggregate(FunctionExpression &aggr, AggregateFu
 	} else if (depth > 0 && !aggregate_binder.HasBoundColumns()) {
 		return BindResult("Aggregate with only constant parameters has to be bound in the root subquery");
 	}
-	if (!filter_error.empty()) {
-		return BindResult(filter_error);
-	}
 
 	if (aggr.filter) {
 		auto &child = BoundExpression::GetExpression(*aggr.filter);

package/src/duckdb/src/storage/checkpoint/row_group_writer.cpp

@@ -20,10 +20,7 @@ PartialBlockAllocation RowGroupWriter::GetBlockAllocation(uint32_t segment_size)
 void SingleFileRowGroupWriter::WriteColumnDataPointers(ColumnCheckpointState &column_checkpoint_state,
                                                        Serializer &serializer) {
 	const auto &data_pointers = column_checkpoint_state.data_pointers;
-	serializer.WriteList(100, "data_pointers", data_pointers.size(), [&](Serializer::List &list, idx_t i) {
-		auto &data_pointer = data_pointers[i];
-		list.WriteElement(data_pointer);
-	});
+	serializer.WriteProperty(100, "data_pointers", data_pointers);
 }
 
 MetadataWriter &SingleFileRowGroupWriter::GetPayloadWriter() {

package/src/duckdb/src/storage/checkpoint/write_overflow_strings_to_disk.cpp

@@ -10,19 +10,42 @@ WriteOverflowStringsToDisk::WriteOverflowStringsToDisk(BlockManager &block_manag
 }
 
 WriteOverflowStringsToDisk::~WriteOverflowStringsToDisk() {
-	if (offset > 0) {
-		block_manager.Write(handle.GetFileBuffer(), block_id);
+	// verify that the overflow writer has been flushed
+	D_ASSERT(Exception::UncaughtException() || offset == 0);
+}
+
+shared_ptr<BlockHandle> UncompressedStringSegmentState::GetHandle(BlockManager &manager, block_id_t block_id) {
+	lock_guard<mutex> lock(block_lock);
+	auto entry = handles.find(block_id);
+	if (entry != handles.end()) {
+		return entry->second;
 	}
+	auto result = manager.RegisterBlock(block_id);
+	handles.insert(make_pair(block_id, result));
+	return result;
 }
 
-void WriteOverflowStringsToDisk::WriteString(string_t string, block_id_t &result_block, int32_t &result_offset) {
+void UncompressedStringSegmentState::RegisterBlock(BlockManager &manager, block_id_t block_id) {
+	lock_guard<mutex> lock(block_lock);
+	auto entry = handles.find(block_id);
+	if (entry != handles.end()) {
+		throw InternalException("UncompressedStringSegmentState::RegisterBlock - block id %llu already exists",
+		                        block_id);
+	}
+	auto result = manager.RegisterBlock(block_id);
+	handles.insert(make_pair(block_id, std::move(result)));
+	on_disk_blocks.push_back(block_id);
+}
+
+void WriteOverflowStringsToDisk::WriteString(UncompressedStringSegmentState &state, string_t string,
+                                             block_id_t &result_block, int32_t &result_offset) {
 	auto &buffer_manager = block_manager.buffer_manager;
 	if (!handle.IsValid()) {
 		handle = buffer_manager.Allocate(Storage::BLOCK_SIZE);
 	}
 	// first write the length of the string
 	if (block_id == INVALID_BLOCK || offset + 2 * sizeof(uint32_t) >= STRING_SPACE) {
-		AllocateNewBlock(block_manager.GetFreeBlockId());
+		AllocateNewBlock(state, block_manager.GetFreeBlockId());
 	}
 	result_block = block_id;
 	result_offset = offset;
@@ -55,23 +78,37 @@ void WriteOverflowStringsToDisk::WriteString(string_t string, block_id_t &result
 			strptr += to_write;
 		}
 		if (remaining > 0) {
+			D_ASSERT(offset == WriteOverflowStringsToDisk::STRING_SPACE);
 			// there is still remaining stuff to write
-			// first get the new block id and write it to the end of the previous block
-			auto new_block_id = block_manager.GetFreeBlockId();
-			Store<block_id_t>(new_block_id, data_ptr + offset);
 			// now write the current block to disk and allocate a new block
-			AllocateNewBlock(new_block_id);
+			AllocateNewBlock(state, block_manager.GetFreeBlockId());
 		}
 	}
 }
 
-void WriteOverflowStringsToDisk::AllocateNewBlock(block_id_t new_block_id) {
+void WriteOverflowStringsToDisk::Flush() {
+	if (block_id != INVALID_BLOCK && offset > 0) {
+		// zero-initialize the empty part of the overflow string buffer (if any)
+		if (offset < STRING_SPACE) {
+			memset(handle.Ptr() + offset, 0, STRING_SPACE - offset);
+		}
+		// write to disk
+		block_manager.Write(handle.GetFileBuffer(), block_id);
+	}
+	block_id = INVALID_BLOCK;
+	offset = 0;
+}
+
+void WriteOverflowStringsToDisk::AllocateNewBlock(UncompressedStringSegmentState &state, block_id_t new_block_id) {
 	if (block_id != INVALID_BLOCK) {
 		// there is an old block, write it first
-		block_manager.Write(handle.GetFileBuffer(), block_id);
+		// write the new block id at the end of the previous block
+		Store<block_id_t>(new_block_id, handle.Ptr() + WriteOverflowStringsToDisk::STRING_SPACE);
+		Flush();
 	}
 	offset = 0;
 	block_id = new_block_id;
+	state.RegisterBlock(block_manager, new_block_id);
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -136,8 +136,6 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
 
 	// truncate the file
 	block_manager.Truncate();
-
-	metadata_manager.MarkBlocksAsModified();
 }
 
 void CheckpointReader::LoadCheckpoint(ClientContext &context, MetadataReader &reader) {

package/src/duckdb/src/storage/compression/fixed_size_uncompressed.cpp

@@ -65,7 +65,7 @@ void UncompressedCompressState::CreateEmptySegment(idx_t row_start) {
 	auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
 	if (type.InternalType() == PhysicalType::VARCHAR) {
 		auto &state = compressed_segment->GetSegmentState()->Cast<UncompressedStringSegmentState>();
-		state.overflow_writer = make_uniq<WriteOverflowStringsToDisk>(checkpointer.GetColumnData().GetBlockManager());
+		state.overflow_writer = make_uniq<WriteOverflowStringsToDisk>(checkpointer.GetRowGroup().GetBlockManager());
 	}
 	current_segment = std::move(compressed_segment);
 	current_segment->InitializeAppend(append_state);
@@ -73,6 +73,11 @@ void UncompressedCompressState::CreateEmptySegment(idx_t row_start) {
 
 void UncompressedCompressState::FlushSegment(idx_t segment_size) {
 	auto &state = checkpointer.GetCheckpointState();
+	if (current_segment->type.InternalType() == PhysicalType::VARCHAR) {
+		auto &segment_state = current_segment->GetSegmentState()->Cast<UncompressedStringSegmentState>();
+		segment_state.overflow_writer->Flush();
+		segment_state.overflow_writer.reset();
+	}
 	state.FlushSegment(std::move(current_segment), segment_size);
 }